1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2008-2011 Kay Sievers
7 Copyright 2012 Lennart Poettering
9 systemd is free software; you can redistribute it and/or modify it
10 under the terms of the GNU Lesser General Public License as published by
11 the Free Software Foundation; either version 2.1 of the License, or
12 (at your option) any later version.
14 systemd is distributed in the hope that it will be useful, but
15 WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 Lesser General Public License for more details.
19 You should have received a copy of the GNU Lesser General Public License
20 along with systemd; If not, see <http://www.gnu.org/licenses/>.
23 /* Parts of this file are based on the GLIB utf8 validation functions. The
24 * original license text follows. */
26 /* gutf8.c - Operations on UTF-8 strings.
28 * Copyright (C) 1999 Tom Tromey
29 * Copyright (C) 2000 Red Hat, Inc.
31 * This library is free software; you can redistribute it and/or
32 * modify it under the terms of the GNU Library General Public
33 * License as published by the Free Software Foundation; either
34 * version 2 of the License, or (at your option) any later version.
36 * This library is distributed in the hope that it will be useful,
37 * but WITHOUT ANY WARRANTY; without even the implied warranty of
38 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
39 * Library General Public License for more details.
41 * You should have received a copy of the GNU Library General Public
42 * License along with this library; if not, write to the Free Software
43 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
52 #include "alloc-util.h"
53 #include "hexdecoct.h"
57 bool unichar_is_valid(uint32_t ch) {
59 if (ch >= 0x110000) /* End of unicode space */
61 if ((ch & 0xFFFFF800) == 0xD800) /* Reserved area for UTF-16 */
63 if ((ch >= 0xFDD0) && (ch <= 0xFDEF)) /* Reserved */
65 if ((ch & 0xFFFE) == 0xFFFE) /* BOM (Byte Order Mark) */
71 static bool unichar_is_control(uint32_t ch) {
74 0 to ' '-1 is the C0 range.
75 DEL=0x7F, and DEL+1 to 0x9F is C1 range.
76 '\t' is in C0 range, but more or less harmless and commonly used.
79 return (ch < ' ' && ch != '\t' && ch != '\n') ||
80 (0x7F <= ch && ch <= 0x9F);
83 /* count of characters used to encode one unicode char */
84 static int utf8_encoded_expected_len(const char *str) {
89 c = (unsigned char) str[0];
92 if ((c & 0xe0) == 0xc0)
94 if ((c & 0xf0) == 0xe0)
96 if ((c & 0xf8) == 0xf0)
98 if ((c & 0xfc) == 0xf8)
100 if ((c & 0xfe) == 0xfc)
106 /* decode one unicode char */
107 int utf8_encoded_to_unichar(const char *str) {
112 len = utf8_encoded_expected_len(str);
118 unichar = str[0] & 0x1f;
121 unichar = (int)str[0] & 0x0f;
124 unichar = (int)str[0] & 0x07;
127 unichar = (int)str[0] & 0x03;
130 unichar = (int)str[0] & 0x01;
136 for (i = 1; i < len; i++) {
137 if (((int)str[i] & 0xc0) != 0x80)
140 unichar |= (int)str[i] & 0x3f;
146 bool utf8_is_printable_newline(const char* str, size_t length, bool newline) {
151 for (p = str; length;) {
152 int encoded_len, val;
154 encoded_len = utf8_encoded_valid_unichar(p);
155 if (encoded_len < 0 ||
156 (size_t) encoded_len > length)
159 val = utf8_encoded_to_unichar(p);
161 unichar_is_control(val) ||
162 (!newline && val == '\n'))
165 length -= encoded_len;
172 const char *utf8_is_valid(const char *str) {
177 for (p = (const uint8_t*) str; *p; ) {
180 len = utf8_encoded_valid_unichar((const char *)p);
190 char *utf8_escape_invalid(const char *str) {
195 p = s = malloc(strlen(str) * 4 + 1);
202 len = utf8_encoded_valid_unichar(str);
204 s = mempcpy(s, str, len);
207 s = stpcpy(s, UTF8_REPLACEMENT_CHARACTER);
217 char *utf8_escape_non_printable(const char *str) {
222 p = s = malloc(strlen(str) * 4 + 1);
229 len = utf8_encoded_valid_unichar(str);
231 if (utf8_is_printable(str, len)) {
232 s = mempcpy(s, str, len);
238 *(s++) = hexchar((int) *str >> 4);
239 *(s++) = hexchar((int) *str);
246 s = stpcpy(s, UTF8_REPLACEMENT_CHARACTER);
256 char *ascii_is_valid(const char *str) {
261 for (p = str; *p; p++)
262 if ((unsigned char) *p >= 128)
269 * utf8_encode_unichar() - Encode single UCS-4 character as UTF-8
270 * @out_utf8: output buffer of at least 4 bytes or NULL
271 * @g: UCS-4 character to encode
273 * This encodes a single UCS-4 character as UTF-8 and writes it into @out_utf8.
274 * The length of the character is returned. It is not zero-terminated! If the
275 * output buffer is NULL, only the length is returned.
277 * Returns: The length in bytes that the UTF-8 representation does or would
280 size_t utf8_encode_unichar(char *out_utf8, uint32_t g) {
284 out_utf8[0] = g & 0x7f;
286 } else if (g < (1 << 11)) {
288 out_utf8[0] = 0xc0 | ((g >> 6) & 0x1f);
289 out_utf8[1] = 0x80 | (g & 0x3f);
292 } else if (g < (1 << 16)) {
294 out_utf8[0] = 0xe0 | ((g >> 12) & 0x0f);
295 out_utf8[1] = 0x80 | ((g >> 6) & 0x3f);
296 out_utf8[2] = 0x80 | (g & 0x3f);
299 } else if (g < (1 << 21)) {
301 out_utf8[0] = 0xf0 | ((g >> 18) & 0x07);
302 out_utf8[1] = 0x80 | ((g >> 12) & 0x3f);
303 out_utf8[2] = 0x80 | ((g >> 6) & 0x3f);
304 out_utf8[3] = 0x80 | (g & 0x3f);
312 char *utf16_to_utf8(const void *s, size_t length) {
316 r = new(char, (length * 4 + 1) / 2 + 1);
323 while (f < (const uint8_t*) s + length) {
326 /* see RFC 2781 section 2.2 */
328 w1 = f[1] << 8 | f[0];
331 if (!utf16_is_surrogate(w1)) {
332 t += utf8_encode_unichar(t, w1);
337 if (utf16_is_trailing_surrogate(w1))
339 else if (f >= (const uint8_t*) s + length)
342 w2 = f[1] << 8 | f[0];
345 if (!utf16_is_trailing_surrogate(w2)) {
350 t += utf8_encode_unichar(t, utf16_surrogate_pair_to_unichar(w1, w2));
357 /* expected size used to encode one unicode char */
358 static int utf8_unichar_to_encoded_len(int unichar) {
364 if (unichar < 0x10000)
366 if (unichar < 0x200000)
368 if (unichar < 0x4000000)
374 /* validate one encoded unicode char and return its length */
375 int utf8_encoded_valid_unichar(const char *str) {
380 len = utf8_encoded_expected_len(str);
388 /* check if expected encoded chars are available */
389 for (i = 0; i < len; i++)
390 if ((str[i] & 0x80) != 0x80)
393 unichar = utf8_encoded_to_unichar(str);
395 /* check if encoded length matches encoded value */
396 if (utf8_unichar_to_encoded_len(unichar) != len)
399 /* check if value has valid range */
400 if (!unichar_is_valid(unichar))