X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?p=elogind.git;a=blobdiff_plain;f=src%2Fshared%2Futf8.c;h=4469a73751c3b5624d10c01a0eac59e74eb4e453;hp=62e2803919555d58af12f3a5960f9ca14812e9f9;hb=6ed62be0b15ac92d9687cff93c63d7b06d11b9f0;hpb=f274ece0f76b5709408821e317e87aef76123db6 diff --git a/src/shared/utf8.c b/src/shared/utf8.c index 62e280391..4469a7375 100644 --- a/src/shared/utf8.c +++ b/src/shared/utf8.c @@ -3,6 +3,7 @@ /*** This file is part of systemd. + Copyright 2008-2011 Kay Sievers Copyright 2012 Lennart Poettering systemd is free software; you can redistribute it and/or modify it @@ -19,7 +20,7 @@ along with systemd; If not, see . ***/ -/* This file is based on the GLIB utf8 validation functions. The +/* Parts of this file are based on the GLIB utf8 validation functions. The * original license text follows. */ /* gutf8.c - Operations on UTF-8 strings. @@ -49,8 +50,7 @@ #include #include "utf8.h" - -#define FILTER_CHAR '_' +#include "util.h" static inline bool is_unicode_valid(uint32_t ch) { @@ -66,17 +66,6 @@ static inline bool is_unicode_valid(uint32_t ch) { return true; } -static inline bool is_continuation_char(uint8_t ch) { - if ((ch & 0xc0) != 0x80) /* 10xxxxxx */ - return false; - return true; -} - -static inline void merge_continuation_char(uint32_t *u_ch, uint8_t ch) { - *u_ch <<= 6; - *u_ch |= ch & 0x3f; -} - static bool is_unicode_control(uint32_t ch) { /* @@ -85,170 +74,181 @@ static bool is_unicode_control(uint32_t ch) { '\t' is in C0 range, but more or less harmless and commonly used. */ - return (ch < ' ' && ch != '\t') || + return (ch < ' ' && ch != '\t' && ch != '\n') || (0x7F <= ch && ch <= 0x9F); } -char* utf8_is_printable_n(const char* str, size_t length) { - uint32_t val = 0; - uint32_t min = 0; - const uint8_t *p; +/* count of characters used to encode one unicode char */ +static int utf8_encoded_expected_len(const char *str) { + unsigned char c; assert(str); - for (p = (const uint8_t*) str; length; p++, length--) { - if (*p < 128) { - val = *p; - } else { - if ((*p & 0xe0) == 0xc0) { /* 110xxxxx two-char seq. */ - min = 128; - val = (uint32_t) (*p & 0x1e); - goto ONE_REMAINING; - } else if ((*p & 0xf0) == 0xe0) { /* 1110xxxx three-char seq.*/ - min = (1 << 11); - val = (uint32_t) (*p & 0x0f); - goto TWO_REMAINING; - } else if ((*p & 0xf8) == 0xf0) { /* 11110xxx four-char seq */ - min = (1 << 16); - val = (uint32_t) (*p & 0x07); - } else - goto error; - - p++; - length--; - if (!length || !is_continuation_char(*p)) - goto error; - merge_continuation_char(&val, *p); - - TWO_REMAINING: - p++; - length--; - if (!is_continuation_char(*p)) - goto error; - merge_continuation_char(&val, *p); - - ONE_REMAINING: - p++; - length--; - if (!is_continuation_char(*p)) - goto error; - merge_continuation_char(&val, *p); - - if (val < min) - goto error; - } + c = (unsigned char) str[0]; + if (c < 0x80) + return 1; + if ((c & 0xe0) == 0xc0) + return 2; + if ((c & 0xf0) == 0xe0) + return 3; + if ((c & 0xf8) == 0xf0) + return 4; + if ((c & 0xfc) == 0xf8) + return 5; + if ((c & 0xfe) == 0xfc) + return 6; + + return 0; +} + +/* decode one unicode char */ +int utf8_encoded_to_unichar(const char *str) { + int unichar, len, i; + + assert(str); - if (is_unicode_control(val)) - goto error; + len = utf8_encoded_expected_len(str); + + switch (len) { + case 1: + return (int)str[0]; + case 2: + unichar = str[0] & 0x1f; + break; + case 3: + unichar = (int)str[0] & 0x0f; + break; + case 4: + unichar = (int)str[0] & 0x07; + break; + case 5: + unichar = (int)str[0] & 0x03; + break; + case 6: + unichar = (int)str[0] & 0x01; + break; + default: + return -EINVAL; } - return (char*) str; + for (i = 1; i < len; i++) { + if (((int)str[i] & 0xc0) != 0x80) + return -EINVAL; + unichar <<= 6; + unichar |= (int)str[i] & 0x3f; + } -error: - return NULL; + return unichar; } -static char* utf8_validate(const char *str, char *output) { - uint32_t val = 0; - uint32_t min = 0; - const uint8_t *p, *last; - int size; - uint8_t *o; +bool utf8_is_printable_newline(const char* str, size_t length, bool newline) { + const char *p; assert(str); - o = (uint8_t*) output; - for (p = (const uint8_t*) str; *p; p++) { - if (*p < 128) { - if (o) - *o = *p; - } else { - last = p; - - if ((*p & 0xe0) == 0xc0) { /* 110xxxxx two-char seq. */ - size = 2; - min = 128; - val = (uint32_t) (*p & 0x1e); - goto ONE_REMAINING; - } else if ((*p & 0xf0) == 0xe0) { /* 1110xxxx three-char seq.*/ - size = 3; - min = (1 << 11); - val = (uint32_t) (*p & 0x0f); - goto TWO_REMAINING; - } else if ((*p & 0xf8) == 0xf0) { /* 11110xxx four-char seq */ - size = 4; - min = (1 << 16); - val = (uint32_t) (*p & 0x07); - } else - goto error; - - p++; - if (!is_continuation_char(*p)) - goto error; - merge_continuation_char(&val, *p); - - TWO_REMAINING: - p++; - if (!is_continuation_char(*p)) - goto error; - merge_continuation_char(&val, *p); - - ONE_REMAINING: - p++; - if (!is_continuation_char(*p)) - goto error; - merge_continuation_char(&val, *p); - - if (val < min) - goto error; - - if (!is_unicode_valid(val)) - goto error; - - if (o) { - memcpy(o, last, (size_t) size); - o += size; - } + for (p = str; length;) { + int encoded_len, val; - continue; + encoded_len = utf8_encoded_valid_unichar(p); + if (encoded_len < 0 || + (size_t) encoded_len > length) + return false; - error: - if (o) { - *o = FILTER_CHAR; - p = last; /* We retry at the next character */ - } else - goto failure; - } + val = utf8_encoded_to_unichar(p); + if (val < 0 || + is_unicode_control(val) || + (!newline && val == '\n')) + return false; - if (o) - o++; + length -= encoded_len; + p += encoded_len; } - if (o) { - *o = '\0'; - return output; - } + return true; +} - return (char*) str; +const char *utf8_is_valid(const char *str) { + const uint8_t *p; + + assert(str); -failure: - return NULL; + for (p = (const uint8_t*) str; *p; ) { + int len; + + len = utf8_encoded_valid_unichar((const char *)p); + if (len < 0) + return NULL; + + p += len; + } + + return str; } -char* utf8_is_valid (const char *str) { - return utf8_validate(str, NULL); +char *utf8_escape_invalid(const char *str) { + char *p, *s; + + assert(str); + + p = s = malloc(strlen(str) * 4 + 1); + if (!p) + return NULL; + + while (*str) { + int len; + + len = utf8_encoded_valid_unichar(str); + if (len > 0) { + s = mempcpy(s, str, len); + str += len; + } else { + s = stpcpy(s, UTF8_REPLACEMENT_CHARACTER); + str += 1; + } + } + + *s = '\0'; + + return p; } -char* utf8_filter (const char *str) { - char *new_str; +char *utf8_escape_non_printable(const char *str) { + char *p, *s; assert(str); - new_str = malloc(strlen(str) + 1); - if (!new_str) + p = s = malloc(strlen(str) * 4 + 1); + if (!p) return NULL; - return utf8_validate(str, new_str); + while (*str) { + int len; + + len = utf8_encoded_valid_unichar(str); + if (len > 0) { + if (utf8_is_printable(str, len)) { + s = mempcpy(s, str, len); + str += len; + } else { + while (len > 0) { + *(s++) = '\\'; + *(s++) = 'x'; + *(s++) = hexchar((int) *str >> 4); + *(s++) = hexchar((int) *str); + + str += 1; + len --; + } + } + } else { + s = stpcpy(s, UTF8_REPLACEMENT_CHARACTER); + str += 1; + } + } + + *s = '\0'; + + return p; } char *ascii_is_valid(const char *str) { @@ -263,23 +263,87 @@ char *ascii_is_valid(const char *str) { return (char*) str; } -char *ascii_filter(const char *str) { - const char *s; - char *r, *d; - size_t l; +char *utf16_to_utf8(const void *s, size_t length) { + char *r; + const uint8_t *f; + uint8_t *t; - assert(str); - - l = strlen(str); - r = malloc(l + 1); + r = new(char, (length*3+1)/2 + 1); if (!r) return NULL; - for (s = str, d = r; *s; s++) - if ((unsigned char) *s < 128) - *(d++) = *s; + t = (uint8_t*) r; - *d = 0; + for (f = s; f < (const uint8_t*) s + length; f += 2) { + uint16_t c; + + c = (f[1] << 8) | f[0]; + + if (c == 0) { + *t = 0; + return r; + } else if (c < 0x80) { + *(t++) = (uint8_t) c; + } else if (c < 0x800) { + *(t++) = (uint8_t) (0xc0 | (c >> 6)); + *(t++) = (uint8_t) (0x80 | (c & 0x3f)); + } else { + *(t++) = (uint8_t) (0xe0 | (c >> 12)); + *(t++) = (uint8_t) (0x80 | ((c >> 6) & 0x3f)); + *(t++) = (uint8_t) (0x80 | (c & 0x3f)); + } + } + + *t = 0; return r; } + +/* expected size used to encode one unicode char */ +static int utf8_unichar_to_encoded_len(int unichar) { + + if (unichar < 0x80) + return 1; + if (unichar < 0x800) + return 2; + if (unichar < 0x10000) + return 3; + if (unichar < 0x200000) + return 4; + if (unichar < 0x4000000) + return 5; + + return 6; +} + +/* validate one encoded unicode char and return its length */ +int utf8_encoded_valid_unichar(const char *str) { + int len, unichar, i; + + assert(str); + + len = utf8_encoded_expected_len(str); + if (len == 0) + return -EINVAL; + + /* ascii is valid */ + if (len == 1) + return 1; + + /* check if expected encoded chars are available */ + for (i = 0; i < len; i++) + if ((str[i] & 0x80) != 0x80) + return -EINVAL; + + unichar = utf8_encoded_to_unichar(str); + + /* check if encoded length matches encoded value */ + if (utf8_unichar_to_encoded_len(unichar) != len) + return -EINVAL; + + /* check if value has valid range */ + if (!is_unicode_valid(unichar)) + return -EINVAL; + + return len; +}