basic/escape: merge utf8 and non-utf8 paths in cunescape_one

Not every byte sequence is valid utf8. We allow escaping of non-utf8
sequences in strings by using octal and hexadecimal escape sequences
(\123 and \0xAB) for bytes at or above 128. Users of cunescape_one
could infer whether such use occured when they received an answer
between 128 and 256 in *ret (a non-ascii one byte character). But this
is subtle and misleading: the comments were wrong, because ascii is a
subset of unicode, so c != 0 did not mean non-unicode, but rather
ascii-subset-of-unicode-or-raw-byte. This was all rather confusing, so
make the "single byte" condition explicit.

I'm not convinced that allowing non-utf8 sequences to be produced is
useful in all cases where we allow it (e.g. in config files), but that
behaviour is unchanged, just made more explicit.

This also fixes an (invalid) gcc warning about unitialized variable
(*ret_unicode) in callers of cunescape_one.

diff --git a/src/basic/escape.c b/src/basic/escape.c
index 42a84c93174e619736e33b7a395179c947deca9b..9dbfa6d6e0f8c8143e6409fd45e2483ae9a1031e 100644 (file)
--- a/src/basic/escape.c
+++ b/src/basic/escape.c
@@ -116,16 +116,18 @@ char *cescape(const char *s) {
         return cescape_length(s, strlen(s));
 }
 
-int cunescape_one(const char *p, size_t length, char *ret, uint32_t *ret_unicode) {
+int cunescape_one(const char *p, size_t length, uint32_t *ret, bool *eight_bit) {
         int r = 1;
 
         assert(p);
         assert(*p);
         assert(ret);
 
-        /* Unescapes C style. Returns the unescaped character in ret,
-         * unless we encountered a \u sequence in which case the full
-         * unicode character is returned in ret_unicode, instead. */
+        /* Unescapes C style. Returns the unescaped character in ret.
+         * Sets *eight_bit to true if the escaped sequence either fits in
+         * one byte in UTF-8 or is a non-unicode literal byte and should
+         * instead be copied directly.
+         */
 
         if (length != (size_t) -1 && length < 1)
                 return -EINVAL;
@@ -187,7 +189,8 @@ int cunescape_one(const char *p, size_t length, char *ret, uint32_t *ret_unicode
                 if (a == 0 && b == 0)
                         return -EINVAL;
 
-                *ret = (char) ((a << 4U) | b);
+                *ret = (a << 4U) | b;
+                *eight_bit = true;
                 r = 3;
                 break;
         }
@@ -214,16 +217,7 @@ int cunescape_one(const char *p, size_t length, char *ret, uint32_t *ret_unicode
                 if (c == 0)
                         return -EINVAL;
 
-                if (c < 128)
-                        *ret = c;
-                else {
-                        if (!ret_unicode)
-                                return -EINVAL;
-
-                        *ret = 0;
-                        *ret_unicode = c;
-                }
-
+                *ret = c;
                 r = 5;
                 break;
         }
@@ -255,16 +249,7 @@ int cunescape_one(const char *p, size_t length, char *ret, uint32_t *ret_unicode
                 if (!unichar_is_valid(c))
                         return -EINVAL;
 
-                if (c < 128)
-                        *ret = c;
-                else {
-                        if (!ret_unicode)
-                                return -EINVAL;
-
-                        *ret = 0;
-                        *ret_unicode = c;
-                }
-
+                *ret = c;
                 r = 9;
                 break;
         }
@@ -306,6 +291,7 @@ int cunescape_one(const char *p, size_t length, char *ret, uint32_t *ret_unicode
                         return -EINVAL;
 
                 *ret = m;
+                *eight_bit = true;
                 r = 3;
                 break;
         }
@@ -339,7 +325,7 @@ int cunescape_length_with_prefix(const char *s, size_t length, const char *prefi
         for (f = s, t = r + pl; f < s + length; f++) {
                 size_t remaining;
                 uint32_t u;
-                char c;
+                bool eight_bit = false;
                 int k;
 
                 remaining = s + length - f;
@@ -362,7 +348,7 @@ int cunescape_length_with_prefix(const char *s, size_t length, const char *prefi
                         return -EINVAL;
                 }
 
-                k = cunescape_one(f + 1, remaining - 1, &c, &u);
+                k = cunescape_one(f + 1, remaining - 1, &u, &eight_bit);
                 if (k < 0) {
                         if (flags & UNESCAPE_RELAX) {
                                 /* Invalid escape code, let's take it literal then */
@@ -374,14 +360,13 @@ int cunescape_length_with_prefix(const char *s, size_t length, const char *prefi
                         return k;
                 }
 
-                if (c != 0)
-                        /* Non-Unicode? Let's encode this directly */
-                        *(t++) = c;
+                f += k;
+                if (eight_bit)
+                        /* One byte? Set directly as specified */
+                        *(t++) = u;
                 else
-                        /* Unicode? Then let's encode this in UTF-8 */
+                        /* Otherwise encode as multi-byte UTF-8 */
                         t += utf8_encode_unichar(t, u);
-
-                f += k;
         }
 
         *t = 0;

diff --git a/src/basic/escape.h b/src/basic/escape.h
index 84916387a835d5e2e264a54f7c44c281ccafe680..6777de692c83698fff1715d753740c3c7c547c00 100644 (file)
--- a/src/basic/escape.h
+++ b/src/basic/escape.h
@@ -41,7 +41,7 @@ size_t cescape_char(char c, char *buf);
 int cunescape(const char *s, UnescapeFlags flags, char **ret);
 int cunescape_length(const char *s, size_t length, UnescapeFlags flags, char **ret);
 int cunescape_length_with_prefix(const char *s, size_t length, const char *prefix, UnescapeFlags flags, char **ret);
-int cunescape_one(const char *p, size_t length, char *ret, uint32_t *ret_unicode);
+int cunescape_one(const char *p, size_t length, uint32_t *ret, bool *eight_bit);
 
 char *xescape(const char *s, const char *bad);
 

diff --git a/src/basic/extract-word.c b/src/basic/extract-word.c
index 5b993e87081d0578782b2e8c08fa39cfb3dc39c5..de56168909c9405d4e66c14965527ffad91befd3 100644 (file)
--- a/src/basic/extract-word.c
+++ b/src/basic/extract-word.c
@@ -98,8 +98,9 @@ int extract_first_word(const char **p, char **ret, const char *separators, Extra
 
                         if (flags & EXTRACT_CUNESCAPE) {
                                 uint32_t u;
+                                bool eight_bit = false;
 
-                                r = cunescape_one(*p, (size_t) -1, &c, &u);
+                                r = cunescape_one(*p, (size_t) -1, &u, &eight_bit);
                                 if (r < 0) {
                                         if (flags & EXTRACT_CUNESCAPE_RELAX) {
                                                 s[sz++] = '\\';
@@ -109,10 +110,10 @@ int extract_first_word(const char **p, char **ret, const char *separators, Extra
                                 } else {
                                         (*p) += r - 1;
 
-                                        if (c != 0)
-                                                s[sz++] = c; /* normal explicit char */
+                                        if (eight_bit)
+                                                s[sz++] = u;
                                         else
-                                                sz += utf8_encode_unichar(s + sz, u); /* unicode chars we'll encode as utf8 */
+                                                sz += utf8_encode_unichar(s + sz, u);
                                 }
                         } else
                                 s[sz++] = c;