shared: utf8 - support ucs4 -> utf8

Originally we only supported ucs2, so move the ucs4 version from libsystemd-terminal to shared
and use that everywhere.

diff --git a/src/libsystemd-terminal/subterm.c b/src/libsystemd-terminal/subterm.c
index 78efc9d7c0f6d7e31f1636cbf7a2effcdee95d2a..7c119ac58aa02eb4e96953bde9c48d5be66af40a 100644 (file)
--- a/src/libsystemd-terminal/subterm.c
+++ b/src/libsystemd-terminal/subterm.c
@@ -41,6 +41,7 @@
 #include "sd-event.h"
 #include "term-internal.h"
 #include "util.h"
 #include "sd-event.h"
 #include "term-internal.h"
 #include "util.h"

+#include "utf8.h"

 
 typedef struct Output Output;
 typedef struct Terminal Terminal;
 
 typedef struct Output Output;
 typedef struct Terminal Terminal;


@@ -459,7 +460,7 @@ static int output_draw_cell_fn(term_screen *screen,
                 output_printf(o, " ");
         } else {
                 for (k = 0; k < n_ch; ++k) {
                 output_printf(o, " ");
         } else {
                 for (k = 0; k < n_ch; ++k) {

-                        ulen = term_utf8_encode(utf8, ch[k]);
+                        ulen = utf8_encode_unichar(utf8, ch[k]);

                         output_write(o, utf8, ulen);
                 }
         }
                         output_write(o, utf8, ulen);
                 }
         }


@@ -625,7 +626,7 @@ static int terminal_push_tmp(Terminal *t, uint32_t ucs4) {
 
         assert(t);
 
 
         assert(t);
 

-        len = term_utf8_encode(buf, ucs4);
+        len = utf8_encode_unichar(buf, ucs4);

         if (len < 1)
                 return 0;
 
         if (len < 1)
                 return 0;
 

diff --git a/src/libsystemd-terminal/term-parser.c b/src/libsystemd-terminal/term-parser.c
index d8206a46ba30872af7513f8c36cf3cbabab7eec4..8dc1da2f9c4bcc7c8be3c35992a6ed838fea8e43 100644 (file)
--- a/src/libsystemd-terminal/term-parser.c
+++ b/src/libsystemd-terminal/term-parser.c
@@ -151,49 +151,6 @@ void term_attr_to_argb32(const term_attr *attr, uint32_t *fg, uint32_t *bg, cons
                 *bg = b;
 }
 
                 *bg = b;
 }
 

-/**
- * term_utf8_encode() - Encode single UCS-4 character as UTF-8
- * @out_utf8: output buffer of at least 4 bytes or NULL
- * @g: UCS-4 character to encode
- *
- * This encodes a single UCS-4 character as UTF-8 and writes it into @out_utf8.
- * The length of the character is returned. It is not zero-terminated! If the
- * output buffer is NULL, only the length is returned.
- *
- * Returns: The length in bytes that the UTF-8 representation does or would
- *          occupy.
- */
-size_t term_utf8_encode(char *out_utf8, uint32_t g) {
-        if (g < (1 << 7)) {
-                if (out_utf8)
-                        out_utf8[0] = g & 0x7f;
-                return 1;
-        } else if (g < (1 << 11)) {
-                if (out_utf8) {
-                        out_utf8[0] = 0xc0 | ((g >> 6) & 0x1f);
-                        out_utf8[1] = 0x80 | (g & 0x3f);
-                }
-                return 2;
-        } else if (g < (1 << 16)) {
-                if (out_utf8) {
-                        out_utf8[0] = 0xe0 | ((g >> 12) & 0x0f);
-                        out_utf8[1] = 0x80 | ((g >> 6) & 0x3f);
-                        out_utf8[2] = 0x80 | (g & 0x3f);
-                }
-                return 3;
-        } else if (g < (1 << 21)) {
-                if (out_utf8) {
-                        out_utf8[0] = 0xf0 | ((g >> 18) & 0x07);
-                        out_utf8[1] = 0x80 | ((g >> 12) & 0x3f);
-                        out_utf8[2] = 0x80 | ((g >> 6) & 0x3f);
-                        out_utf8[3] = 0x80 | (g & 0x3f);
-                }
-                return 4;
-        } else {
-                return 0;
-        }
-}
-

 /**
  * term_utf8_decode() - Try decoding the next UCS-4 character
  * @p: decoder object to operate on or NULL
 /**
  * term_utf8_decode() - Try decoding the next UCS-4 character
  * @p: decoder object to operate on or NULL

diff --git a/src/libsystemd-terminal/term-screen.c b/src/libsystemd-terminal/term-screen.c
index f021ffee23c960c80347a047c21ac3de51a51e86..0e38ff41c63f5b5278979a5109620ab1e7b5be49 100644 (file)
--- a/src/libsystemd-terminal/term-screen.c
+++ b/src/libsystemd-terminal/term-screen.c
@@ -51,6 +51,7 @@
 #include "macro.h"
 #include "term-internal.h"
 #include "util.h"
 #include "macro.h"
 #include "term-internal.h"
 #include "util.h"

+#include "utf8.h"

 
 int term_screen_new(term_screen **out, term_screen_write_fn write_fn, void *write_fn_data, term_screen_cmd_fn cmd_fn, void *cmd_fn_data) {
         _cleanup_(term_screen_unrefp) term_screen *screen = NULL;
 
 int term_screen_new(term_screen **out, term_screen_write_fn write_fn, void *write_fn_data, term_screen_cmd_fn cmd_fn, void *cmd_fn_data) {
         _cleanup_(term_screen_unrefp) term_screen *screen = NULL;


@@ -4107,7 +4108,7 @@ static char *screen_map_key(term_screen *screen,
 
         /* map unicode keys */
         for (i = 0; i < n_syms; ++i)
 
         /* map unicode keys */
         for (i = 0; i < n_syms; ++i)

-                p += term_utf8_encode(p, ucs4[i]);
+                p += utf8_encode_unichar(p, ucs4[i]);

 
         return p;
 }
 
         return p;
 }

diff --git a/src/libsystemd-terminal/term.h b/src/libsystemd-terminal/term.h
index eae6c6352f911ac0ae8c6c5089aedf33a7d3a0bf..1a78a811841850713a7ddcdcb4656fc5aee857af 100644 (file)
--- a/src/libsystemd-terminal/term.h
+++ b/src/libsystemd-terminal/term.h
@@ -112,7 +112,6 @@ struct term_utf8 {
         unsigned int valid : 1;
 };
 
         unsigned int valid : 1;
 };
 

-size_t term_utf8_encode(char *out_utf8, uint32_t g);

 size_t term_utf8_decode(term_utf8 *p, uint32_t **out_buf, char c);
 
 /*
 size_t term_utf8_decode(term_utf8 *p, uint32_t **out_buf, char c);
 
 /*

diff --git a/src/libsystemd-terminal/test-term-parser.c b/src/libsystemd-terminal/test-term-parser.c
index e8d5dcfbf2dd64d70528dbc1466ca3493ff14742..e22614d06dd2e521081297c78826f25d7b9e0a31 100644 (file)
--- a/src/libsystemd-terminal/test-term-parser.c
+++ b/src/libsystemd-terminal/test-term-parser.c
@@ -30,6 +30,7 @@
 #include "macro.h"
 #include "term-internal.h"
 #include "util.h"
 #include "macro.h"
 #include "term-internal.h"
 #include "util.h"

+#include "utf8.h"

 
 static void test_term_utf8_invalid(void) {
         term_utf8 p = { };
 
 static void test_term_utf8_invalid(void) {
         term_utf8 p = { };


@@ -74,7 +75,7 @@ static void test_term_utf8_range(void) {
         /* Convert all ucs-4 chars to utf-8 and back */
 
         for (i = 0; i < 0x10FFFF; ++i) {
         /* Convert all ucs-4 chars to utf-8 and back */
 
         for (i = 0; i < 0x10FFFF; ++i) {

-                ulen = term_utf8_encode(u8, i);
+                ulen = utf8_encode_unichar(u8, i);

                 if (!ulen)
                         continue;
 
                 if (!ulen)
                         continue;
 

diff --git a/src/shared/json.c b/src/shared/json.c
index f1495e99c8f551ea52d22c8feae7d98bfdb3826a..47f801c8589223d511e7ebf1ec3e03bce14af64c 100644 (file)
--- a/src/shared/json.c
+++ b/src/shared/json.c
@@ -150,7 +150,7 @@ static int json_parse_string(const char **p, char **ret) {
                                 if (!GREEDY_REALLOC(s, allocated, n + 4))
                                         return -ENOMEM;
 
                                 if (!GREEDY_REALLOC(s, allocated, n + 4))
                                         return -ENOMEM;
 

-                                n += utf8_encode_unichar(x, s + n);
+                                n += utf8_encode_unichar(s + n, x);

                                 c += 5;
                                 continue;
                         } else
                                 c += 5;
                                 continue;
                         } else

diff --git a/src/shared/utf8.c b/src/shared/utf8.c
index 03a0abe44b7df0bd05d676476cf38db5412ba800..ab57f4276b038734f6c65a6e39b41b954c90502f 100644 (file)
--- a/src/shared/utf8.c
+++ b/src/shared/utf8.c
@@ -263,21 +263,46 @@ char *ascii_is_valid(const char *str) {
         return (char*) str;
 }
 
         return (char*) str;
 }
 

-int utf8_encode_unichar(uint16_t c, char *p) {
-        uint8_t *t = (uint8_t*) p;
-
-        if (c < 0x80) {
-                t[0] = (uint8_t) c;
+/**
+ * utf8_encode_unichar() - Encode single UCS-4 character as UTF-8
+ * @out_utf8: output buffer of at least 4 bytes or NULL
+ * @g: UCS-4 character to encode
+ *
+ * This encodes a single UCS-4 character as UTF-8 and writes it into @out_utf8.
+ * The length of the character is returned. It is not zero-terminated! If the
+ * output buffer is NULL, only the length is returned.
+ *
+ * Returns: The length in bytes that the UTF-8 representation does or would
+ *          occupy.
+ */
+size_t utf8_encode_unichar(char *out_utf8, uint32_t g) {
+        if (g < (1 << 7)) {
+                if (out_utf8)
+                        out_utf8[0] = g & 0x7f;

                 return 1;
                 return 1;

-        } else if (c < 0x800) {
-                t[0] = (uint8_t) (0xc0 | (c >> 6));
-                t[1] = (uint8_t) (0x80 | (c & 0x3f));
+        } else if (g < (1 << 11)) {
+                if (out_utf8) {
+                        out_utf8[0] = 0xc0 | ((g >> 6) & 0x1f);
+                        out_utf8[1] = 0x80 | (g & 0x3f);
+                }

                 return 2;
                 return 2;

-        } else {
-                t[0] = (uint8_t) (0xe0 | (c >> 12));
-                t[1] = (uint8_t) (0x80 | ((c >> 6) & 0x3f));
-                t[2] = (uint8_t) (0x80 | (c & 0x3f));
+        } else if (g < (1 << 16)) {
+                if (out_utf8) {
+                        out_utf8[0] = 0xe0 | ((g >> 12) & 0x0f);
+                        out_utf8[1] = 0x80 | ((g >> 6) & 0x3f);
+                        out_utf8[2] = 0x80 | (g & 0x3f);
+                }

                 return 3;
                 return 3;

+        } else if (g < (1 << 21)) {
+                if (out_utf8) {
+                        out_utf8[0] = 0xf0 | ((g >> 18) & 0x07);
+                        out_utf8[1] = 0x80 | ((g >> 12) & 0x3f);
+                        out_utf8[2] = 0x80 | ((g >> 6) & 0x3f);
+                        out_utf8[3] = 0x80 | (g & 0x3f);
+                }
+                return 4;
+        } else {
+                return 0;

         }
 }
 
         }
 }
 


@@ -290,7 +315,7 @@ char *utf16_to_utf8(const void *s, size_t length) {
                 return NULL;
 
         for (f = s, t = r; f < (const uint8_t*) s + length; f += 2)
                 return NULL;
 
         for (f = s, t = r; f < (const uint8_t*) s + length; f += 2)

-                t += utf8_encode_unichar((f[1] << 8) | f[0], t);
+                t += utf8_encode_unichar(t, (f[1] << 8) | f[0]);

 
         *t = 0;
         return r;
 
         *t = 0;
         return r;

diff --git a/src/shared/utf8.h b/src/shared/utf8.h
index dcf8588d32aaa7b8a68fab9694f9be2249ebb152..3d5a4c3a9c0c2e81db9bcd973cddf4d07591f3bf 100644 (file)
--- a/src/shared/utf8.h
+++ b/src/shared/utf8.h
@@ -36,7 +36,7 @@ bool utf8_is_printable_newline(const char* str, size_t length, bool newline) _pu
 char *utf8_escape_invalid(const char *s);
 char *utf8_escape_non_printable(const char *str);
 
 char *utf8_escape_invalid(const char *s);
 char *utf8_escape_non_printable(const char *str);
 

-int utf8_encode_unichar(uint16_t c, char *p);
+size_t utf8_encode_unichar(char *out_utf8, uint32_t g);

 char *utf16_to_utf8(const void *s, size_t length);
 
 int utf8_encoded_valid_unichar(const char *str);
 char *utf16_to_utf8(const void *s, size_t length);
 
 int utf8_encoded_valid_unichar(const char *str);

diff --git a/src/test/test-json.c b/src/test/test-json.c
index 00768358bf4469105b7ee05e43b2f0c6a3c51559..e53e8ed50f18d932ebd1c6bb03b29253a895434f 100644 (file)
--- a/src/test/test-json.c
+++ b/src/test/test-json.c
@@ -98,6 +98,7 @@ int main(int argc, char *argv[]) {
         test_one("{\"foo\" : [true, false]}", JSON_OBJECT_OPEN, JSON_STRING, "foo", JSON_COLON, JSON_ARRAY_OPEN, JSON_BOOLEAN, true, JSON_COMMA, JSON_BOOLEAN, false, JSON_ARRAY_CLOSE, JSON_OBJECT_CLOSE, JSON_END);
         test_one("\"\xef\xbf\xbd\"", JSON_STRING, "\xef\xbf\xbd", JSON_END);
         test_one("\"\\ufffd\"", JSON_STRING, "\xef\xbf\xbd", JSON_END);
         test_one("{\"foo\" : [true, false]}", JSON_OBJECT_OPEN, JSON_STRING, "foo", JSON_COLON, JSON_ARRAY_OPEN, JSON_BOOLEAN, true, JSON_COMMA, JSON_BOOLEAN, false, JSON_ARRAY_CLOSE, JSON_OBJECT_CLOSE, JSON_END);
         test_one("\"\xef\xbf\xbd\"", JSON_STRING, "\xef\xbf\xbd", JSON_END);
         test_one("\"\\ufffd\"", JSON_STRING, "\xef\xbf\xbd", JSON_END);

+        test_one("\"\\uf\"", -EINVAL);

 
         return 0;
 }
 
         return 0;
 }