utf32_word_split() and utf8_word_split() splits a string into words

[disorder] / lib / unicode.h

diff --git a/lib/unicode.h b/lib/unicode.h
index 0f156c46f854874b01ca04d67980aa9c7720759e..7f3220773537d0447c2fbc810a1e517b61e1dcbf 100644 (file)
--- a/lib/unicode.h
+++ b/lib/unicode.h
@@ -38,6 +38,7 @@ typedef struct utf32_iterator_data *utf32_iterator;
 
 char *utf32_to_utf8(const uint32_t *s, size_t ns, size_t *nd);
 uint32_t *utf8_to_utf32(const char *s, size_t ns, size_t *nd);
+int utf8_valid(const char *s, size_t ns);
 
 size_t utf32_len(const uint32_t *s);
 int utf32_cmp(const uint32_t *a, const uint32_t *b);
@@ -48,6 +49,12 @@ char *utf8_decompose_canon(const char *s, size_t ns, size_t *ndp);
 uint32_t *utf32_decompose_compat(const uint32_t *s, size_t ns, size_t *ndp);
 char *utf8_decompose_compat(const char *s, size_t ns, size_t *ndp);
 
+uint32_t *utf32_compose_canon(const uint32_t *s, size_t ns, size_t *ndp);
+char *utf8_compose_canon(const char *s, size_t ns, size_t *ndp);
+
+uint32_t *utf32_compose_compat(const uint32_t *s, size_t ns, size_t *ndp);
+char *utf8_compose_compat(const char *s, size_t ns, size_t *ndp);
+
 uint32_t *utf32_casefold_canon(const uint32_t *s, size_t ns, size_t *ndp);
 char *utf8_casefold_canon(const char *s, size_t ns, size_t *ndp);
 
@@ -67,6 +74,29 @@ uint32_t utf32_iterator_code(utf32_iterator it);
 int utf32_iterator_grapheme_boundary(utf32_iterator it);
 int utf32_iterator_word_boundary(utf32_iterator it);
 
+uint32_t **utf32_word_split(const uint32_t *s, size_t ns, size_t *nwp);
+char **utf8_word_split(const char *s, size_t ns, size_t *nwp);
+
+/** @brief Convert 0-terminated UTF-32 to UTF-8
+ * @param s 0-terminated UTF-32 string
+ * @return 0-terminated UTF-8 string or 0 on error
+ *
+ * See utf32_to_utf8() for possible causes of errors.
+ */
+static inline char *utf32nt_to_utf8(const uint32_t *s) {
+  return utf32_to_utf8(s, utf32_len(s), 0);
+}
+
+/** @brief Convert 0-terminated UTF-8 to UTF-32
+ * @param s 0-terminated UTF-8 string
+ * @return 0-terminated UTF-32 string or 0 on error
+ *
+ * See utf8_to_utf32() for possible causes of errors.
+ */
+static inline uint32_t *utf8nt_to_utf32(const char *s) {
+  return utf8_to_utf32(s, strlen(s), 0);
+}
+
 #endif /* UNICODE_H */
 
 /*