* @brief Unicode support functions
*
* Here by UTF-8 and UTF-8 we mean the encoding forms of those names (not the
- * encoding schemes).
+ * encoding schemes). The primary encoding form is UTF-32 but convenience
+ * wrappers using UTF-8 are provided for a number of functions.
*
* The idea is that all the strings that hit the database will be in a
* particular normalization form, and for the search and tags database
* @param ndp Where to store length of destination string (or NULL)
* @return Newly allocated destination string or NULL on error
*
- * If the UTF-32 is not valid then NULL is returned. A UTF-32 code
- * point is invalid if:
+ * If the UTF-32 is not valid then NULL is returned. A UTF-32 code point is
+ * invalid if:
* - it codes for a UTF-16 surrogate
* - it codes for a value outside the unicode code space
*
- * The return value is always 0-terminated. The value returned via @p
- * *ndp does not include the terminator.
+ * The return value is always 0-terminated. The value returned via @p *ndp
+ * does not include the terminator.
*/
char *utf32_to_utf8(const uint32_t *s, size_t ns, size_t *ndp) {
struct dynstr d;
dynstr_append(&d, 0xC0 | (c >> 6));
dynstr_append(&d, 0x80 | (c & 0x3F));
} else if(c < 0x10000) {
- if(c >= 0xDF800 && c <= 0xDFFF)
+ if(c >= 0xD800 && c <= 0xDFFF)
goto error;
dynstr_append(&d, 0xE0 | (c >> 12));
dynstr_append(&d, 0x80 | ((c >> 6) & 0x3F));
* @param ndp Where to store length of destination string (or NULL)
* @return Newly allocated destination string or NULL
*
- * The return value is always 0-terminated. The value returned via @p
- * *ndp does not include the terminator.
+ * The return value is always 0-terminated. The value returned via @p *ndp
+ * does not include the terminator.
*
* If the UTF-8 is not valid then NULL is returned. A UTF-8 sequence
* for a code point is invalid if:
while(ns > 0) {
c = *ss++;
--ns;
- /*
- * Acceptable UTF-8 is:
+ /* Acceptable UTF-8 is that which codes for Unicode Scalar Values
+ * (Unicode 5.0.0 s3.9 D76)
*
* 0xxxxxxx
* 7 data bits gives 0x00 - 0x7F and all are acceptable
* 21 data bits gives 0x00000000 - 0x001FFFFF
* but only 0x00010000 - 0x0010FFFF are acceptable
*
- * It is NOT always the case that the data bits in the first byte
- * are always non-0 for the acceptable values, so we do a separate
- * check after decoding.
+ * It is NOT always the case that the data bits in the first byte are
+ * always non-0 for the acceptable values, so we do a separate check after
+ * decoding.
*/
if(c < 0x80)
c32 = c;
* @param s Pointer to 0-terminated string
* @return Length of string in code points (excluding terminator)
*
- * Unlike the conversion functions no validity checking is done on the
- * string.
+ * Unlike the conversion functions no validity checking is done on the string.
*/
size_t utf32_len(const uint32_t *s) {
const uint32_t *t = s;
* @param ns Length of @p s
* @return 0 on success, -1 on error
*
- * @p s is modified in-place. See Unicode 5.0 s3.11 for details of
- * the ordering.
+ * @p s is modified in-place. See Unicode 5.0 s3.11 for details of the
+ * ordering.
*
- * Currently we only support a maximum of 1024 combining characters
- * after each base character. If this limit is exceeded then -1 is
- * returned.
+ * Currently we only support a maximum of 1024 combining characters after each
+ * base character. If this limit is exceeded then -1 is returned.
*/
static int utf32__canonical_ordering(uint32_t *s, size_t ns) {
size_t nc;
dynstr_ucs4_init(&d); \
while(ns) { \
c = *s++; \
- if((c >= 0xDF800 && c <= 0xDFFF) || c > 0x10FFFF) \
+ if((c >= 0xD800 && c <= 0xDFFF) || c > 0x10FFFF) \
goto error; \
utf32__decompose_one_##WHICH(&d, c); \
--ns; \
* (at the time of writing!) passes the NFD tests defined in Unicode 5.0's
* NormalizationTest.txt.
*
- * Returns NULL if the string is not valid for either of the following
- * reasons:
+ * Returns NULL if the string is not valid for either of the following reasons:
* - it codes for a UTF-16 surrogate
* - it codes for a value outside the unicode code space
*/
* Form KD and (at the time of writing!) passes the NFKD tests defined in
* Unicode 5.0's NormalizationTest.txt.
*
- * Returns NULL if the string is not valid for either of the following
- * reasons:
+ * Returns NULL if the string is not valid for either of the following reasons:
* - it codes for a UTF-16 surrogate
* - it codes for a value outside the unicode code space
*/
utf32__decompose_generic(compat);
}
-/** @brief Case-fold @p C
- * @param D String to append to
- * @param C Character to fold
- */
-static inline void utf32__casefold_one_canon(struct dynstr_ucs4 *d, uint32_t c) {
- const uint32_t *cf =
- (c < UNICODE_NCHARS
- ? unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].casefold
- : 0);
- if(cf) {
- /* Found a case-fold mapping in the table */
- while(*cf)
- utf32__decompose_one_canon(d, *cf++);
- } else
- utf32__decompose_one_canon(d, c);
-}
+/** @brief Single-character case-fold and decompose operation */
+#define utf32__casefold_one(WHICH) do { \
+ const uint32_t *cf = \
+ (c < UNICODE_NCHARS \
+ ? unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].casefold \
+ : 0); \
+ if(cf) { \
+ /* Found a case-fold mapping in the table */ \
+ while(*cf) \
+ utf32__decompose_one_##WHICH(&d, *cf++); \
+ } else \
+ utf32__decompose_one_##WHICH(&d, c); \
+} while(0)
/** @brief Case-fold @p [s,s+ns)
* @param s Pointer to string
* @return Pointer to result string, or NULL
*
* Case-fold the string at @p s according to full default case-folding rules
- * (s3.13). The result will be in NFD.
+ * (s3.13) for caseless matching. The result will be in NFD.
*
- * Returns NULL if the string is not valid for either of the following
- * reasons:
+ * Returns NULL if the string is not valid for either of the following reasons:
* - it codes for a UTF-16 surrogate
* - it codes for a value outside the unicode code space
*/
dynstr_ucs4_init(&d);
while(ns) {
c = *s++;
- if((c >= 0xDF800 && c <= 0xDFFF) || c > 0x10FFFF)
+ if((c >= 0xD800 && c <= 0xDFFF) || c > 0x10FFFF)
goto error;
- utf32__casefold_one_canon(&d, c);
+ utf32__casefold_one(canon);
--ns;
}
if(utf32__canonical_ordering(d.vec, d.nvec))
return 0;
}
+/** @brief Compatibilit case-fold @p [s,s+ns)
+ * @param s Pointer to string
+ * @param ns Length of string
+ * @param ndp Where to store length of result
+ * @return Pointer to result string, or NULL
+ *
+ * Case-fold the string at @p s according to full default case-folding rules
+ * (s3.13) for compatibility caseless matching. The result will be in NFKD.
+ *
+ * Returns NULL if the string is not valid for either of the following reasons:
+ * - it codes for a UTF-16 surrogate
+ * - it codes for a value outside the unicode code space
+ */
+uint32_t *utf32_casefold_compat(const uint32_t *s, size_t ns, size_t *ndp) {
+ struct dynstr_ucs4 d;
+ uint32_t c;
+ size_t n;
+ uint32_t *ss = 0;
+
+ for(n = 0; n < ns; ++n) {
+ c = s[n];
+ if(c < UNICODE_NCHARS
+ && (unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].flags
+ & unicode_normalize_before_casefold))
+ break;
+ }
+ if(n < ns) {
+ /* We need a preliminary _canonical_ decomposition */
+ if(!(ss = utf32_decompose_canon(s, ns, &ns)))
+ return 0;
+ s = ss;
+ }
+ /* This computes NFKD(toCaseFold(s)) */
+#define compat_casefold_middle() do { \
+ dynstr_ucs4_init(&d); \
+ while(ns) { \
+ c = *s++; \
+ if((c >= 0xD800 && c <= 0xDFFF) || c > 0x10FFFF) \
+ goto error; \
+ utf32__casefold_one(compat); \
+ --ns; \
+ } \
+ if(utf32__canonical_ordering(d.vec, d.nvec)) \
+ goto error; \
+} while(0)
+ /* Do the inner (NFKD o toCaseFold) */
+ compat_casefold_middle();
+ /* We can do away with the NFD'd copy of the input now */
+ xfree(ss);
+ s = ss = d.vec;
+ ns = d.nvec;
+ /* Do the outer (NFKD o toCaseFold) */
+ compat_casefold_middle();
+ /* That's all */
+ dynstr_ucs4_terminate(&d);
+ if(ndp)
+ *ndp = d.nvec;
+ return d.vec;
+error:
+ xfree(d.vec);
+ xfree(ss);
+ return 0;
+}
+
/** @brief Order a pair of UTF-32 strings
* @param a First 0-terminated string
* @param b Second 0-terminated string
return *a < *b ? -1 : (*a > *b ? 1 : 0);
}
+/** @brief Return the General_Category value for @p c
+ * @param Code point
+ * @return General_Category property value
+ */
+static inline enum unicode_gc_cat utf32__general_category(uint32_t c) {
+ if(c < UNICODE_NCHARS) {
+ const struct unidata *const ud = &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
+ return ud->gc;
+ } else
+ return unicode_gc_Cn;
+}
+
+/** @brief Check Grapheme_Cluster_Break property
+ * @param c Code point
+ * @return 0 if it is as described, 1 otherwise
+ */
+static int utf32__is_control_or_cr_or_lf(uint32_t c) {
+ switch(utf32__general_category(c)) {
+ default:
+ return 0;
+ case unicode_gc_Zl:
+ case unicode_gc_Zp:
+ case unicode_gc_Cc:
+ return 1;
+ case unicode_gc_Cf:
+ if(c == 0x200C || c == 0x200D)
+ return 0;
+ return 1;
+ }
+}
+
+#define Hangul_Syllable_Type_NA 0
+#define Hangul_Syllable_Type_L 0x1100
+#define Hangul_Syllable_Type_V 0x1160
+#define Hangul_Syllable_Type_T 0x11A8
+#define Hangul_Syllable_Type_LV 0xAC00
+#define Hangul_Syllable_Type_LVT 0xAC01
+
+/** @brief Determine Hangul_Syllable_Type of @p c
+ * @param c Code point
+ * @return Equivalance class of @p c, or Hangul_Syllable_Type_NA
+ *
+ * If this is a Hangul character then a representative member of its
+ * equivalence class is returned. Otherwise Hangul_Syllable_Type_NA is
+ * returned.
+ */
+static uint32_t utf32__hangul_syllable_type(uint32_t c) {
+ /* Dispose of the bulk of the non-Hangul code points first */
+ if(c < 0x1100) return Hangul_Syllable_Type_NA;
+ if(c > 0x1200 && c < 0xAC00) return Hangul_Syllable_Type_NA;
+ if(c >= 0xD800) return Hangul_Syllable_Type_NA;
+ /* Now we pick out the assigned Hangul code points */
+ if((c >= 0x1100 && c <= 0x1159) || c == 0x115F) return Hangul_Syllable_Type_L;
+ if(c >= 0x1160 && c <= 0x11A2) return Hangul_Syllable_Type_V;
+ if(c >= 0x11A8 && c <= 0x11F9) return Hangul_Syllable_Type_T;
+ if(c >= 0xAC00 && c <= 0xD7A3) {
+ if(c % 28 == 16)
+ return Hangul_Syllable_Type_LV;
+ else
+ return Hangul_Syllable_Type_LVT;
+ }
+ return Hangul_Syllable_Type_NA;
+}
+
+/** @brief Determine Word_Break property
+ * @param c Code point
+ * @return Word_Break property value of @p c
+ */
+static enum unicode_Word_Break utf32__word_break(uint32_t c) {
+ if(c < 0xAC00 || c > 0xD7A3) {
+ if(c < UNICODE_NCHARS)
+ return unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].word_break;
+ else
+ return unicode_Word_Break_Other;
+ } else
+ return unicode_Word_Break_ALetter;
+}
+
+/** @brief Identify a grapheme cluster boundary
+ * @param s Start of string (must be NFD)
+ * @param ns Length of string
+ * @param n Index within string (in [0,ns].)
+ * @return 1 at a grapheme cluster boundary, 0 otherwise
+ *
+ * This function identifies default grapheme cluster boundaries as described in
+ * UAX #29 s3. It returns 1 if @p n points at the code point just after a
+ * grapheme cluster boundary (including the hypothetical code point just after
+ * the end of the string).
+ */
+int utf32_is_gcb(const uint32_t *s, size_t ns, size_t n) {
+ uint32_t before, after;
+ uint32_t hbefore, hafter;
+ /* GB1 and GB2 */
+ if(n == 0 || n == ns)
+ return 1;
+ /* Now we know that s[n-1] and s[n] are safe to inspect */
+ /* GB3 */
+ before = s[n-1];
+ after = s[n];
+ if(before == 0x000D && after == 0x000A)
+ return 0;
+ /* GB4 and GB5 */
+ if(utf32__is_control_or_cr_or_lf(before)
+ || utf32__is_control_or_cr_or_lf(after))
+ return 1;
+ hbefore = utf32__hangul_syllable_type(before);
+ hafter = utf32__hangul_syllable_type(after);
+ /* GB6 */
+ if(hbefore == Hangul_Syllable_Type_L
+ && (hafter == Hangul_Syllable_Type_L
+ || hafter == Hangul_Syllable_Type_V
+ || hafter == Hangul_Syllable_Type_LV
+ || hafter == Hangul_Syllable_Type_LVT))
+ return 0;
+ /* GB7 */
+ if((hbefore == Hangul_Syllable_Type_LV
+ || hbefore == Hangul_Syllable_Type_V)
+ && (hafter == Hangul_Syllable_Type_V
+ || hafter == Hangul_Syllable_Type_T))
+ return 0;
+ /* GB8 */
+ if((hbefore == Hangul_Syllable_Type_LVT
+ || hbefore == Hangul_Syllable_Type_T)
+ && hafter == Hangul_Syllable_Type_T)
+ return 0;
+ /* GB9 */
+ if(utf32__word_break(after) == unicode_Word_Break_Extend)
+ return 0;
+ /* GB10 */
+ return 1;
+}
+
+/** @brief Return true if @p c is ignorable for boundary specifications */
+static inline int utf32__boundary_ignorable(enum unicode_Word_Break wb) {
+ return (wb == unicode_Word_Break_Extend
+ || wb == unicode_Word_Break_Format);
+}
+
+/** @brief Identify a word boundary
+ * @param s Start of string (must be NFD)
+ * @param ns Length of string
+ * @param n Index within string (in [0,ns].)
+ * @return 1 at a word boundary, 0 otherwise
+ *
+ * This function identifies default word boundaries as described in UAX #29 s4.
+ * It returns 1 if @p n points at the code point just after a word boundary
+ * (including the hypothetical code point just after the end of the string).
+ */
+int utf32_is_word_boundary(const uint32_t *s, size_t ns, size_t n) {
+ enum unicode_Word_Break twobefore, before, after, twoafter;
+ size_t nn;
+
+ /* WB1 and WB2 */
+ if(n == 0 || n == ns)
+ return 1;
+ /* WB3 */
+ if(s[n-1] == 0x000D && s[n] == 0x000A)
+ return 0;
+ /* WB4 */
+ /* (!Sep) x (Extend|Format) as in UAX #29 s6.2 */
+ switch(s[n-1]) { /* bit of a bodge */
+ case 0x000A:
+ case 0x000D:
+ case 0x0085:
+ case 0x2028:
+ case 0x2029:
+ break;
+ default:
+ if(utf32__boundary_ignorable(utf32__word_break(s[n])))
+ return 0;
+ break;
+ }
+ /* Gather the property values we'll need for the rest of the test taking the
+ * s6.2 changes into account */
+ /* First we look at the code points after the proposed boundary */
+ nn = n; /* <ns */
+ after = utf32__word_break(s[nn++]);
+ if(!utf32__boundary_ignorable(after)) {
+ /* X (Extend|Format)* -> X */
+ while(nn < ns && utf32__boundary_ignorable(utf32__word_break(s[nn])))
+ ++nn;
+ }
+ /* It's possible now that nn=ns */
+ if(nn < ns)
+ twoafter = utf32__word_break(s[nn]);
+ else
+ twoafter = unicode_Word_Break_Other;
+
+ /* Next we look at the code points before the proposed boundary. This is a
+ * bit fiddlier. */
+ nn = n;
+ while(nn > 0 && utf32__boundary_ignorable(utf32__word_break(s[nn - 1])))
+ --nn;
+ if(nn == 0) {
+ /* s[nn] must be ignorable */
+ before = utf32__word_break(s[nn]);
+ twobefore = unicode_Word_Break_Other;
+ } else {
+ /* s[nn] is ignorable or after the proposed boundary; but s[nn-1] is not
+ * ignorable. */
+ before = utf32__word_break(s[nn - 1]);
+ --nn;
+ /* Repeat the exercise */
+ while(nn > 0 && utf32__boundary_ignorable(utf32__word_break(s[nn - 1])))
+ --nn;
+ if(nn == 0)
+ twobefore = utf32__word_break(s[nn]);
+ else
+ twobefore = utf32__word_break(s[nn - 1]);
+ }
+
+ /* WB5 */
+ if(before == unicode_Word_Break_ALetter
+ && after == unicode_Word_Break_ALetter)
+ return 0;
+ /* WB6 */
+ if(before == unicode_Word_Break_ALetter
+ && after == unicode_Word_Break_MidLetter
+ && twoafter == unicode_Word_Break_ALetter)
+ return 0;
+ /* WB7 */
+ if(twobefore == unicode_Word_Break_ALetter
+ && before == unicode_Word_Break_MidLetter
+ && after == unicode_Word_Break_ALetter)
+ return 0;
+ /* WB8 */
+ if(before == unicode_Word_Break_Numeric
+ && after == unicode_Word_Break_Numeric)
+ return 0;
+ /* WB9 */
+ if(before == unicode_Word_Break_ALetter
+ && after == unicode_Word_Break_Numeric)
+ return 0;
+ /* WB10 */
+ if(before == unicode_Word_Break_Numeric
+ && after == unicode_Word_Break_ALetter)
+ return 0;
+ /* WB11 */
+ if(twobefore == unicode_Word_Break_Numeric
+ && before == unicode_Word_Break_MidNum
+ && after == unicode_Word_Break_Numeric)
+ return 0;
+ /* WB12 */
+ if(before == unicode_Word_Break_Numeric
+ && after == unicode_Word_Break_MidNum
+ && twoafter == unicode_Word_Break_Numeric)
+ return 0;
+ /* WB13 */
+ if(before == unicode_Word_Break_Katakana
+ && after == unicode_Word_Break_Katakana)
+ return 0;
+ /* WB13a */
+ if((before == unicode_Word_Break_ALetter
+ || before == unicode_Word_Break_Numeric
+ || before == unicode_Word_Break_Katakana
+ || before == unicode_Word_Break_ExtendNumLet)
+ && after == unicode_Word_Break_ExtendNumLet)
+ return 0;
+ /* WB13b */
+ if(before == unicode_Word_Break_ExtendNumLet
+ && (after == unicode_Word_Break_ALetter
+ || after == unicode_Word_Break_Numeric
+ || after == unicode_Word_Break_Katakana))
+ return 0;
+ /* WB14 */
+ return 1;
+}
+
/*@}*/
/** @defgroup Functions that operate on UTF-8 strings */
/*@{*/
* Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
* this might be.
*/
-#if 0
char *utf8_casefold_compat(const char *s, size_t ns, size_t *ndp) {
utf8__transform(utf32_casefold_compat);
}
-#endif
/*@}*/
~mdw / disorder / blobdiff