#include "unicode.h"
#include "unidata.h"
+/** @defgroup utf32props Unicode Code Point Properties */
+/*@{*/
+
+static const struct unidata *utf32__unidata_hard(uint32_t c);
+
+/** @brief Find definition of code point @p c
+ * @param c Code point
+ * @return Pointer to @ref unidata structure for @p c
+ *
+ * @p c can be any 32-bit value, a sensible value will be returned regardless.
+ * The returned pointer is NOT guaranteed to be unique to @p c.
+ */
+static inline const struct unidata *utf32__unidata(uint32_t c) {
+ /* The bottom half of the table contains almost everything of interest
+ * and we can just return the right thing straight away */
+ if(c < UNICODE_BREAK_START)
+ return &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
+ else
+ return utf32__unidata_hard(c);
+}
+
+/** @brief Find definition of code point @p c
+ * @param c Code point
+ * @return Pointer to @ref unidata structure for @p c
+ *
+ * @p c can be any 32-bit value, a sensible value will be returned regardless.
+ * The returned pointer is NOT guaranteed to be unique to @p c.
+ *
+ * Don't use this function (although it will work fine) - use utf32__unidata()
+ * instead.
+ */
+static const struct unidata *utf32__unidata_hard(uint32_t c) {
+ if(c < UNICODE_BREAK_START)
+ return &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
+ /* Within the break everything is unassigned */
+ if(c < UNICODE_BREAK_END)
+ return utf32__unidata(0xFFFF); /* guaranteed to be Cn */
+ /* Planes 15 and 16 are (mostly) private use */
+ if((c >= 0xF0000 && c <= 0xFFFFD)
+ || (c >= 0x100000 && c <= 0x10FFFD))
+ return utf32__unidata(0xE000); /* first Co code point */
+ /* Everything else above the break top is unassigned */
+ if(c >= UNICODE_BREAK_TOP)
+ return utf32__unidata(0xFFFF); /* guaranteed to be Cn */
+ /* Currently the rest is language tags and variation selectors */
+ c -= (UNICODE_BREAK_END - UNICODE_BREAK_START);
+ return &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
+}
+
+/** @brief Return the combining class of @p c
+ * @param c Code point
+ * @return Combining class of @p c
+ *
+ * @p c can be any 32-bit value, a sensible value will be returned regardless.
+ */
+static inline int utf32__combining_class(uint32_t c) {
+ return utf32__unidata(c)->ccc;
+}
+
+/** @brief Return the General_Category value for @p c
+ * @param Code point
+ * @return General_Category property value
+ *
+ * @p c can be any 32-bit value, a sensible value will be returned regardless.
+ */
+static inline enum unicode_General_Category utf32__general_category(uint32_t c) {
+ return utf32__unidata(c)->general_category;
+}
+
+/** @brief Determine Grapheme_Break property
+ * @param c Code point
+ * @return Grapheme_Break property value of @p c
+ *
+ * @p c can be any 32-bit value, a sensible value will be returned regardless.
+ */
+static inline enum unicode_Grapheme_Break utf32__grapheme_break(uint32_t c) {
+ return utf32__unidata(c)->grapheme_break;
+}
+
+/** @brief Determine Word_Break property
+ * @param c Code point
+ * @return Word_Break property value of @p c
+ *
+ * @p c can be any 32-bit value, a sensible value will be returned regardless.
+ */
+static inline enum unicode_Word_Break utf32__word_break(uint32_t c) {
+ return utf32__unidata(c)->word_break;
+}
+
+/** @brief Determine Sentence_Break property
+ * @param c Code point
+ * @return Word_Break property value of @p c
+ *
+ * @p c can be any 32-bit value, a sensible value will be returned regardless.
+ */
+static inline enum unicode_Sentence_Break utf32__sentence_break(uint32_t c) {
+ return utf32__unidata(c)->sentence_break;
+}
+
+/** @brief Return true if @p c is ignorable for boundary specifications
+ * @param wb Word break property value
+ * @return non-0 if @p wb is unicode_Word_Break_Extend or unicode_Word_Break_Format
+ */
+static inline int utf32__boundary_ignorable(enum unicode_Word_Break wb) {
+ return (wb == unicode_Word_Break_Extend
+ || wb == unicode_Word_Break_Format);
+}
+
+/*@}*/
/** @defgroup utftransform Functions that transform between different Unicode encoding forms */
/*@{*/
*/
uint32_t *utf8_to_utf32(const char *s, size_t ns, size_t *ndp) {
struct dynstr_ucs4 d;
- uint32_t c32, c;
+ uint32_t c32;
const uint8_t *ss = (const uint8_t *)s;
+ int n;
dynstr_ucs4_init(&d);
while(ns > 0) {
- c = *ss++;
- --ns;
- /* Acceptable UTF-8 is that which codes for Unicode Scalar Values
- * (Unicode 5.0.0 s3.9 D76)
- *
- * 0xxxxxxx
- * 7 data bits gives 0x00 - 0x7F and all are acceptable
- *
- * 110xxxxx 10xxxxxx
- * 11 data bits gives 0x0000 - 0x07FF but only 0x0080 - 0x07FF acceptable
- *
- * 1110xxxx 10xxxxxx 10xxxxxx
- * 16 data bits gives 0x0000 - 0xFFFF but only 0x0800 - 0xFFFF acceptable
- * (and UTF-16 surrogates are not acceptable)
- *
- * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
- * 21 data bits gives 0x00000000 - 0x001FFFFF
- * but only 0x00010000 - 0x0010FFFF are acceptable
- *
- * It is NOT always the case that the data bits in the first byte are
- * always non-0 for the acceptable values, so we do a separate check after
- * decoding.
- */
- if(c < 0x80)
- c32 = c;
- else if(c <= 0xDF) {
- if(ns < 1) goto error;
- c32 = c & 0x1F;
- c = *ss++;
- if((c & 0xC0) != 0x80) goto error;
- c32 = (c32 << 6) | (c & 0x3F);
- if(c32 < 0x80) goto error;
- } else if(c <= 0xEF) {
- if(ns < 2) goto error;
- c32 = c & 0x0F;
- c = *ss++;
- if((c & 0xC0) != 0x80) goto error;
- c32 = (c32 << 6) | (c & 0x3F);
- c = *ss++;
- if((c & 0xC0) != 0x80) goto error;
- c32 = (c32 << 6) | (c & 0x3F);
- if(c32 < 0x0800 || (c32 >= 0xD800 && c32 <= 0xDFFF)) goto error;
- } else if(c <= 0xF7) {
- if(ns < 3) goto error;
- c32 = c & 0x07;
- c = *ss++;
- if((c & 0xC0) != 0x80) goto error;
- c32 = (c32 << 6) | (c & 0x3F);
- c = *ss++;
- if((c & 0xC0) != 0x80) goto error;
- c32 = (c32 << 6) | (c & 0x3F);
- c = *ss++;
- if((c & 0xC0) != 0x80) goto error;
- c32 = (c32 << 6) | (c & 0x3F);
- if(c32 < 0x00010000 || c32 > 0x0010FFFF) goto error;
+ const struct unicode_utf8_row *const r = &unicode_utf8_valid[*ss];
+ if(r->count <= ns) {
+ switch(r->count) {
+ case 1:
+ c32 = *ss;
+ break;
+ case 2:
+ if(ss[1] < r->min2 || ss[1] > r->max2)
+ goto error;
+ c32 = *ss & 0x1F;
+ break;
+ case 3:
+ if(ss[1] < r->min2 || ss[1] > r->max2)
+ goto error;
+ c32 = *ss & 0x0F;
+ break;
+ case 4:
+ if(ss[1] < r->min2 || ss[1] > r->max2)
+ goto error;
+ c32 = *ss & 0x07;
+ break;
+ default:
+ goto error;
+ }
} else
goto error;
+ for(n = 1; n < r->count; ++n) {
+ if(ss[n] < 0x80 || ss[n] > 0xBF)
+ goto error;
+ c32 = (c32 << 6) | (ss[n] & 0x3F);
+ }
dynstr_ucs4_append(&d, c32);
+ ss += r->count;
+ ns -= r->count;
}
dynstr_ucs4_terminate(&d);
if(ndp)
return 0;
}
+/** @brief Test whether [s,s+ns) is valid UTF-8
+ * @param s Start of string
+ * @param ns Length of string
+ * @return non-0 if @p s is valid UTF-8, 0 if it is not valid
+ *
+ * This function is intended to be much faster than calling utf8_to_utf32() and
+ * throwing away the result.
+ */
+int utf8_valid(const char *s, size_t ns) {
+ const uint8_t *ss = (const uint8_t *)s;
+ while(ns > 0) {
+ const struct unicode_utf8_row *const r = &unicode_utf8_valid[*ss];
+ if(r->count <= ns) {
+ switch(r->count) {
+ case 1:
+ break;
+ case 2:
+ if(ss[1] < r->min2 || ss[1] > r->max2)
+ return 0;
+ break;
+ case 3:
+ if(ss[1] < r->min2 || ss[1] > r->max2)
+ return 0;
+ if(ss[2] < 0x80 || ss[2] > 0xBF)
+ return 0;
+ break;
+ case 4:
+ if(ss[1] < r->min2 || ss[1] > r->max2)
+ return 0;
+ if(ss[2] < 0x80 || ss[2] > 0xBF)
+ return 0;
+ if(ss[3] < 0x80 || ss[3] > 0xBF)
+ return 0;
+ break;
+ default:
+ return 0;
+ }
+ } else
+ return 0;
+ ss += r->count;
+ ns -= r->count;
+ }
+ return 1;
+}
+
+/*@}*/
+/** @defgroup utf32iterator UTF-32 string iterators */
+/*@{*/
+
+struct utf32_iterator_data {
+ /** @brief Start of string */
+ const uint32_t *s;
+
+ /** @brief Length of string */
+ size_t ns;
+
+ /** @brief Current position */
+ size_t n;
+
+ /** @brief Last two non-ignorable characters or (uint32_t)-1
+ *
+ * last[1] is the non-Extend/Format character just before position @p n;
+ * last[0] is the one just before that.
+ *
+ * Exception 1: if there is no such non-Extend/Format character then an
+ * Extend/Format character is accepted instead.
+ *
+ * Exception 2: if there is no such character even taking that into account
+ * the value is (uint32_t)-1.
+ */
+ uint32_t last[2];
+};
+
+/** @brief Create a new iterator pointing at the start of a string
+ * @param s Start of string
+ * @param ns Length of string
+ * @return New iterator
+ */
+utf32_iterator utf32_iterator_new(const uint32_t *s, size_t ns) {
+ utf32_iterator it = xmalloc(sizeof *it);
+ it->s = s;
+ it->ns = ns;
+ it->n = 0;
+ it->last[0] = it->last[1] = -1;
+ return it;
+}
+
+/** @brief Initialize an internal private iterator
+ * @param it Iterator
+ * @param s Start of string
+ * @param ns Length of string
+ * @param n Absolute position
+ */
+static void utf32__iterator_init(utf32_iterator it,
+ const uint32_t *s, size_t ns, size_t n) {
+ it->s = s;
+ it->ns = ns;
+ it->n = 0;
+ it->last[0] = it->last[1] = -1;
+ utf32_iterator_set(it, n);
+}
+
+/** @brief Destroy an iterator
+ * @param it Iterator
+ */
+void utf32_iterator_destroy(utf32_iterator it) {
+ xfree(it);
+}
+
+/** @brief Find the current position of an interator
+ * @param it Iterator
+ */
+size_t utf32_iterator_where(utf32_iterator it) {
+ return it->n;
+}
+
+/** @brief Set an iterator's absolute position
+ * @param it Iterator
+ * @param n Absolute position
+ * @return 0 on success, non-0 on error
+ *
+ * It is an error to position the iterator outside the string (but acceptable
+ * to point it at the hypothetical post-final character). If an invalid value
+ * of @p n is specified then the iterator is not changed.
+ */
+int utf32_iterator_set(utf32_iterator it, size_t n) {
+ /* We can't just jump to position @p n; the @p last[] values will be wrong.
+ * What we need is to jump a bit behind @p n and then advance forward,
+ * updating @p last[] along the way. How far back? We need to cross two
+ * non-ignorable code points as we advance forwards, so we'd better pass two
+ * such characters on the way back (if such are available).
+ */
+ size_t m;
+
+ if(n > it->ns) /* range check */
+ return -1;
+ /* Walk backwards skipping ignorable code points */
+ m = n;
+ while(m > 0 && (utf32__boundary_ignorable(utf32__word_break(it->s[m-1]))))
+ --m;
+ /* Either m=0 or s[m-1] is not ignorable */
+ if(m > 0) {
+ --m;
+ /* s[m] is our first non-ignorable code; look for a second in the same
+ way **/
+ while(m > 0 && (utf32__boundary_ignorable(utf32__word_break(it->s[m-1]))))
+ --m;
+ /* Either m=0 or s[m-1] is not ignorable */
+ if(m > 0)
+ --m;
+ }
+ it->last[0] = it->last[1] = -1;
+ it->n = m;
+ return utf32_iterator_advance(it, n - m);
+}
+
+/** @brief Advance an iterator
+ * @param it Iterator
+ * @param count Number of code points to advance by
+ * @return 0 on success, non-0 on error
+ *
+ * It is an error to advance an iterator beyond the hypothetical post-final
+ * character of the string. If an invalid value of @p n is specified then the
+ * iterator is not changed.
+ *
+ * This function has O(n) time complexity: it works by advancing naively
+ * forwards through the string.
+ */
+int utf32_iterator_advance(utf32_iterator it, size_t count) {
+ if(count <= it->ns - it->n) {
+ while(count > 0) {
+ const uint32_t c = it->s[it->n];
+ const enum unicode_Word_Break wb = utf32__word_break(c);
+ if(it->last[1] == (uint32_t)-1
+ || !utf32__boundary_ignorable(wb)) {
+ it->last[0] = it->last[1];
+ it->last[1] = c;
+ }
+ ++it->n;
+ --count;
+ }
+ return 0;
+ } else
+ return -1;
+}
+
+/** @brief Find the current code point
+ * @param it Iterator
+ * @return Current code point or 0
+ *
+ * If the iterator points at the hypothetical post-final character of the
+ * string then 0 is returned. NB that this doesn't mean that there aren't any
+ * 0 code points inside the string!
+ */
+uint32_t utf32_iterator_code(utf32_iterator it) {
+ if(it->n < it->ns)
+ return it->s[it->n];
+ else
+ return 0;
+}
+
+/** @brief Test for a grapheme boundary
+ * @param it Iterator
+ * @return Non-0 if pointing just after a grapheme boundary, otherwise 0
+ */
+int utf32_iterator_grapheme_boundary(utf32_iterator it) {
+ uint32_t before, after;
+ enum unicode_Grapheme_Break gbbefore, gbafter;
+ /* GB1 and GB2 */
+ if(it->n == 0 || it->n == it->ns)
+ return 1;
+ /* Now we know that s[n-1] and s[n] are safe to inspect */
+ /* GB3 */
+ before = it->s[it->n-1];
+ after = it->s[it->n];
+ if(before == 0x000D && after == 0x000A)
+ return 0;
+ gbbefore = utf32__grapheme_break(before);
+ gbafter = utf32__grapheme_break(after);
+ /* GB4 */
+ if(gbbefore == unicode_Grapheme_Break_Control
+ || before == 0x000D
+ || before == 0x000A)
+ return 1;
+ /* GB5 */
+ if(gbafter == unicode_Grapheme_Break_Control
+ || after == 0x000D
+ || after == 0x000A)
+ return 1;
+ /* GB6 */
+ if(gbbefore == unicode_Grapheme_Break_L
+ && (gbafter == unicode_Grapheme_Break_L
+ || gbafter == unicode_Grapheme_Break_V
+ || gbafter == unicode_Grapheme_Break_LV
+ || gbafter == unicode_Grapheme_Break_LVT))
+ return 0;
+ /* GB7 */
+ if((gbbefore == unicode_Grapheme_Break_LV
+ || gbbefore == unicode_Grapheme_Break_V)
+ && (gbafter == unicode_Grapheme_Break_V
+ || gbafter == unicode_Grapheme_Break_T))
+ return 0;
+ /* GB8 */
+ if((gbbefore == unicode_Grapheme_Break_LVT
+ || gbbefore == unicode_Grapheme_Break_T)
+ && gbafter == unicode_Grapheme_Break_T)
+ return 0;
+ /* GB9 */
+ if(gbafter == unicode_Grapheme_Break_Extend)
+ return 0;
+ /* GB10 */
+ return 1;
+
+}
+
+/** @brief Test for a word boundary
+ * @param it Iterator
+ * @return Non-0 if pointing just after a word boundary, otherwise 0
+ */
+int utf32_iterator_word_boundary(utf32_iterator it) {
+ enum unicode_Word_Break twobefore, before, after, twoafter;
+ size_t nn;
+
+ /* WB1 and WB2 */
+ if(it->n == 0 || it->n == it->ns)
+ return 1;
+ /* WB3 */
+ if(it->s[it->n-1] == 0x000D && it->s[it->n] == 0x000A)
+ return 0;
+ /* WB4 */
+ /* (!Sep) x (Extend|Format) as in UAX #29 s6.2 */
+ if(utf32__sentence_break(it->s[it->n-1]) != unicode_Sentence_Break_Sep
+ && utf32__boundary_ignorable(utf32__word_break(it->s[it->n])))
+ return 0;
+ /* Gather the property values we'll need for the rest of the test taking the
+ * s6.2 changes into account */
+ /* First we look at the code points after the proposed boundary */
+ nn = it->n; /* <it->ns */
+ after = utf32__word_break(it->s[nn++]);
+ if(!utf32__boundary_ignorable(after)) {
+ /* X (Extend|Format)* -> X */
+ while(nn < it->ns
+ && utf32__boundary_ignorable(utf32__word_break(it->s[nn])))
+ ++nn;
+ }
+ /* It's possible now that nn=ns */
+ if(nn < it->ns)
+ twoafter = utf32__word_break(it->s[nn]);
+ else
+ twoafter = unicode_Word_Break_Other;
+
+ /* We've already recorded the non-ignorable code points before the proposed
+ * boundary */
+ before = utf32__word_break(it->last[1]);
+ twobefore = utf32__word_break(it->last[0]);
+
+ /* WB5 */
+ if(before == unicode_Word_Break_ALetter
+ && after == unicode_Word_Break_ALetter)
+ return 0;
+ /* WB6 */
+ if(before == unicode_Word_Break_ALetter
+ && after == unicode_Word_Break_MidLetter
+ && twoafter == unicode_Word_Break_ALetter)
+ return 0;
+ /* WB7 */
+ if(twobefore == unicode_Word_Break_ALetter
+ && before == unicode_Word_Break_MidLetter
+ && after == unicode_Word_Break_ALetter)
+ return 0;
+ /* WB8 */
+ if(before == unicode_Word_Break_Numeric
+ && after == unicode_Word_Break_Numeric)
+ return 0;
+ /* WB9 */
+ if(before == unicode_Word_Break_ALetter
+ && after == unicode_Word_Break_Numeric)
+ return 0;
+ /* WB10 */
+ if(before == unicode_Word_Break_Numeric
+ && after == unicode_Word_Break_ALetter)
+ return 0;
+ /* WB11 */
+ if(twobefore == unicode_Word_Break_Numeric
+ && before == unicode_Word_Break_MidNum
+ && after == unicode_Word_Break_Numeric)
+ return 0;
+ /* WB12 */
+ if(before == unicode_Word_Break_Numeric
+ && after == unicode_Word_Break_MidNum
+ && twoafter == unicode_Word_Break_Numeric)
+ return 0;
+ /* WB13 */
+ if(before == unicode_Word_Break_Katakana
+ && after == unicode_Word_Break_Katakana)
+ return 0;
+ /* WB13a */
+ if((before == unicode_Word_Break_ALetter
+ || before == unicode_Word_Break_Numeric
+ || before == unicode_Word_Break_Katakana
+ || before == unicode_Word_Break_ExtendNumLet)
+ && after == unicode_Word_Break_ExtendNumLet)
+ return 0;
+ /* WB13b */
+ if(before == unicode_Word_Break_ExtendNumLet
+ && (after == unicode_Word_Break_ALetter
+ || after == unicode_Word_Break_Numeric
+ || after == unicode_Word_Break_Katakana))
+ return 0;
+ /* WB14 */
+ return 1;
+}
+
/*@}*/
/** @defgroup utf32 Functions that operate on UTF-32 strings */
/*@{*/
return (size_t)(t - s);
}
-/** @brief Return the @ref unidata structure for code point @p c
- *
- * @p c can be any 32-bit value, a sensible value will be returned regardless.
- */
-static const struct unidata *utf32__unidata(uint32_t c) {
- /* The bottom half of the table contains almost everything of interest
- * and we can just return the right thing straight away */
- if(c < UNICODE_BREAK_START)
- return &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
- /* Within the break everything is unassigned */
- if(c < UNICODE_BREAK_END)
- return utf32__unidata(0xFFFF); /* guaranteed to be Cn */
- /* Planes 15 and 16 are (mostly) private use */
- if((c >= 0xF0000 && c <= 0xFFFFD)
- || (c >= 0x100000 && c <= 0x10FFFD))
- return utf32__unidata(0xE000); /* first Co code point */
- /* Everything else above the break top is unassigned */
- if(c >= UNICODE_BREAK_TOP)
- return utf32__unidata(0xFFFF); /* guaranteed to be Cn */
- /* Currently the rest is language tags and variation selectors */
- c -= (UNICODE_BREAK_END - UNICODE_BREAK_START);
- return &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
-}
-
-/** @brief Return the combining class of @p c
- * @param c Code point
- * @return Combining class of @p c
- */
-static inline int utf32__combining_class(uint32_t c) {
- return utf32__unidata(c)->ccc;
-}
-
/** @brief Stably sort [s,s+ns) into descending order of combining class
* @param s Start of array
* @param ns Number of elements, must be at least 1
return *a < *b ? -1 : (*a > *b ? 1 : 0);
}
-/** @brief Return the General_Category value for @p c
- * @param Code point
- * @return General_Category property value
- */
-static inline enum unicode_General_Category utf32__general_category(uint32_t c) {
- return utf32__unidata(c)->general_category;
-}
-
-/** @brief Check Grapheme_Cluster_Break property
- * @param c Code point
- * @return 0 if it is as described, 1 otherwise
- */
-static int utf32__is_control_or_cr_or_lf(uint32_t c) {
- switch(utf32__general_category(c)) {
- default:
- return 0;
- case unicode_General_Category_Zl:
- case unicode_General_Category_Zp:
- case unicode_General_Category_Cc:
- return 1;
- case unicode_General_Category_Cf:
- if(c == 0x200C || c == 0x200D)
- return 0;
- return 1;
- }
-}
-
-#define Hangul_Syllable_Type_NA 0
-#define Hangul_Syllable_Type_L 0x1100
-#define Hangul_Syllable_Type_V 0x1160
-#define Hangul_Syllable_Type_T 0x11A8
-#define Hangul_Syllable_Type_LV 0xAC00
-#define Hangul_Syllable_Type_LVT 0xAC01
-
-/** @brief Determine Hangul_Syllable_Type of @p c
- * @param c Code point
- * @return Equivalance class of @p c, or Hangul_Syllable_Type_NA
- *
- * If this is a Hangul character then a representative member of its
- * equivalence class is returned. Otherwise Hangul_Syllable_Type_NA is
- * returned.
- */
-static uint32_t utf32__hangul_syllable_type(uint32_t c) {
- /* Dispose of the bulk of the non-Hangul code points first */
- if(c < 0x1100) return Hangul_Syllable_Type_NA;
- if(c > 0x1200 && c < 0xAC00) return Hangul_Syllable_Type_NA;
- if(c >= 0xD800) return Hangul_Syllable_Type_NA;
- /* Now we pick out the assigned Hangul code points */
- if((c >= 0x1100 && c <= 0x1159) || c == 0x115F) return Hangul_Syllable_Type_L;
- if(c >= 0x1160 && c <= 0x11A2) return Hangul_Syllable_Type_V;
- if(c >= 0x11A8 && c <= 0x11F9) return Hangul_Syllable_Type_T;
- if(c >= 0xAC00 && c <= 0xD7A3) {
- if(c % 28 == 16)
- return Hangul_Syllable_Type_LV;
- else
- return Hangul_Syllable_Type_LVT;
- }
- return Hangul_Syllable_Type_NA;
-}
-
-/** @brief Determine Word_Break property
- * @param c Code point
- * @return Word_Break property value of @p c
- */
-static enum unicode_Word_Break utf32__word_break(uint32_t c) {
- return utf32__unidata(c)->word_break;
-}
-
/** @brief Identify a grapheme cluster boundary
* @param s Start of string (must be NFD)
* @param ns Length of string
* grapheme cluster boundary (including the hypothetical code point just after
* the end of the string).
*/
-int utf32_is_gcb(const uint32_t *s, size_t ns, size_t n) {
- uint32_t before, after;
- uint32_t hbefore, hafter;
- /* GB1 and GB2 */
- if(n == 0 || n == ns)
- return 1;
- /* Now we know that s[n-1] and s[n] are safe to inspect */
- /* GB3 */
- before = s[n-1];
- after = s[n];
- if(before == 0x000D && after == 0x000A)
- return 0;
- /* GB4 and GB5 */
- if(utf32__is_control_or_cr_or_lf(before)
- || utf32__is_control_or_cr_or_lf(after))
- return 1;
- hbefore = utf32__hangul_syllable_type(before);
- hafter = utf32__hangul_syllable_type(after);
- /* GB6 */
- if(hbefore == Hangul_Syllable_Type_L
- && (hafter == Hangul_Syllable_Type_L
- || hafter == Hangul_Syllable_Type_V
- || hafter == Hangul_Syllable_Type_LV
- || hafter == Hangul_Syllable_Type_LVT))
- return 0;
- /* GB7 */
- if((hbefore == Hangul_Syllable_Type_LV
- || hbefore == Hangul_Syllable_Type_V)
- && (hafter == Hangul_Syllable_Type_V
- || hafter == Hangul_Syllable_Type_T))
- return 0;
- /* GB8 */
- if((hbefore == Hangul_Syllable_Type_LVT
- || hbefore == Hangul_Syllable_Type_T)
- && hafter == Hangul_Syllable_Type_T)
- return 0;
- /* GB9 */
- if(utf32__word_break(after) == unicode_Word_Break_Extend)
- return 0;
- /* GB10 */
- return 1;
-}
+int utf32_is_grapheme_boundary(const uint32_t *s, size_t ns, size_t n) {
+ struct utf32_iterator_data it[1];
-/** @brief Return true if @p c is ignorable for boundary specifications */
-static inline int utf32__boundary_ignorable(enum unicode_Word_Break wb) {
- return (wb == unicode_Word_Break_Extend
- || wb == unicode_Word_Break_Format);
+ utf32__iterator_init(it, s, ns, n);
+ return utf32_iterator_grapheme_boundary(it);
}
/** @brief Identify a word boundary
* (including the hypothetical code point just after the end of the string).
*/
int utf32_is_word_boundary(const uint32_t *s, size_t ns, size_t n) {
- enum unicode_Word_Break twobefore, before, after, twoafter;
- size_t nn;
+ struct utf32_iterator_data it[1];
- /* WB1 and WB2 */
- if(n == 0 || n == ns)
- return 1;
- /* WB3 */
- if(s[n-1] == 0x000D && s[n] == 0x000A)
- return 0;
- /* WB4 */
- /* (!Sep) x (Extend|Format) as in UAX #29 s6.2 */
- switch(s[n-1]) { /* bit of a bodge */
- case 0x000A:
- case 0x000D:
- case 0x0085:
- case 0x2028:
- case 0x2029:
- break;
- default:
- if(utf32__boundary_ignorable(utf32__word_break(s[n])))
- return 0;
- break;
- }
- /* Gather the property values we'll need for the rest of the test taking the
- * s6.2 changes into account */
- /* First we look at the code points after the proposed boundary */
- nn = n; /* <ns */
- after = utf32__word_break(s[nn++]);
- if(!utf32__boundary_ignorable(after)) {
- /* X (Extend|Format)* -> X */
- while(nn < ns && utf32__boundary_ignorable(utf32__word_break(s[nn])))
- ++nn;
- }
- /* It's possible now that nn=ns */
- if(nn < ns)
- twoafter = utf32__word_break(s[nn]);
- else
- twoafter = unicode_Word_Break_Other;
-
- /* Next we look at the code points before the proposed boundary. This is a
- * bit fiddlier. */
- nn = n;
- while(nn > 0 && utf32__boundary_ignorable(utf32__word_break(s[nn - 1])))
- --nn;
- if(nn == 0) {
- /* s[nn] must be ignorable */
- before = utf32__word_break(s[nn]);
- twobefore = unicode_Word_Break_Other;
- } else {
- /* s[nn] is ignorable or after the proposed boundary; but s[nn-1] is not
- * ignorable. */
- before = utf32__word_break(s[nn - 1]);
- --nn;
- /* Repeat the exercise */
- while(nn > 0 && utf32__boundary_ignorable(utf32__word_break(s[nn - 1])))
- --nn;
- if(nn == 0)
- twobefore = utf32__word_break(s[nn]);
- else
- twobefore = utf32__word_break(s[nn - 1]);
- }
-
- /* WB5 */
- if(before == unicode_Word_Break_ALetter
- && after == unicode_Word_Break_ALetter)
- return 0;
- /* WB6 */
- if(before == unicode_Word_Break_ALetter
- && after == unicode_Word_Break_MidLetter
- && twoafter == unicode_Word_Break_ALetter)
- return 0;
- /* WB7 */
- if(twobefore == unicode_Word_Break_ALetter
- && before == unicode_Word_Break_MidLetter
- && after == unicode_Word_Break_ALetter)
- return 0;
- /* WB8 */
- if(before == unicode_Word_Break_Numeric
- && after == unicode_Word_Break_Numeric)
- return 0;
- /* WB9 */
- if(before == unicode_Word_Break_ALetter
- && after == unicode_Word_Break_Numeric)
- return 0;
- /* WB10 */
- if(before == unicode_Word_Break_Numeric
- && after == unicode_Word_Break_ALetter)
- return 0;
- /* WB11 */
- if(twobefore == unicode_Word_Break_Numeric
- && before == unicode_Word_Break_MidNum
- && after == unicode_Word_Break_Numeric)
- return 0;
- /* WB12 */
- if(before == unicode_Word_Break_Numeric
- && after == unicode_Word_Break_MidNum
- && twoafter == unicode_Word_Break_Numeric)
- return 0;
- /* WB13 */
- if(before == unicode_Word_Break_Katakana
- && after == unicode_Word_Break_Katakana)
- return 0;
- /* WB13a */
- if((before == unicode_Word_Break_ALetter
- || before == unicode_Word_Break_Numeric
- || before == unicode_Word_Break_Katakana
- || before == unicode_Word_Break_ExtendNumLet)
- && after == unicode_Word_Break_ExtendNumLet)
- return 0;
- /* WB13b */
- if(before == unicode_Word_Break_ExtendNumLet
- && (after == unicode_Word_Break_ALetter
- || after == unicode_Word_Break_Numeric
- || after == unicode_Word_Break_Katakana))
- return 0;
- /* WB14 */
- return 1;
+ utf32__iterator_init(it, s, ns, n);
+ return utf32_iterator_word_boundary(it);
}
/*@}*/
~mdw / disorder / blobdiff