return (size_t)(t - s);
}
+/** @brief Return the @ref unidata structure for code point @p c
+ *
+ * @p c can be any 32-bit value, a sensible value will be returned regardless.
+ */
+static const struct unidata *utf32__unidata(uint32_t c) {
+ /* The bottom half of the table contains almost everything of interest
+ * and we can just return the right thing straight away */
+ if(c < UNICODE_BREAK_START)
+ return &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
+ /* Within the break everything is unassigned */
+ if(c < UNICODE_BREAK_END)
+ return utf32__unidata(0xFFFF); /* guaranteed to be Cn */
+ /* Planes 15 and 16 are (mostly) private use */
+ if((c >= 0xF0000 && c <= 0xFFFFD)
+ || (c >= 0x100000 && c <= 0x10FFFD))
+ return utf32__unidata(0xE000); /* first Co code point */
+ /* Everything else above the break top is unassigned */
+ if(c >= UNICODE_BREAK_TOP)
+ return utf32__unidata(0xFFFF); /* guaranteed to be Cn */
+ /* Currently the rest is language tags and variation selectors */
+ c -= (UNICODE_BREAK_END - UNICODE_BREAK_START);
+ return &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
+}
+
/** @brief Return the combining class of @p c
* @param c Code point
* @return Combining class of @p c
*/
static inline int utf32__combining_class(uint32_t c) {
- if(c < UNICODE_NCHARS)
- return unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].ccc;
- return 0;
+ return utf32__unidata(c)->ccc;
}
/** @brief Stably sort [s,s+ns) into descending order of combining class
/** @brief Guts of the decomposition lookup functions */
#define utf32__decompose_one_generic(WHICH) do { \
- const uint32_t *dc = \
- (c < UNICODE_NCHARS \
- ? unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].WHICH \
- : 0); \
+ const uint32_t *dc = utf32__unidata(c)->WHICH; \
if(dc) { \
/* Found a canonical decomposition in the table */ \
while(*dc) \
/** @brief Single-character case-fold and decompose operation */
#define utf32__casefold_one(WHICH) do { \
- const uint32_t *cf = \
- (c < UNICODE_NCHARS \
- ? unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].casefold \
- : 0); \
+ const uint32_t *cf = utf32__unidata(c)->casefold; \
if(cf) { \
/* Found a case-fold mapping in the table */ \
while(*cf) \
* normalize before we fold. In Unicode 5.0.0 this means 0345 COMBINING
* GREEK YPOGEGRAMMENI in its decomposition and the various characters that
* canonically decompose to it. */
- for(n = 0; n < ns; ++n) {
- c = s[n];
- if(c < UNICODE_NCHARS
- && (unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].flags
- & unicode_normalize_before_casefold))
+ for(n = 0; n < ns; ++n)
+ if(utf32__unidata(s[n])->flags & unicode_normalize_before_casefold)
break;
- }
if(n < ns) {
/* We need a preliminary decomposition */
if(!(ss = utf32_decompose_canon(s, ns, &ns)))
size_t n;
uint32_t *ss = 0;
- for(n = 0; n < ns; ++n) {
- c = s[n];
- if(c < UNICODE_NCHARS
- && (unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].flags
- & unicode_normalize_before_casefold))
+ for(n = 0; n < ns; ++n)
+ if(utf32__unidata(s[n])->flags & unicode_normalize_before_casefold)
break;
- }
if(n < ns) {
/* We need a preliminary _canonical_ decomposition */
if(!(ss = utf32_decompose_canon(s, ns, &ns)))
* @param Code point
* @return General_Category property value
*/
-static inline enum unicode_gc_cat utf32__general_category(uint32_t c) {
- if(c < UNICODE_NCHARS) {
- const struct unidata *const ud = &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
- return ud->gc;
- } else
- return unicode_gc_Cn;
+static inline enum unicode_General_Category utf32__general_category(uint32_t c) {
+ return utf32__unidata(c)->general_category;
}
/** @brief Check Grapheme_Cluster_Break property
switch(utf32__general_category(c)) {
default:
return 0;
- case unicode_gc_Zl:
- case unicode_gc_Zp:
- case unicode_gc_Cc:
+ case unicode_General_Category_Zl:
+ case unicode_General_Category_Zp:
+ case unicode_General_Category_Cc:
return 1;
- case unicode_gc_Cf:
+ case unicode_General_Category_Cf:
if(c == 0x200C || c == 0x200D)
return 0;
return 1;
* @return Word_Break property value of @p c
*/
static enum unicode_Word_Break utf32__word_break(uint32_t c) {
- if(c < 0xAC00 || c > 0xD7A3) {
- if(c < UNICODE_NCHARS)
- return unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].word_break;
- else
- return unicode_Word_Break_Other;
- } else
- return unicode_Word_Break_ALetter;
+ return utf32__unidata(c)->word_break;
}
/** @brief Identify a grapheme cluster boundary
return 1;
}
-/** @brief Return true if code point @p n is part of an initial sequence of Format/Extend
- * @param s Start of string
- * @param ns Length of string
- * @param n Start position
- * @return True if it is, false otherwise
- *
- * This whole stack is not very efficient; we assume we don't see many of the
- * problematic characters.
- */
-static int utf32__is_initial_sequence(const uint32_t *s,
- size_t attribute((unused)) ns,
- size_t n) {
- enum unicode_Word_Break wb;
-
- while(n > 0) {
- --n;
- wb = utf32__word_break(s[n]);
- if(wb != unicode_Word_Break_Extend
- && wb != unicode_Word_Break_Format)
- return 0;
- }
- return 1;
-}
-
-/** @brief Return the index of the first non-Extend/Format character from n
- * @param s Start of string
- * @param ns Length of string
- * @param n Start position
- * @return Index of first suitable character or @p ns
- */
-static size_t utf32__first_not_ignorable(const uint32_t *s, size_t ns,
- size_t n) {
- while(n < ns) {
- const enum unicode_Word_Break wb = utf32__word_break(s[n]);
- if((wb != unicode_Word_Break_Extend
- && wb != unicode_Word_Break_Format)
- || utf32__is_initial_sequence(s, ns, n))
- return n;
- ++n;
- }
- return ns;
-}
-
-/** @brief Return the index of the last non-Extend/Format character from n
- * @param s Start of string
- * @param ns Length of string
- * @param n Start position
- * @return Index of first suitable character or (size_t)-1
- */
-static size_t utf32__last_not_ignorable(const uint32_t *s, size_t ns,
- size_t n) {
- do {
- const enum unicode_Word_Break wb = utf32__word_break(s[n]);
- if((wb != unicode_Word_Break_Extend
- && wb != unicode_Word_Break_Format)
- || utf32__is_initial_sequence(s, ns, n))
- return n;
- } while(n--);
- return n; /* will be (size_t)-1 */
+/** @brief Return true if @p c is ignorable for boundary specifications */
+static inline int utf32__boundary_ignorable(enum unicode_Word_Break wb) {
+ return (wb == unicode_Word_Break_Extend
+ || wb == unicode_Word_Break_Format);
}
/** @brief Identify a word boundary
if(s[n-1] == 0x000D && s[n] == 0x000A)
return 0;
/* WB4 */
- /* The stated requirement here is to ignore code points with a Word_Break
- * value of _Extend or _Format wherever they may appear unless they are part
- * of an initial sequence of such characters. */
- twobefore = before = after = twoafter = unicode_Word_Break_Other;
- nn = utf32__last_not_ignorable(s, ns, n - 1/* > 0 */);
- if(nn != (size_t)-1) {
- before = utf32__word_break(s[nn]);
- if(nn != 0) {
- nn = utf32__last_not_ignorable(s, ns, nn - 1);
- if(nn != (size_t)-1)
- twobefore = utf32__word_break(s[nn]);
- }
+ /* (!Sep) x (Extend|Format) as in UAX #29 s6.2 */
+ switch(s[n-1]) { /* bit of a bodge */
+ case 0x000A:
+ case 0x000D:
+ case 0x0085:
+ case 0x2028:
+ case 0x2029:
+ break;
+ default:
+ if(utf32__boundary_ignorable(utf32__word_break(s[n])))
+ return 0;
+ break;
}
- nn = utf32__first_not_ignorable(s, ns, n);
- if(nn < ns) {
- after = utf32__word_break(s[nn]);
- if(nn < ns) {
- nn = utf32__first_not_ignorable(s, ns, nn + 1);
- if(nn < ns)
- twoafter = utf32__word_break(s[nn]);
- }
+ /* Gather the property values we'll need for the rest of the test taking the
+ * s6.2 changes into account */
+ /* First we look at the code points after the proposed boundary */
+ nn = n; /* <ns */
+ after = utf32__word_break(s[nn++]);
+ if(!utf32__boundary_ignorable(after)) {
+ /* X (Extend|Format)* -> X */
+ while(nn < ns && utf32__boundary_ignorable(utf32__word_break(s[nn])))
+ ++nn;
}
+ /* It's possible now that nn=ns */
+ if(nn < ns)
+ twoafter = utf32__word_break(s[nn]);
+ else
+ twoafter = unicode_Word_Break_Other;
+
+ /* Next we look at the code points before the proposed boundary. This is a
+ * bit fiddlier. */
+ nn = n;
+ while(nn > 0 && utf32__boundary_ignorable(utf32__word_break(s[nn - 1])))
+ --nn;
+ if(nn == 0) {
+ /* s[nn] must be ignorable */
+ before = utf32__word_break(s[nn]);
+ twobefore = unicode_Word_Break_Other;
+ } else {
+ /* s[nn] is ignorable or after the proposed boundary; but s[nn-1] is not
+ * ignorable. */
+ before = utf32__word_break(s[nn - 1]);
+ --nn;
+ /* Repeat the exercise */
+ while(nn > 0 && utf32__boundary_ignorable(utf32__word_break(s[nn - 1])))
+ --nn;
+ if(nn == 0)
+ twobefore = utf32__word_break(s[nn]);
+ else
+ twobefore = utf32__word_break(s[nn - 1]);
+ }
+
/* WB5 */
if(before == unicode_Word_Break_ALetter
&& after == unicode_Word_Break_ALetter)
}
/*@}*/
-/** @defgroup Functions that operate on UTF-8 strings */
+/** @defgroup utf8 Functions that operate on UTF-8 strings */
/*@{*/
/** @brief Wrapper to transform a UTF-8 string using the UTF-32 function */
~mdw / disorder / blobdiff