2 * This file is part of DisOrder
3 * Copyright (C) 2007 Richard Kettlewell
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
20 /** @file lib/unicode.c
21 * @brief Unicode support functions
23 * Here by UTF-8 and UTF-8 we mean the encoding forms of those names (not the
24 * encoding schemes). The primary encoding form is UTF-32 but convenience
25 * wrappers using UTF-8 are provided for a number of functions.
27 * The idea is that all the strings that hit the database will be in a
28 * particular normalization form, and for the search and tags database
29 * in case-folded form, so they can be naively compared within the
32 * As the code stands this guarantee is not well met!
39 #include <stdio.h> /* TODO */
46 /** @defgroup utf32props Unicode Code Point Properties */
49 static const struct unidata *utf32__unidata_hard(uint32_t c);
51 /** @brief Find definition of code point @p c
53 * @return Pointer to @ref unidata structure for @p c
55 * @p c can be any 32-bit value, a sensible value will be returned regardless.
56 * The returned pointer is NOT guaranteed to be unique to @p c.
58 static inline const struct unidata *utf32__unidata(uint32_t c) {
59 /* The bottom half of the table contains almost everything of interest
60 * and we can just return the right thing straight away */
61 if(c < UNICODE_BREAK_START)
62 return &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
64 return utf32__unidata_hard(c);
67 /** @brief Find definition of code point @p c
69 * @return Pointer to @ref unidata structure for @p c
71 * @p c can be any 32-bit value, a sensible value will be returned regardless.
72 * The returned pointer is NOT guaranteed to be unique to @p c.
74 * Don't use this function (although it will work fine) - use utf32__unidata()
77 static const struct unidata *utf32__unidata_hard(uint32_t c) {
78 if(c < UNICODE_BREAK_START)
79 return &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
80 /* Within the break everything is unassigned */
81 if(c < UNICODE_BREAK_END)
82 return utf32__unidata(0xFFFF); /* guaranteed to be Cn */
83 /* Planes 15 and 16 are (mostly) private use */
84 if((c >= 0xF0000 && c <= 0xFFFFD)
85 || (c >= 0x100000 && c <= 0x10FFFD))
86 return utf32__unidata(0xE000); /* first Co code point */
87 /* Everything else above the break top is unassigned */
88 if(c >= UNICODE_BREAK_TOP)
89 return utf32__unidata(0xFFFF); /* guaranteed to be Cn */
90 /* Currently the rest is language tags and variation selectors */
91 c -= (UNICODE_BREAK_END - UNICODE_BREAK_START);
92 return &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
95 /** @brief Return the combining class of @p c
97 * @return Combining class of @p c
99 * @p c can be any 32-bit value, a sensible value will be returned regardless.
101 static inline int utf32__combining_class(uint32_t c) {
102 return utf32__unidata(c)->ccc;
105 /** @brief Return the General_Category value for @p c
107 * @return General_Category property value
109 * @p c can be any 32-bit value, a sensible value will be returned regardless.
111 static inline enum unicode_General_Category utf32__general_category(uint32_t c) {
112 return utf32__unidata(c)->general_category;
115 /** @brief Determine Grapheme_Break property
116 * @param c Code point
117 * @return Grapheme_Break property value of @p c
119 * @p c can be any 32-bit value, a sensible value will be returned regardless.
121 static inline enum unicode_Grapheme_Break utf32__grapheme_break(uint32_t c) {
122 return utf32__unidata(c)->grapheme_break;
125 /** @brief Determine Word_Break property
126 * @param c Code point
127 * @return Word_Break property value of @p c
129 * @p c can be any 32-bit value, a sensible value will be returned regardless.
131 static inline enum unicode_Word_Break utf32__word_break(uint32_t c) {
132 return utf32__unidata(c)->word_break;
135 /** @brief Determine Sentence_Break property
136 * @param c Code point
137 * @return Word_Break property value of @p c
139 * @p c can be any 32-bit value, a sensible value will be returned regardless.
141 static inline enum unicode_Sentence_Break utf32__sentence_break(uint32_t c) {
142 return utf32__unidata(c)->sentence_break;
145 /** @brief Return true if @p c is ignorable for boundary specifications
146 * @param wb Word break property value
147 * @return non-0 if @p wb is unicode_Word_Break_Extend or unicode_Word_Break_Format
149 static inline int utf32__boundary_ignorable(enum unicode_Word_Break wb) {
150 return (wb == unicode_Word_Break_Extend
151 || wb == unicode_Word_Break_Format);
155 /** @defgroup utftransform Functions that transform between different Unicode encoding forms */
158 /** @brief Convert UTF-32 to UTF-8
159 * @param s Source string
160 * @param ns Length of source string in code points
161 * @param ndp Where to store length of destination string (or NULL)
162 * @return Newly allocated destination string or NULL on error
164 * If the UTF-32 is not valid then NULL is returned. A UTF-32 code point is
166 * - it codes for a UTF-16 surrogate
167 * - it codes for a value outside the unicode code space
169 * The return value is always 0-terminated. The value returned via @p *ndp
170 * does not include the terminator.
172 char *utf32_to_utf8(const uint32_t *s, size_t ns, size_t *ndp) {
180 dynstr_append(&d, c);
181 else if(c < 0x0800) {
182 dynstr_append(&d, 0xC0 | (c >> 6));
183 dynstr_append(&d, 0x80 | (c & 0x3F));
184 } else if(c < 0x10000) {
185 if(c >= 0xD800 && c <= 0xDFFF)
187 dynstr_append(&d, 0xE0 | (c >> 12));
188 dynstr_append(&d, 0x80 | ((c >> 6) & 0x3F));
189 dynstr_append(&d, 0x80 | (c & 0x3F));
190 } else if(c < 0x110000) {
191 dynstr_append(&d, 0xF0 | (c >> 18));
192 dynstr_append(&d, 0x80 | ((c >> 12) & 0x3F));
193 dynstr_append(&d, 0x80 | ((c >> 6) & 0x3F));
194 dynstr_append(&d, 0x80 | (c & 0x3F));
199 dynstr_terminate(&d);
208 /** @brief Convert UTF-8 to UTF-32
209 * @param s Source string
210 * @param ns Length of source string in code points
211 * @param ndp Where to store length of destination string (or NULL)
212 * @return Newly allocated destination string or NULL
214 * The return value is always 0-terminated. The value returned via @p *ndp
215 * does not include the terminator.
217 * If the UTF-8 is not valid then NULL is returned. A UTF-8 sequence
218 * for a code point is invalid if:
219 * - it is not the shortest possible sequence for the code point
220 * - it codes for a UTF-16 surrogate
221 * - it codes for a value outside the unicode code space
223 uint32_t *utf8_to_utf32(const char *s, size_t ns, size_t *ndp) {
224 struct dynstr_ucs4 d;
226 const uint8_t *ss = (const uint8_t *)s;
228 dynstr_ucs4_init(&d);
232 /* Acceptable UTF-8 is that which codes for Unicode Scalar Values
233 * (Unicode 5.0.0 s3.9 D76)
236 * 7 data bits gives 0x00 - 0x7F and all are acceptable
239 * 11 data bits gives 0x0000 - 0x07FF but only 0x0080 - 0x07FF acceptable
241 * 1110xxxx 10xxxxxx 10xxxxxx
242 * 16 data bits gives 0x0000 - 0xFFFF but only 0x0800 - 0xFFFF acceptable
243 * (and UTF-16 surrogates are not acceptable)
245 * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
246 * 21 data bits gives 0x00000000 - 0x001FFFFF
247 * but only 0x00010000 - 0x0010FFFF are acceptable
249 * It is NOT always the case that the data bits in the first byte are
250 * always non-0 for the acceptable values, so we do a separate check after
256 if(ns < 1) goto error;
259 if((c & 0xC0) != 0x80) goto error;
260 c32 = (c32 << 6) | (c & 0x3F);
261 if(c32 < 0x80) goto error;
262 } else if(c <= 0xEF) {
263 if(ns < 2) goto error;
266 if((c & 0xC0) != 0x80) goto error;
267 c32 = (c32 << 6) | (c & 0x3F);
269 if((c & 0xC0) != 0x80) goto error;
270 c32 = (c32 << 6) | (c & 0x3F);
271 if(c32 < 0x0800 || (c32 >= 0xD800 && c32 <= 0xDFFF)) goto error;
272 } else if(c <= 0xF7) {
273 if(ns < 3) goto error;
276 if((c & 0xC0) != 0x80) goto error;
277 c32 = (c32 << 6) | (c & 0x3F);
279 if((c & 0xC0) != 0x80) goto error;
280 c32 = (c32 << 6) | (c & 0x3F);
282 if((c & 0xC0) != 0x80) goto error;
283 c32 = (c32 << 6) | (c & 0x3F);
284 if(c32 < 0x00010000 || c32 > 0x0010FFFF) goto error;
287 dynstr_ucs4_append(&d, c32);
289 dynstr_ucs4_terminate(&d);
298 /** @brief Test whether [s,s+ns) is valid UTF-8
299 * @param s Start of string
300 * @param ns Length of string
301 * @return non-0 if @p s is valid UTF-8, 0 if it is not valid
303 * This function is intended to be much faster than calling utf8_to_utf32() and
304 * throwing away the result.
306 int utf8_valid(const char *s, size_t ns) {
307 const uint8_t *ss = (const uint8_t *)s;
309 const struct unicode_utf8_row *const r = &unicode_utf8_valid[*ss];
315 if(ss[1] < r->min2 || ss[1] > r->max2)
319 if(ss[1] < r->min2 || ss[1] > r->max2)
321 if(ss[2] < 0x80 || ss[2] > 0xBF)
325 if(ss[1] < r->min2 || ss[1] > r->max2)
327 if(ss[2] < 0x80 || ss[2] > 0xBF)
329 if(ss[3] < 0x80 || ss[3] > 0xBF)
344 /** @defgroup utf32iterator UTF-32 string iterators */
347 struct utf32_iterator_data {
348 /** @brief Start of string */
351 /** @brief Length of string */
354 /** @brief Current position */
357 /** @brief Last two non-ignorable characters or (uint32_t)-1
359 * last[1] is the non-Extend/Format character just before position @p n;
360 * last[0] is the one just before that.
362 * Exception 1: if there is no such non-Extend/Format character then an
363 * Extend/Format character is accepted instead.
365 * Exception 2: if there is no such character even taking that into account
366 * the value is (uint32_t)-1.
371 /** @brief Create a new iterator pointing at the start of a string
372 * @param s Start of string
373 * @param ns Length of string
374 * @return New iterator
376 utf32_iterator utf32_iterator_new(const uint32_t *s, size_t ns) {
377 utf32_iterator it = xmalloc(sizeof *it);
381 it->last[0] = it->last[1] = -1;
385 /** @brief Initialize an internal private iterator
387 * @param s Start of string
388 * @param ns Length of string
389 * @param n Absolute position
391 static void utf32__iterator_init(utf32_iterator it,
392 const uint32_t *s, size_t ns, size_t n) {
396 it->last[0] = it->last[1] = -1;
397 utf32_iterator_advance(it, n);
400 /** @brief Destroy an iterator
403 void utf32_iterator_destroy(utf32_iterator it) {
407 /** @brief Find the current position of an interator
410 size_t utf32_iterator_where(utf32_iterator it) {
414 /** @brief Set an iterator's absolute position
416 * @param n Absolute position
417 * @return 0 on success, non-0 on error
419 * It is an error to position the iterator outside the string (but acceptable
420 * to point it at the hypothetical post-final character). If an invalid value
421 * of @p n is specified then the iterator is not changed.
423 int utf32_iterator_set(utf32_iterator it, size_t n) {
424 /* TODO figure out how far we must back up to be able to re-synchronize; see
432 it->last[0] = it->last[1] = -1;
434 return utf32_iterator_advance(it, n);
437 /** @brief Advance an iterator
439 * @param count Number of code points to advance by
440 * @return 0 on success, non-0 on error
442 * It is an error to advance an iterator beyond the hypothetical post-final
443 * character of the string. If an invalid value of @p n is specified then the
444 * iterator is not changed.
446 * This function has O(n) time complexity: it works by advancing naively
447 * forwards through the string.
449 int utf32_iterator_advance(utf32_iterator it, size_t count) {
450 if(count <= it->ns - it->n) {
452 const uint32_t c = it->s[it->n];
453 const enum unicode_Word_Break wb = utf32__word_break(c);
454 if(it->last[1] == (uint32_t)-1
455 || !utf32__boundary_ignorable(wb)) {
456 it->last[0] = it->last[1];
467 /** @brief Find the current code point
469 * @return Current code point or 0
471 * If the iterator points at the hypothetical post-final character of the
472 * string then 0 is returned. NB that this doesn't mean that there aren't any
473 * 0 code points inside the string!
475 uint32_t utf32_iterator_code(utf32_iterator it) {
482 /** @brief Test for a grapheme boundary
484 * @return Non-0 if pointing just after a grapheme boundary, otherwise 0
486 int utf32_iterator_grapheme_boundary(utf32_iterator it) {
487 uint32_t before, after;
488 enum unicode_Grapheme_Break gbbefore, gbafter;
490 if(it->n == 0 || it->n == it->ns)
492 /* Now we know that s[n-1] and s[n] are safe to inspect */
494 before = it->s[it->n-1];
495 after = it->s[it->n];
496 if(before == 0x000D && after == 0x000A)
498 gbbefore = utf32__grapheme_break(before);
499 gbafter = utf32__grapheme_break(after);
501 if(gbbefore == unicode_Grapheme_Break_Control
506 if(gbafter == unicode_Grapheme_Break_Control
511 if(gbbefore == unicode_Grapheme_Break_L
512 && (gbafter == unicode_Grapheme_Break_L
513 || gbafter == unicode_Grapheme_Break_V
514 || gbafter == unicode_Grapheme_Break_LV
515 || gbafter == unicode_Grapheme_Break_LVT))
518 if((gbbefore == unicode_Grapheme_Break_LV
519 || gbbefore == unicode_Grapheme_Break_V)
520 && (gbafter == unicode_Grapheme_Break_V
521 || gbafter == unicode_Grapheme_Break_T))
524 if((gbbefore == unicode_Grapheme_Break_LVT
525 || gbbefore == unicode_Grapheme_Break_T)
526 && gbafter == unicode_Grapheme_Break_T)
529 if(gbafter == unicode_Grapheme_Break_Extend)
536 /** @brief Test for a word boundary
538 * @return Non-0 if pointing just after a word boundary, otherwise 0
540 int utf32_iterator_word_boundary(utf32_iterator it) {
541 enum unicode_Word_Break twobefore, before, after, twoafter;
545 if(it->n == 0 || it->n == it->ns)
548 if(it->s[it->n-1] == 0x000D && it->s[it->n] == 0x000A)
551 /* (!Sep) x (Extend|Format) as in UAX #29 s6.2 */
552 if(utf32__sentence_break(it->s[it->n-1]) != unicode_Sentence_Break_Sep
553 && utf32__boundary_ignorable(utf32__word_break(it->s[it->n])))
555 /* Gather the property values we'll need for the rest of the test taking the
556 * s6.2 changes into account */
557 /* First we look at the code points after the proposed boundary */
558 nn = it->n; /* <it->ns */
559 after = utf32__word_break(it->s[nn++]);
560 if(!utf32__boundary_ignorable(after)) {
561 /* X (Extend|Format)* -> X */
563 && utf32__boundary_ignorable(utf32__word_break(it->s[nn])))
566 /* It's possible now that nn=ns */
568 twoafter = utf32__word_break(it->s[nn]);
570 twoafter = unicode_Word_Break_Other;
572 /* We've already recorded the non-ignorable code points before the proposed
574 before = utf32__word_break(it->last[1]);
575 twobefore = utf32__word_break(it->last[0]);
578 if(before == unicode_Word_Break_ALetter
579 && after == unicode_Word_Break_ALetter)
582 if(before == unicode_Word_Break_ALetter
583 && after == unicode_Word_Break_MidLetter
584 && twoafter == unicode_Word_Break_ALetter)
587 if(twobefore == unicode_Word_Break_ALetter
588 && before == unicode_Word_Break_MidLetter
589 && after == unicode_Word_Break_ALetter)
592 if(before == unicode_Word_Break_Numeric
593 && after == unicode_Word_Break_Numeric)
596 if(before == unicode_Word_Break_ALetter
597 && after == unicode_Word_Break_Numeric)
600 if(before == unicode_Word_Break_Numeric
601 && after == unicode_Word_Break_ALetter)
604 if(twobefore == unicode_Word_Break_Numeric
605 && before == unicode_Word_Break_MidNum
606 && after == unicode_Word_Break_Numeric)
609 if(before == unicode_Word_Break_Numeric
610 && after == unicode_Word_Break_MidNum
611 && twoafter == unicode_Word_Break_Numeric)
614 if(before == unicode_Word_Break_Katakana
615 && after == unicode_Word_Break_Katakana)
618 if((before == unicode_Word_Break_ALetter
619 || before == unicode_Word_Break_Numeric
620 || before == unicode_Word_Break_Katakana
621 || before == unicode_Word_Break_ExtendNumLet)
622 && after == unicode_Word_Break_ExtendNumLet)
625 if(before == unicode_Word_Break_ExtendNumLet
626 && (after == unicode_Word_Break_ALetter
627 || after == unicode_Word_Break_Numeric
628 || after == unicode_Word_Break_Katakana))
635 /** @defgroup utf32 Functions that operate on UTF-32 strings */
638 /** @brief Return the length of a 0-terminated UTF-32 string
639 * @param s Pointer to 0-terminated string
640 * @return Length of string in code points (excluding terminator)
642 * Unlike the conversion functions no validity checking is done on the string.
644 size_t utf32_len(const uint32_t *s) {
645 const uint32_t *t = s;
649 return (size_t)(t - s);
652 /** @brief Stably sort [s,s+ns) into descending order of combining class
653 * @param s Start of array
654 * @param ns Number of elements, must be at least 1
655 * @param buffer Buffer of at least @p ns elements
657 static void utf32__sort_ccc(uint32_t *s, size_t ns, uint32_t *buffer) {
658 uint32_t *a, *b, *bp;
662 case 1: /* 1-element array is always sorted */
664 case 2: /* 2-element arrays are trivial to sort */
665 if(utf32__combining_class(s[0]) > utf32__combining_class(s[1])) {
672 /* Partition the array */
677 /* Sort the two halves of the array */
678 utf32__sort_ccc(a, na, buffer);
679 utf32__sort_ccc(b, nb, buffer);
680 /* Merge them back into one, via the buffer */
682 while(na > 0 && nb > 0) {
683 /* We want descending order of combining class (hence <)
684 * and we want stability within combining classes (hence <=)
686 if(utf32__combining_class(*a) <= utf32__combining_class(*b)) {
702 memcpy(s, buffer, ns * sizeof(uint32_t));
707 /** @brief Put combining characters into canonical order
708 * @param s Pointer to UTF-32 string
709 * @param ns Length of @p s
710 * @return 0 on success, -1 on error
712 * @p s is modified in-place. See Unicode 5.0 s3.11 for details of the
715 * Currently we only support a maximum of 1024 combining characters after each
716 * base character. If this limit is exceeded then -1 is returned.
718 static int utf32__canonical_ordering(uint32_t *s, size_t ns) {
720 uint32_t buffer[1024];
722 /* The ordering amounts to a stable sort of each contiguous group of
723 * characters with non-0 combining class. */
725 /* Skip non-combining characters */
726 if(utf32__combining_class(*s) == 0) {
731 /* We must now have at least one combining character; see how many
733 for(nc = 1; nc < ns && utf32__combining_class(s[nc]) != 0; ++nc)
738 utf32__sort_ccc(s, nc, buffer);
745 /* Magic numbers from UAX #15 s16 */
753 #define NCount (VCount * TCount)
754 #define SCount (LCount * NCount)
756 /** @brief Guts of the decomposition lookup functions */
757 #define utf32__decompose_one_generic(WHICH) do { \
758 const uint32_t *dc = utf32__unidata(c)->WHICH; \
760 /* Found a canonical decomposition in the table */ \
762 utf32__decompose_one_##WHICH(d, *dc++); \
763 } else if(c >= SBase && c < SBase + SCount) { \
764 /* Mechanically decomposable Hangul syllable (UAX #15 s16) */ \
765 const uint32_t SIndex = c - SBase; \
766 const uint32_t L = LBase + SIndex / NCount; \
767 const uint32_t V = VBase + (SIndex % NCount) / TCount; \
768 const uint32_t T = TBase + SIndex % TCount; \
769 dynstr_ucs4_append(d, L); \
770 dynstr_ucs4_append(d, V); \
772 dynstr_ucs4_append(d, T); \
774 /* Equal to own canonical decomposition */ \
775 dynstr_ucs4_append(d, c); \
778 /** @brief Recursively compute the canonical decomposition of @p c
779 * @param d Dynamic string to store decomposition in
780 * @param c Code point to decompose (must be a valid!)
781 * @return 0 on success, -1 on error
783 static void utf32__decompose_one_canon(struct dynstr_ucs4 *d, uint32_t c) {
784 utf32__decompose_one_generic(canon);
787 /** @brief Recursively compute the compatibility decomposition of @p c
788 * @param d Dynamic string to store decomposition in
789 * @param c Code point to decompose (must be a valid!)
790 * @return 0 on success, -1 on error
792 static void utf32__decompose_one_compat(struct dynstr_ucs4 *d, uint32_t c) {
793 utf32__decompose_one_generic(compat);
796 /** @brief Guts of the decomposition functions */
797 #define utf32__decompose_generic(WHICH) do { \
798 struct dynstr_ucs4 d; \
801 dynstr_ucs4_init(&d); \
804 if((c >= 0xD800 && c <= 0xDFFF) || c > 0x10FFFF) \
806 utf32__decompose_one_##WHICH(&d, c); \
809 if(utf32__canonical_ordering(d.vec, d.nvec)) \
811 dynstr_ucs4_terminate(&d); \
820 /** @brief Canonically decompose @p [s,s+ns)
821 * @param s Pointer to string
822 * @param ns Length of string
823 * @param ndp Where to store length of result
824 * @return Pointer to result string, or NULL
826 * Computes the canonical decomposition of a string and stably sorts combining
827 * characters into canonical order. The result is in Normalization Form D and
828 * (at the time of writing!) passes the NFD tests defined in Unicode 5.0's
829 * NormalizationTest.txt.
831 * Returns NULL if the string is not valid for either of the following reasons:
832 * - it codes for a UTF-16 surrogate
833 * - it codes for a value outside the unicode code space
835 uint32_t *utf32_decompose_canon(const uint32_t *s, size_t ns, size_t *ndp) {
836 utf32__decompose_generic(canon);
839 /** @brief Compatibility decompose @p [s,s+ns)
840 * @param s Pointer to string
841 * @param ns Length of string
842 * @param ndp Where to store length of result
843 * @return Pointer to result string, or NULL
845 * Computes the compatibility decomposition of a string and stably sorts
846 * combining characters into canonical order. The result is in Normalization
847 * Form KD and (at the time of writing!) passes the NFKD tests defined in
848 * Unicode 5.0's NormalizationTest.txt.
850 * Returns NULL if the string is not valid for either of the following reasons:
851 * - it codes for a UTF-16 surrogate
852 * - it codes for a value outside the unicode code space
854 uint32_t *utf32_decompose_compat(const uint32_t *s, size_t ns, size_t *ndp) {
855 utf32__decompose_generic(compat);
858 /** @brief Single-character case-fold and decompose operation */
859 #define utf32__casefold_one(WHICH) do { \
860 const uint32_t *cf = utf32__unidata(c)->casefold; \
862 /* Found a case-fold mapping in the table */ \
864 utf32__decompose_one_##WHICH(&d, *cf++); \
866 utf32__decompose_one_##WHICH(&d, c); \
869 /** @brief Case-fold @p [s,s+ns)
870 * @param s Pointer to string
871 * @param ns Length of string
872 * @param ndp Where to store length of result
873 * @return Pointer to result string, or NULL
875 * Case-fold the string at @p s according to full default case-folding rules
876 * (s3.13) for caseless matching. The result will be in NFD.
878 * Returns NULL if the string is not valid for either of the following reasons:
879 * - it codes for a UTF-16 surrogate
880 * - it codes for a value outside the unicode code space
882 uint32_t *utf32_casefold_canon(const uint32_t *s, size_t ns, size_t *ndp) {
883 struct dynstr_ucs4 d;
888 /* If the canonical decomposition of the string includes any combining
889 * character that case-folds to a non-combining character then we must
890 * normalize before we fold. In Unicode 5.0.0 this means 0345 COMBINING
891 * GREEK YPOGEGRAMMENI in its decomposition and the various characters that
892 * canonically decompose to it. */
893 for(n = 0; n < ns; ++n)
894 if(utf32__unidata(s[n])->flags & unicode_normalize_before_casefold)
897 /* We need a preliminary decomposition */
898 if(!(ss = utf32_decompose_canon(s, ns, &ns)))
902 dynstr_ucs4_init(&d);
905 if((c >= 0xD800 && c <= 0xDFFF) || c > 0x10FFFF)
907 utf32__casefold_one(canon);
910 if(utf32__canonical_ordering(d.vec, d.nvec))
912 dynstr_ucs4_terminate(&d);
922 /** @brief Compatibilit case-fold @p [s,s+ns)
923 * @param s Pointer to string
924 * @param ns Length of string
925 * @param ndp Where to store length of result
926 * @return Pointer to result string, or NULL
928 * Case-fold the string at @p s according to full default case-folding rules
929 * (s3.13) for compatibility caseless matching. The result will be in NFKD.
931 * Returns NULL if the string is not valid for either of the following reasons:
932 * - it codes for a UTF-16 surrogate
933 * - it codes for a value outside the unicode code space
935 uint32_t *utf32_casefold_compat(const uint32_t *s, size_t ns, size_t *ndp) {
936 struct dynstr_ucs4 d;
941 for(n = 0; n < ns; ++n)
942 if(utf32__unidata(s[n])->flags & unicode_normalize_before_casefold)
945 /* We need a preliminary _canonical_ decomposition */
946 if(!(ss = utf32_decompose_canon(s, ns, &ns)))
950 /* This computes NFKD(toCaseFold(s)) */
951 #define compat_casefold_middle() do { \
952 dynstr_ucs4_init(&d); \
955 if((c >= 0xD800 && c <= 0xDFFF) || c > 0x10FFFF) \
957 utf32__casefold_one(compat); \
960 if(utf32__canonical_ordering(d.vec, d.nvec)) \
963 /* Do the inner (NFKD o toCaseFold) */
964 compat_casefold_middle();
965 /* We can do away with the NFD'd copy of the input now */
969 /* Do the outer (NFKD o toCaseFold) */
970 compat_casefold_middle();
972 dynstr_ucs4_terminate(&d);
982 /** @brief Order a pair of UTF-32 strings
983 * @param a First 0-terminated string
984 * @param b Second 0-terminated string
985 * @return -1, 0 or 1 for a less than, equal to or greater than b
987 * "Comparable to strcmp() at its best."
989 int utf32_cmp(const uint32_t *a, const uint32_t *b) {
990 while(*a && *b && *a == *b) {
994 return *a < *b ? -1 : (*a > *b ? 1 : 0);
997 /** @brief Identify a grapheme cluster boundary
998 * @param s Start of string (must be NFD)
999 * @param ns Length of string
1000 * @param n Index within string (in [0,ns].)
1001 * @return 1 at a grapheme cluster boundary, 0 otherwise
1003 * This function identifies default grapheme cluster boundaries as described in
1004 * UAX #29 s3. It returns 1 if @p n points at the code point just after a
1005 * grapheme cluster boundary (including the hypothetical code point just after
1006 * the end of the string).
1008 int utf32_is_grapheme_boundary(const uint32_t *s, size_t ns, size_t n) {
1009 struct utf32_iterator_data it[1];
1011 utf32__iterator_init(it, s, ns, n);
1012 return utf32_iterator_grapheme_boundary(it);
1015 /** @brief Identify a word boundary
1016 * @param s Start of string (must be NFD)
1017 * @param ns Length of string
1018 * @param n Index within string (in [0,ns].)
1019 * @return 1 at a word boundary, 0 otherwise
1021 * This function identifies default word boundaries as described in UAX #29 s4.
1022 * It returns 1 if @p n points at the code point just after a word boundary
1023 * (including the hypothetical code point just after the end of the string).
1025 int utf32_is_word_boundary(const uint32_t *s, size_t ns, size_t n) {
1026 struct utf32_iterator_data it[1];
1028 utf32__iterator_init(it, s, ns, n);
1029 return utf32_iterator_word_boundary(it);
1033 /** @defgroup utf8 Functions that operate on UTF-8 strings */
1036 /** @brief Wrapper to transform a UTF-8 string using the UTF-32 function */
1037 #define utf8__transform(FN) do { \
1038 uint32_t *to32 = 0, *decomp32 = 0; \
1039 size_t nto32, ndecomp32; \
1040 char *decomp8 = 0; \
1042 if(!(to32 = utf8_to_utf32(s, ns, &nto32))) goto error; \
1043 if(!(decomp32 = FN(to32, nto32, &ndecomp32))) goto error; \
1044 decomp8 = utf32_to_utf8(decomp32, ndecomp32, ndp); \
1051 /** @brief Canonically decompose @p [s,s+ns)
1052 * @param s Pointer to string
1053 * @param ns Length of string
1054 * @param ndp Where to store length of result
1055 * @return Pointer to result string, or NULL
1057 * Computes the canonical decomposition of a string and stably sorts combining
1058 * characters into canonical order. The result is in Normalization Form D and
1059 * (at the time of writing!) passes the NFD tests defined in Unicode 5.0's
1060 * NormalizationTest.txt.
1062 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1065 * See also utf32_decompose_canon().
1067 char *utf8_decompose_canon(const char *s, size_t ns, size_t *ndp) {
1068 utf8__transform(utf32_decompose_canon);
1071 /** @brief Compatibility decompose @p [s,s+ns)
1072 * @param s Pointer to string
1073 * @param ns Length of string
1074 * @param ndp Where to store length of result
1075 * @return Pointer to result string, or NULL
1077 * Computes the compatibility decomposition of a string and stably sorts
1078 * combining characters into canonical order. The result is in Normalization
1079 * Form KD and (at the time of writing!) passes the NFKD tests defined in
1080 * Unicode 5.0's NormalizationTest.txt.
1082 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1085 * See also utf32_decompose_compat().
1087 char *utf8_decompose_compat(const char *s, size_t ns, size_t *ndp) {
1088 utf8__transform(utf32_decompose_compat);
1091 /** @brief Case-fold @p [s,s+ns)
1092 * @param s Pointer to string
1093 * @param ns Length of string
1094 * @param ndp Where to store length of result
1095 * @return Pointer to result string, or NULL
1097 * Case-fold the string at @p s according to full default case-folding rules
1098 * (s3.13). The result will be in NFD.
1100 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1103 char *utf8_casefold_canon(const char *s, size_t ns, size_t *ndp) {
1104 utf8__transform(utf32_casefold_canon);
1107 /** @brief Compatibility case-fold @p [s,s+ns)
1108 * @param s Pointer to string
1109 * @param ns Length of string
1110 * @param ndp Where to store length of result
1111 * @return Pointer to result string, or NULL
1113 * Case-fold the string at @p s according to full default case-folding rules
1114 * (s3.13). The result will be in NFKD.
1116 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1119 char *utf8_casefold_compat(const char *s, size_t ns, size_t *ndp) {
1120 utf8__transform(utf32_casefold_compat);
1130 indent-tabs-mode:nil