chiark / gitweb /
Switch to GPL v3
[disorder] / lib / unicode.c
CommitLineData
e5a5a138
RK
1/*
2 * This file is part of DisOrder
3 * Copyright (C) 2007 Richard Kettlewell
4 *
e7eb3a27 5 * This program is free software: you can redistribute it and/or modify
e5a5a138 6 * it under the terms of the GNU General Public License as published by
e7eb3a27 7 * the Free Software Foundation, either version 3 of the License, or
e5a5a138 8 * (at your option) any later version.
e7eb3a27
RK
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
e5a5a138 15 * You should have received a copy of the GNU General Public License
e7eb3a27 16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
e5a5a138
RK
17 */
18/** @file lib/unicode.c
19 * @brief Unicode support functions
20 *
21 * Here by UTF-8 and UTF-8 we mean the encoding forms of those names (not the
35b651f0
RK
22 * encoding schemes). The primary encoding form is UTF-32 but convenience
23 * wrappers using UTF-8 are provided for a number of functions.
e5a5a138
RK
24 *
25 * The idea is that all the strings that hit the database will be in a
26 * particular normalization form, and for the search and tags database
27 * in case-folded form, so they can be naively compared within the
28 * database code.
29 *
30 * As the code stands this guarantee is not well met!
0ae60b83
RK
31 *
32 * Subpages:
33 * - @ref utf32props
34 * - @ref utftransform
35 * - @ref utf32iterator
36 * - @ref utf32
37 * - @ref utf8
e5a5a138
RK
38 */
39
05b75f8d 40#include "common.h"
e5a5a138
RK
41
42#include "mem.h"
43#include "vector.h"
44#include "unicode.h"
45#include "unidata.h"
46
092f426f
RK
47/** @defgroup utf32props Unicode Code Point Properties */
48/*@{*/
49
50static const struct unidata *utf32__unidata_hard(uint32_t c);
51
52/** @brief Find definition of code point @p c
53 * @param c Code point
54 * @return Pointer to @ref unidata structure for @p c
55 *
56 * @p c can be any 32-bit value, a sensible value will be returned regardless.
57 * The returned pointer is NOT guaranteed to be unique to @p c.
58 */
59static inline const struct unidata *utf32__unidata(uint32_t c) {
60 /* The bottom half of the table contains almost everything of interest
61 * and we can just return the right thing straight away */
62 if(c < UNICODE_BREAK_START)
63 return &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
64 else
65 return utf32__unidata_hard(c);
66}
67
68/** @brief Find definition of code point @p c
69 * @param c Code point
70 * @return Pointer to @ref unidata structure for @p c
71 *
72 * @p c can be any 32-bit value, a sensible value will be returned regardless.
73 * The returned pointer is NOT guaranteed to be unique to @p c.
74 *
75 * Don't use this function (although it will work fine) - use utf32__unidata()
76 * instead.
77 */
78static const struct unidata *utf32__unidata_hard(uint32_t c) {
79 if(c < UNICODE_BREAK_START)
80 return &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
81 /* Within the break everything is unassigned */
82 if(c < UNICODE_BREAK_END)
83 return utf32__unidata(0xFFFF); /* guaranteed to be Cn */
84 /* Planes 15 and 16 are (mostly) private use */
85 if((c >= 0xF0000 && c <= 0xFFFFD)
86 || (c >= 0x100000 && c <= 0x10FFFD))
87 return utf32__unidata(0xE000); /* first Co code point */
88 /* Everything else above the break top is unassigned */
89 if(c >= UNICODE_BREAK_TOP)
90 return utf32__unidata(0xFFFF); /* guaranteed to be Cn */
91 /* Currently the rest is language tags and variation selectors */
92 c -= (UNICODE_BREAK_END - UNICODE_BREAK_START);
93 return &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
94}
95
96/** @brief Return the combining class of @p c
97 * @param c Code point
98 * @return Combining class of @p c
99 *
100 * @p c can be any 32-bit value, a sensible value will be returned regardless.
101 */
102static inline int utf32__combining_class(uint32_t c) {
103 return utf32__unidata(c)->ccc;
104}
105
3c82b504
RK
106/** @brief Return the combining class of @p c
107 * @param c Code point
108 * @return Combining class of @p c
109 *
110 * @p c can be any 32-bit value, a sensible value will be returned regardless.
111 */
112int utf32_combining_class(uint32_t c) {
113 return utf32__combining_class(c);
114}
115
092f426f 116/** @brief Return the General_Category value for @p c
0ae60b83 117 * @param c Code point
092f426f
RK
118 * @return General_Category property value
119 *
120 * @p c can be any 32-bit value, a sensible value will be returned regardless.
121 */
122static inline enum unicode_General_Category utf32__general_category(uint32_t c) {
123 return utf32__unidata(c)->general_category;
124}
125
126/** @brief Determine Grapheme_Break property
127 * @param c Code point
128 * @return Grapheme_Break property value of @p c
129 *
130 * @p c can be any 32-bit value, a sensible value will be returned regardless.
131 */
132static inline enum unicode_Grapheme_Break utf32__grapheme_break(uint32_t c) {
133 return utf32__unidata(c)->grapheme_break;
134}
135
136/** @brief Determine Word_Break property
137 * @param c Code point
138 * @return Word_Break property value of @p c
139 *
140 * @p c can be any 32-bit value, a sensible value will be returned regardless.
141 */
142static inline enum unicode_Word_Break utf32__word_break(uint32_t c) {
143 return utf32__unidata(c)->word_break;
144}
145
146/** @brief Determine Sentence_Break property
147 * @param c Code point
148 * @return Word_Break property value of @p c
149 *
150 * @p c can be any 32-bit value, a sensible value will be returned regardless.
151 */
152static inline enum unicode_Sentence_Break utf32__sentence_break(uint32_t c) {
153 return utf32__unidata(c)->sentence_break;
154}
155
156/** @brief Return true if @p c is ignorable for boundary specifications
157 * @param wb Word break property value
158 * @return non-0 if @p wb is unicode_Word_Break_Extend or unicode_Word_Break_Format
159 */
160static inline int utf32__boundary_ignorable(enum unicode_Word_Break wb) {
161 return (wb == unicode_Word_Break_Extend
162 || wb == unicode_Word_Break_Format);
163}
164
f98fcddb
RK
165/** @brief Return the canonical decomposition of @p c
166 * @param c Code point
167 * @return 0-terminated canonical decomposition, or 0
168 */
169static inline const uint32_t *utf32__decomposition_canon(uint32_t c) {
170 const struct unidata *const data = utf32__unidata(c);
171 const uint32_t *const decomp = data->decomp;
172
173 if(decomp && !(data->flags & unicode_compatibility_decomposition))
174 return decomp;
175 else
176 return 0;
177}
178
179/** @brief Return the compatibility decomposition of @p c
180 * @param c Code point
181 * @return 0-terminated decomposition, or 0
182 */
183static inline const uint32_t *utf32__decomposition_compat(uint32_t c) {
184 return utf32__unidata(c)->decomp;
185}
186
092f426f 187/*@}*/
e5a5a138
RK
188/** @defgroup utftransform Functions that transform between different Unicode encoding forms */
189/*@{*/
190
191/** @brief Convert UTF-32 to UTF-8
192 * @param s Source string
193 * @param ns Length of source string in code points
194 * @param ndp Where to store length of destination string (or NULL)
195 * @return Newly allocated destination string or NULL on error
196 *
56fd389c
RK
197 * If the UTF-32 is not valid then NULL is returned. A UTF-32 code point is
198 * invalid if:
e5a5a138
RK
199 * - it codes for a UTF-16 surrogate
200 * - it codes for a value outside the unicode code space
201 *
56fd389c
RK
202 * The return value is always 0-terminated. The value returned via @p *ndp
203 * does not include the terminator.
e5a5a138
RK
204 */
205char *utf32_to_utf8(const uint32_t *s, size_t ns, size_t *ndp) {
206 struct dynstr d;
207 uint32_t c;
208
209 dynstr_init(&d);
210 while(ns > 0) {
211 c = *s++;
212 if(c < 0x80)
213 dynstr_append(&d, c);
214 else if(c < 0x0800) {
215 dynstr_append(&d, 0xC0 | (c >> 6));
216 dynstr_append(&d, 0x80 | (c & 0x3F));
217 } else if(c < 0x10000) {
56fd389c 218 if(c >= 0xD800 && c <= 0xDFFF)
e5a5a138
RK
219 goto error;
220 dynstr_append(&d, 0xE0 | (c >> 12));
221 dynstr_append(&d, 0x80 | ((c >> 6) & 0x3F));
222 dynstr_append(&d, 0x80 | (c & 0x3F));
223 } else if(c < 0x110000) {
224 dynstr_append(&d, 0xF0 | (c >> 18));
225 dynstr_append(&d, 0x80 | ((c >> 12) & 0x3F));
226 dynstr_append(&d, 0x80 | ((c >> 6) & 0x3F));
227 dynstr_append(&d, 0x80 | (c & 0x3F));
228 } else
229 goto error;
230 --ns;
231 }
232 dynstr_terminate(&d);
233 if(ndp)
234 *ndp = d.nvec;
235 return d.vec;
236error:
237 xfree(d.vec);
238 return 0;
239}
240
241/** @brief Convert UTF-8 to UTF-32
242 * @param s Source string
243 * @param ns Length of source string in code points
244 * @param ndp Where to store length of destination string (or NULL)
f98fcddb 245 * @return Newly allocated destination string or NULL on error
e5a5a138 246 *
56fd389c
RK
247 * The return value is always 0-terminated. The value returned via @p *ndp
248 * does not include the terminator.
e5a5a138
RK
249 *
250 * If the UTF-8 is not valid then NULL is returned. A UTF-8 sequence
251 * for a code point is invalid if:
252 * - it is not the shortest possible sequence for the code point
253 * - it codes for a UTF-16 surrogate
254 * - it codes for a value outside the unicode code space
255 */
256uint32_t *utf8_to_utf32(const char *s, size_t ns, size_t *ndp) {
257 struct dynstr_ucs4 d;
32b158f2 258 uint32_t c32;
e5a5a138 259 const uint8_t *ss = (const uint8_t *)s;
32b158f2 260 int n;
e5a5a138
RK
261
262 dynstr_ucs4_init(&d);
263 while(ns > 0) {
32b158f2
RK
264 const struct unicode_utf8_row *const r = &unicode_utf8_valid[*ss];
265 if(r->count <= ns) {
266 switch(r->count) {
267 case 1:
268 c32 = *ss;
269 break;
270 case 2:
271 if(ss[1] < r->min2 || ss[1] > r->max2)
272 goto error;
273 c32 = *ss & 0x1F;
274 break;
275 case 3:
276 if(ss[1] < r->min2 || ss[1] > r->max2)
277 goto error;
278 c32 = *ss & 0x0F;
279 break;
280 case 4:
281 if(ss[1] < r->min2 || ss[1] > r->max2)
282 goto error;
283 c32 = *ss & 0x07;
284 break;
285 default:
286 goto error;
287 }
e5a5a138
RK
288 } else
289 goto error;
32b158f2
RK
290 for(n = 1; n < r->count; ++n) {
291 if(ss[n] < 0x80 || ss[n] > 0xBF)
292 goto error;
293 c32 = (c32 << 6) | (ss[n] & 0x3F);
294 }
e5a5a138 295 dynstr_ucs4_append(&d, c32);
32b158f2
RK
296 ss += r->count;
297 ns -= r->count;
e5a5a138
RK
298 }
299 dynstr_ucs4_terminate(&d);
300 if(ndp)
301 *ndp = d.nvec;
302 return d.vec;
303error:
304 xfree(d.vec);
305 return 0;
306}
307
18cda350
RK
308/** @brief Test whether [s,s+ns) is valid UTF-8
309 * @param s Start of string
310 * @param ns Length of string
311 * @return non-0 if @p s is valid UTF-8, 0 if it is not valid
312 *
313 * This function is intended to be much faster than calling utf8_to_utf32() and
314 * throwing away the result.
315 */
316int utf8_valid(const char *s, size_t ns) {
317 const uint8_t *ss = (const uint8_t *)s;
318 while(ns > 0) {
319 const struct unicode_utf8_row *const r = &unicode_utf8_valid[*ss];
320 if(r->count <= ns) {
321 switch(r->count) {
322 case 1:
323 break;
324 case 2:
325 if(ss[1] < r->min2 || ss[1] > r->max2)
326 return 0;
327 break;
328 case 3:
329 if(ss[1] < r->min2 || ss[1] > r->max2)
330 return 0;
331 if(ss[2] < 0x80 || ss[2] > 0xBF)
332 return 0;
333 break;
334 case 4:
335 if(ss[1] < r->min2 || ss[1] > r->max2)
336 return 0;
337 if(ss[2] < 0x80 || ss[2] > 0xBF)
338 return 0;
339 if(ss[3] < 0x80 || ss[3] > 0xBF)
340 return 0;
341 break;
342 default:
343 return 0;
344 }
345 } else
346 return 0;
347 ss += r->count;
348 ns -= r->count;
349 }
350 return 1;
351}
352
e5a5a138 353/*@}*/
092f426f
RK
354/** @defgroup utf32iterator UTF-32 string iterators */
355/*@{*/
356
357struct utf32_iterator_data {
358 /** @brief Start of string */
359 const uint32_t *s;
360
361 /** @brief Length of string */
362 size_t ns;
363
364 /** @brief Current position */
365 size_t n;
366
367 /** @brief Last two non-ignorable characters or (uint32_t)-1
368 *
369 * last[1] is the non-Extend/Format character just before position @p n;
370 * last[0] is the one just before that.
371 *
372 * Exception 1: if there is no such non-Extend/Format character then an
373 * Extend/Format character is accepted instead.
374 *
375 * Exception 2: if there is no such character even taking that into account
376 * the value is (uint32_t)-1.
377 */
378 uint32_t last[2];
092f426f 379
c85b7022
RK
380 /** @brief Tailoring for Word_Break */
381 unicode_property_tailor *word_break;
382};
092f426f
RK
383
384/** @brief Initialize an internal private iterator
385 * @param it Iterator
386 * @param s Start of string
387 * @param ns Length of string
388 * @param n Absolute position
389 */
390static void utf32__iterator_init(utf32_iterator it,
391 const uint32_t *s, size_t ns, size_t n) {
392 it->s = s;
393 it->ns = ns;
394 it->n = 0;
395 it->last[0] = it->last[1] = -1;
c85b7022 396 it->word_break = 0;
b21a155c 397 utf32_iterator_set(it, n);
092f426f
RK
398}
399
c85b7022
RK
400/** @brief Create a new iterator pointing at the start of a string
401 * @param s Start of string
402 * @param ns Length of string
403 * @return New iterator
404 */
405utf32_iterator utf32_iterator_new(const uint32_t *s, size_t ns) {
406 utf32_iterator it = xmalloc(sizeof *it);
407 utf32__iterator_init(it, s, ns, 0);
408 return it;
409}
410
411/** @brief Tailor this iterator's interpretation of the Word_Break property.
412 * @param it Iterator
413 * @param pt Property tailor function or NULL
414 *
415 * After calling this the iterator will call @p pt to determine the Word_Break
416 * property of each code point. If it returns -1 the default value will be
417 * used otherwise the returned value will be used.
418 *
419 * @p pt can be NULL to revert to the default value of the property.
420 *
421 * It is safe to call this function at any time; the iterator's internal state
422 * will be reset to suit the new tailoring.
423 */
424void utf32_iterator_tailor_word_break(utf32_iterator it,
425 unicode_property_tailor *pt) {
426 it->word_break = pt;
427 utf32_iterator_set(it, it->n);
428}
429
430static inline enum unicode_Word_Break utf32__iterator_word_break(utf32_iterator it,
431 uint32_t c) {
432 if(!it->word_break)
433 return utf32__word_break(c);
434 else {
435 const int t = it->word_break(c);
436
437 if(t < 0)
438 return utf32__word_break(c);
439 else
440 return t;
441 }
442}
443
092f426f
RK
444/** @brief Destroy an iterator
445 * @param it Iterator
446 */
447void utf32_iterator_destroy(utf32_iterator it) {
448 xfree(it);
449}
450
451/** @brief Find the current position of an interator
452 * @param it Iterator
453 */
454size_t utf32_iterator_where(utf32_iterator it) {
455 return it->n;
456}
457
458/** @brief Set an iterator's absolute position
459 * @param it Iterator
460 * @param n Absolute position
461 * @return 0 on success, non-0 on error
462 *
463 * It is an error to position the iterator outside the string (but acceptable
464 * to point it at the hypothetical post-final character). If an invalid value
465 * of @p n is specified then the iterator is not changed.
f98fcddb
RK
466 *
467 * This function works by backing up and then advancing to reconstruct the
468 * iterator's internal state for position @p n. The worst case will be O(n)
469 * time complexity (with a worse constant factor that utf32_iterator_advance())
470 * but the typical case is essentially constant-time.
092f426f
RK
471 */
472int utf32_iterator_set(utf32_iterator it, size_t n) {
5617aaff
RK
473 /* We can't just jump to position @p n; the @p last[] values will be wrong.
474 * What we need is to jump a bit behind @p n and then advance forward,
475 * updating @p last[] along the way. How far back? We need to cross two
476 * non-ignorable code points as we advance forwards, so we'd better pass two
477 * such characters on the way back (if such are available).
478 */
b21a155c 479 size_t m;
5617aaff
RK
480
481 if(n > it->ns) /* range check */
092f426f 482 return -1;
b21a155c
RK
483 /* Walk backwards skipping ignorable code points */
484 m = n;
c85b7022
RK
485 while(m > 0
486 && (utf32__boundary_ignorable(utf32__iterator_word_break(it,
487 it->s[m-1]))))
b21a155c
RK
488 --m;
489 /* Either m=0 or s[m-1] is not ignorable */
490 if(m > 0) {
491 --m;
492 /* s[m] is our first non-ignorable code; look for a second in the same
493 way **/
c85b7022
RK
494 while(m > 0
495 && (utf32__boundary_ignorable(utf32__iterator_word_break(it,
496 it->s[m-1]))))
5617aaff 497 --m;
b21a155c
RK
498 /* Either m=0 or s[m-1] is not ignorable */
499 if(m > 0)
500 --m;
501 }
502 it->last[0] = it->last[1] = -1;
5617aaff
RK
503 it->n = m;
504 return utf32_iterator_advance(it, n - m);
092f426f
RK
505}
506
507/** @brief Advance an iterator
508 * @param it Iterator
509 * @param count Number of code points to advance by
510 * @return 0 on success, non-0 on error
511 *
512 * It is an error to advance an iterator beyond the hypothetical post-final
513 * character of the string. If an invalid value of @p n is specified then the
514 * iterator is not changed.
515 *
516 * This function has O(n) time complexity: it works by advancing naively
517 * forwards through the string.
518 */
519int utf32_iterator_advance(utf32_iterator it, size_t count) {
520 if(count <= it->ns - it->n) {
521 while(count > 0) {
522 const uint32_t c = it->s[it->n];
c85b7022 523 const enum unicode_Word_Break wb = utf32__iterator_word_break(it, c);
092f426f
RK
524 if(it->last[1] == (uint32_t)-1
525 || !utf32__boundary_ignorable(wb)) {
526 it->last[0] = it->last[1];
527 it->last[1] = c;
528 }
529 ++it->n;
530 --count;
531 }
532 return 0;
533 } else
534 return -1;
535}
536
537/** @brief Find the current code point
538 * @param it Iterator
539 * @return Current code point or 0
540 *
541 * If the iterator points at the hypothetical post-final character of the
542 * string then 0 is returned. NB that this doesn't mean that there aren't any
543 * 0 code points inside the string!
544 */
545uint32_t utf32_iterator_code(utf32_iterator it) {
546 if(it->n < it->ns)
547 return it->s[it->n];
548 else
549 return 0;
550}
551
552/** @brief Test for a grapheme boundary
553 * @param it Iterator
554 * @return Non-0 if pointing just after a grapheme boundary, otherwise 0
f98fcddb
RK
555 *
556 * This function identifies default grapheme cluster boundaries as described in
557 * UAX #29 s3. It returns non-0 if @p it points at the code point just after a
558 * grapheme cluster boundary (including the hypothetical code point just after
559 * the end of the string).
092f426f
RK
560 */
561int utf32_iterator_grapheme_boundary(utf32_iterator it) {
562 uint32_t before, after;
563 enum unicode_Grapheme_Break gbbefore, gbafter;
564 /* GB1 and GB2 */
565 if(it->n == 0 || it->n == it->ns)
566 return 1;
567 /* Now we know that s[n-1] and s[n] are safe to inspect */
568 /* GB3 */
569 before = it->s[it->n-1];
570 after = it->s[it->n];
571 if(before == 0x000D && after == 0x000A)
572 return 0;
573 gbbefore = utf32__grapheme_break(before);
574 gbafter = utf32__grapheme_break(after);
575 /* GB4 */
576 if(gbbefore == unicode_Grapheme_Break_Control
577 || before == 0x000D
578 || before == 0x000A)
579 return 1;
580 /* GB5 */
581 if(gbafter == unicode_Grapheme_Break_Control
582 || after == 0x000D
583 || after == 0x000A)
584 return 1;
585 /* GB6 */
586 if(gbbefore == unicode_Grapheme_Break_L
587 && (gbafter == unicode_Grapheme_Break_L
588 || gbafter == unicode_Grapheme_Break_V
589 || gbafter == unicode_Grapheme_Break_LV
590 || gbafter == unicode_Grapheme_Break_LVT))
591 return 0;
592 /* GB7 */
593 if((gbbefore == unicode_Grapheme_Break_LV
594 || gbbefore == unicode_Grapheme_Break_V)
595 && (gbafter == unicode_Grapheme_Break_V
596 || gbafter == unicode_Grapheme_Break_T))
597 return 0;
598 /* GB8 */
599 if((gbbefore == unicode_Grapheme_Break_LVT
600 || gbbefore == unicode_Grapheme_Break_T)
601 && gbafter == unicode_Grapheme_Break_T)
602 return 0;
603 /* GB9 */
604 if(gbafter == unicode_Grapheme_Break_Extend)
605 return 0;
606 /* GB10 */
607 return 1;
608
609}
610
611/** @brief Test for a word boundary
612 * @param it Iterator
613 * @return Non-0 if pointing just after a word boundary, otherwise 0
f98fcddb
RK
614 *
615 * This function identifies default word boundaries as described in UAX #29 s4.
616 * It returns non-0 if @p it points at the code point just after a word
617 * boundary (including the hypothetical code point just after the end of the
618 * string) and 0 otherwise.
092f426f
RK
619 */
620int utf32_iterator_word_boundary(utf32_iterator it) {
621 enum unicode_Word_Break twobefore, before, after, twoafter;
622 size_t nn;
623
624 /* WB1 and WB2 */
625 if(it->n == 0 || it->n == it->ns)
626 return 1;
627 /* WB3 */
628 if(it->s[it->n-1] == 0x000D && it->s[it->n] == 0x000A)
629 return 0;
630 /* WB4 */
631 /* (!Sep) x (Extend|Format) as in UAX #29 s6.2 */
632 if(utf32__sentence_break(it->s[it->n-1]) != unicode_Sentence_Break_Sep
c85b7022 633 && utf32__boundary_ignorable(utf32__iterator_word_break(it, it->s[it->n])))
092f426f
RK
634 return 0;
635 /* Gather the property values we'll need for the rest of the test taking the
636 * s6.2 changes into account */
637 /* First we look at the code points after the proposed boundary */
638 nn = it->n; /* <it->ns */
c85b7022 639 after = utf32__iterator_word_break(it, it->s[nn++]);
092f426f
RK
640 if(!utf32__boundary_ignorable(after)) {
641 /* X (Extend|Format)* -> X */
642 while(nn < it->ns
c85b7022
RK
643 && utf32__boundary_ignorable(utf32__iterator_word_break(it,
644 it->s[nn])))
092f426f
RK
645 ++nn;
646 }
647 /* It's possible now that nn=ns */
648 if(nn < it->ns)
c85b7022 649 twoafter = utf32__iterator_word_break(it, it->s[nn]);
092f426f
RK
650 else
651 twoafter = unicode_Word_Break_Other;
652
653 /* We've already recorded the non-ignorable code points before the proposed
654 * boundary */
c85b7022
RK
655 before = utf32__iterator_word_break(it, it->last[1]);
656 twobefore = utf32__iterator_word_break(it, it->last[0]);
092f426f
RK
657
658 /* WB5 */
659 if(before == unicode_Word_Break_ALetter
660 && after == unicode_Word_Break_ALetter)
661 return 0;
662 /* WB6 */
663 if(before == unicode_Word_Break_ALetter
664 && after == unicode_Word_Break_MidLetter
665 && twoafter == unicode_Word_Break_ALetter)
666 return 0;
667 /* WB7 */
668 if(twobefore == unicode_Word_Break_ALetter
669 && before == unicode_Word_Break_MidLetter
670 && after == unicode_Word_Break_ALetter)
671 return 0;
c85b7022 672 /* WB8 */
092f426f
RK
673 if(before == unicode_Word_Break_Numeric
674 && after == unicode_Word_Break_Numeric)
675 return 0;
676 /* WB9 */
677 if(before == unicode_Word_Break_ALetter
678 && after == unicode_Word_Break_Numeric)
679 return 0;
680 /* WB10 */
681 if(before == unicode_Word_Break_Numeric
682 && after == unicode_Word_Break_ALetter)
683 return 0;
684 /* WB11 */
685 if(twobefore == unicode_Word_Break_Numeric
686 && before == unicode_Word_Break_MidNum
687 && after == unicode_Word_Break_Numeric)
688 return 0;
689 /* WB12 */
690 if(before == unicode_Word_Break_Numeric
691 && after == unicode_Word_Break_MidNum
692 && twoafter == unicode_Word_Break_Numeric)
693 return 0;
694 /* WB13 */
695 if(before == unicode_Word_Break_Katakana
696 && after == unicode_Word_Break_Katakana)
697 return 0;
698 /* WB13a */
699 if((before == unicode_Word_Break_ALetter
700 || before == unicode_Word_Break_Numeric
701 || before == unicode_Word_Break_Katakana
702 || before == unicode_Word_Break_ExtendNumLet)
703 && after == unicode_Word_Break_ExtendNumLet)
704 return 0;
705 /* WB13b */
706 if(before == unicode_Word_Break_ExtendNumLet
707 && (after == unicode_Word_Break_ALetter
708 || after == unicode_Word_Break_Numeric
709 || after == unicode_Word_Break_Katakana))
710 return 0;
711 /* WB14 */
712 return 1;
713}
714
715/*@}*/
e5a5a138
RK
716/** @defgroup utf32 Functions that operate on UTF-32 strings */
717/*@{*/
718
719/** @brief Return the length of a 0-terminated UTF-32 string
720 * @param s Pointer to 0-terminated string
721 * @return Length of string in code points (excluding terminator)
722 *
56fd389c 723 * Unlike the conversion functions no validity checking is done on the string.
e5a5a138
RK
724 */
725size_t utf32_len(const uint32_t *s) {
726 const uint32_t *t = s;
727
728 while(*t)
729 ++t;
730 return (size_t)(t - s);
731}
732
e5a5a138
RK
733/** @brief Stably sort [s,s+ns) into descending order of combining class
734 * @param s Start of array
735 * @param ns Number of elements, must be at least 1
736 * @param buffer Buffer of at least @p ns elements
737 */
738static void utf32__sort_ccc(uint32_t *s, size_t ns, uint32_t *buffer) {
739 uint32_t *a, *b, *bp;
740 size_t na, nb;
741
742 switch(ns) {
743 case 1: /* 1-element array is always sorted */
744 return;
745 case 2: /* 2-element arrays are trivial to sort */
746 if(utf32__combining_class(s[0]) > utf32__combining_class(s[1])) {
747 uint32_t tmp = s[0];
748 s[0] = s[1];
749 s[1] = tmp;
750 }
751 return;
752 default:
753 /* Partition the array */
754 na = ns / 2;
755 nb = ns - na;
756 a = s;
757 b = s + na;
758 /* Sort the two halves of the array */
759 utf32__sort_ccc(a, na, buffer);
760 utf32__sort_ccc(b, nb, buffer);
761 /* Merge them back into one, via the buffer */
762 bp = buffer;
763 while(na > 0 && nb > 0) {
16506c9d 764 /* We want ascending order of combining class (hence <)
e5a5a138
RK
765 * and we want stability within combining classes (hence <=)
766 */
767 if(utf32__combining_class(*a) <= utf32__combining_class(*b)) {
768 *bp++ = *a++;
769 --na;
770 } else {
771 *bp++ = *b++;
772 --nb;
773 }
774 }
775 while(na > 0) {
776 *bp++ = *a++;
777 --na;
778 }
779 while(nb > 0) {
780 *bp++ = *b++;
781 --nb;
782 }
783 memcpy(s, buffer, ns * sizeof(uint32_t));
784 return;
785 }
786}
787
788/** @brief Put combining characters into canonical order
789 * @param s Pointer to UTF-32 string
790 * @param ns Length of @p s
f98fcddb 791 * @return 0 on success, non-0 on error
e5a5a138 792 *
56fd389c
RK
793 * @p s is modified in-place. See Unicode 5.0 s3.11 for details of the
794 * ordering.
e5a5a138 795 *
56fd389c 796 * Currently we only support a maximum of 1024 combining characters after each
f98fcddb 797 * base character. If this limit is exceeded then a non-0 value is returned.
e5a5a138
RK
798 */
799static int utf32__canonical_ordering(uint32_t *s, size_t ns) {
800 size_t nc;
801 uint32_t buffer[1024];
802
803 /* The ordering amounts to a stable sort of each contiguous group of
804 * characters with non-0 combining class. */
805 while(ns > 0) {
806 /* Skip non-combining characters */
807 if(utf32__combining_class(*s) == 0) {
808 ++s;
809 --ns;
810 continue;
811 }
812 /* We must now have at least one combining character; see how many
813 * there are */
814 for(nc = 1; nc < ns && utf32__combining_class(s[nc]) != 0; ++nc)
815 ;
816 if(nc > 1024)
817 return -1;
818 /* Sort the array */
819 utf32__sort_ccc(s, nc, buffer);
820 s += nc;
821 ns -= nc;
822 }
823 return 0;
824}
825
826/* Magic numbers from UAX #15 s16 */
827#define SBase 0xAC00
828#define LBase 0x1100
829#define VBase 0x1161
830#define TBase 0x11A7
831#define LCount 19
832#define VCount 21
833#define TCount 28
834#define NCount (VCount * TCount)
835#define SCount (LCount * NCount)
836
837/** @brief Guts of the decomposition lookup functions */
838#define utf32__decompose_one_generic(WHICH) do { \
f98fcddb 839 const uint32_t *dc = utf32__decomposition_##WHICH(c); \
e5a5a138
RK
840 if(dc) { \
841 /* Found a canonical decomposition in the table */ \
842 while(*dc) \
843 utf32__decompose_one_##WHICH(d, *dc++); \
844 } else if(c >= SBase && c < SBase + SCount) { \
845 /* Mechanically decomposable Hangul syllable (UAX #15 s16) */ \
846 const uint32_t SIndex = c - SBase; \
847 const uint32_t L = LBase + SIndex / NCount; \
848 const uint32_t V = VBase + (SIndex % NCount) / TCount; \
849 const uint32_t T = TBase + SIndex % TCount; \
850 dynstr_ucs4_append(d, L); \
851 dynstr_ucs4_append(d, V); \
852 if(T != TBase) \
853 dynstr_ucs4_append(d, T); \
854 } else \
855 /* Equal to own canonical decomposition */ \
856 dynstr_ucs4_append(d, c); \
857} while(0)
858
859/** @brief Recursively compute the canonical decomposition of @p c
860 * @param d Dynamic string to store decomposition in
861 * @param c Code point to decompose (must be a valid!)
f98fcddb 862 * @return 0 on success, non-0 on error
e5a5a138
RK
863 */
864static void utf32__decompose_one_canon(struct dynstr_ucs4 *d, uint32_t c) {
865 utf32__decompose_one_generic(canon);
866}
867
868/** @brief Recursively compute the compatibility decomposition of @p c
869 * @param d Dynamic string to store decomposition in
870 * @param c Code point to decompose (must be a valid!)
f98fcddb 871 * @return 0 on success, non-0 on error
e5a5a138
RK
872 */
873static void utf32__decompose_one_compat(struct dynstr_ucs4 *d, uint32_t c) {
874 utf32__decompose_one_generic(compat);
875}
876
16506c9d
RK
877/** @brief Magic utf32__compositions() return value for Hangul Choseong */
878static const uint32_t utf32__hangul_L[1];
879
880/** @brief Return the list of compositions that @p c starts
881 * @param c Starter code point
882 * @return Composition list or NULL
883 *
884 * For Hangul leading (Choseong) jamo we return the special value
885 * utf32__hangul_L. These code points are not listed as the targets of
886 * canonical decompositions (make-unidata checks) so there is no confusion with
887 * real decompositions here.
888 */
889static const uint32_t *utf32__compositions(uint32_t c) {
890 const uint32_t *compositions = utf32__unidata(c)->composed;
891
892 if(compositions)
893 return compositions;
894 /* Special-casing for Hangul */
895 switch(utf32__grapheme_break(c)) {
896 default:
897 return 0;
898 case unicode_Grapheme_Break_L:
899 return utf32__hangul_L;
900 }
901}
902
903/** @brief Composition step
904 * @param s Start of string
905 * @param ns Length of string
906 * @return New length of string
907 *
908 * This is called from utf32__decompose_generic() to compose the result string
909 * in place.
910 */
911static size_t utf32__compose(uint32_t *s, size_t ns) {
912 const uint32_t *compositions;
913 uint32_t *start = s, *t = s, *tt, cc;
914
915 while(ns > 0) {
916 uint32_t starter = *s++;
917 int block_starters = 0;
918 --ns;
919 /* We don't attempt to compose the following things:
920 * - final characters whatever kind they are
921 * - non-starter characters
922 * - starters that don't take part in a canonical decomposition mapping
923 */
924 if(ns == 0
925 || utf32__combining_class(starter)
926 || !(compositions = utf32__compositions(starter))) {
927 *t++ = starter;
928 continue;
929 }
930 if(compositions != utf32__hangul_L) {
931 /* Where we'll put the eventual starter */
932 tt = t++;
933 do {
934 /* See if we can find composition of starter+*s */
935 const uint32_t cchar = *s, *cp = compositions;
936 while((cc = *cp++)) {
937 const uint32_t *decomp = utf32__decomposition_canon(cc);
938 /* We know decomp[0] == starter */
939 if(decomp[1] == cchar)
940 break;
941 }
942 if(cc) {
943 /* Found a composition: cc decomposes to starter,*s */
944 starter = cc;
945 compositions = utf32__compositions(starter);
946 ++s;
947 --ns;
948 } else {
949 /* No composition found. */
950 const int class = utf32__combining_class(*s);
951 if(class) {
952 /* Transfer the uncomposable combining character to the output */
953 *t++ = *s++;
954 --ns;
955 /* All the combining characters of the same class of the
956 * uncomposable character are blocked by it, but there may be
957 * others of higher class later. We eat the uncomposable and
958 * blocked characters and go back round the loop for that higher
959 * class. */
960 while(ns > 0 && utf32__combining_class(*s) == class) {
961 *t++ = *s++;
962 --ns;
963 }
964 /* Block any subsequent starters */
965 block_starters = 1;
966 } else {
967 /* The uncombinable character is itself a starter, so we don't
968 * transfer it to the output but instead go back round the main
969 * loop. */
970 break;
971 }
972 }
973 /* Keep going while there are still characters and the starter takes
974 * part in some composition */
975 } while(ns > 0 && compositions
976 && (!block_starters || utf32__combining_class(*s)));
977 /* Store any remaining combining characters */
978 while(ns > 0 && utf32__combining_class(*s)) {
979 *t++ = *s++;
980 --ns;
981 }
982 /* Store the resulting starter */
983 *tt = starter;
984 } else {
985 /* Special-casing for Hangul
986 *
987 * If there are combining characters between the L and the V then they
988 * will block the V and so no composition happens. Similarly combining
989 * characters between V and T will block the T and so we only get as far
990 * as LV.
991 */
992 if(utf32__grapheme_break(*s) == unicode_Grapheme_Break_V) {
993 const uint32_t V = *s++;
994 const uint32_t LIndex = starter - LBase;
995 const uint32_t VIndex = V - VBase;
996 uint32_t TIndex;
997 --ns;
998 if(ns > 0
999 && utf32__grapheme_break(*s) == unicode_Grapheme_Break_T) {
1000 /* We have an L V T sequence */
1001 const uint32_t T = *s++;
1002 TIndex = T - TBase;
1003 --ns;
1004 } else
1005 /* It's just L V */
1006 TIndex = 0;
1007 /* Compose to LVT or LV as appropriate */
1008 starter = (LIndex * VCount + VIndex) * TCount + TIndex + SBase;
1009 } /* else we only have L or LV and no V or T */
1010 *t++ = starter;
1011 /* There could be some combining characters that belong to the V or T.
1012 * These will be treated as non-starter characters at the top of the loop
1013 * and thuss transferred to the output. */
1014 }
1015 }
1016 return t - start;
1017}
1018
1019/** @brief Guts of the composition and decomposition functions
1020 * @param WHICH @c canon or @c compat to choose decomposition
1021 * @param COMPOSE @c 0 or @c 1 to compose
1022 */
1023#define utf32__decompose_generic(WHICH, COMPOSE) do { \
e5a5a138
RK
1024 struct dynstr_ucs4 d; \
1025 uint32_t c; \
1026 \
1027 dynstr_ucs4_init(&d); \
1028 while(ns) { \
1029 c = *s++; \
56fd389c 1030 if((c >= 0xD800 && c <= 0xDFFF) || c > 0x10FFFF) \
e5a5a138
RK
1031 goto error; \
1032 utf32__decompose_one_##WHICH(&d, c); \
1033 --ns; \
1034 } \
1035 if(utf32__canonical_ordering(d.vec, d.nvec)) \
1036 goto error; \
16506c9d
RK
1037 if(COMPOSE) \
1038 d.nvec = utf32__compose(d.vec, d.nvec); \
e5a5a138
RK
1039 dynstr_ucs4_terminate(&d); \
1040 if(ndp) \
1041 *ndp = d.nvec; \
1042 return d.vec; \
1043error: \
1044 xfree(d.vec); \
1045 return 0; \
1046} while(0)
1047
1048/** @brief Canonically decompose @p [s,s+ns)
1049 * @param s Pointer to string
1050 * @param ns Length of string
1051 * @param ndp Where to store length of result
f98fcddb 1052 * @return Pointer to result string, or NULL on error
e5a5a138 1053 *
16506c9d
RK
1054 * Computes NFD (Normalization Form D) of the string at @p s. This implies
1055 * performing all canonical decompositions and then normalizing the order of
1056 * combining characters.
e5a5a138 1057 *
56fd389c 1058 * Returns NULL if the string is not valid for either of the following reasons:
e5a5a138
RK
1059 * - it codes for a UTF-16 surrogate
1060 * - it codes for a value outside the unicode code space
16506c9d
RK
1061 *
1062 * See also:
1063 * - utf32_decompose_compat()
1064 * - utf32_compose_canon()
e5a5a138
RK
1065 */
1066uint32_t *utf32_decompose_canon(const uint32_t *s, size_t ns, size_t *ndp) {
16506c9d 1067 utf32__decompose_generic(canon, 0);
e5a5a138
RK
1068}
1069
1070/** @brief Compatibility decompose @p [s,s+ns)
1071 * @param s Pointer to string
1072 * @param ns Length of string
1073 * @param ndp Where to store length of result
f98fcddb 1074 * @return Pointer to result string, or NULL on error
e5a5a138 1075 *
16506c9d
RK
1076 * Computes NFKD (Normalization Form KD) of the string at @p s. This implies
1077 * performing all canonical and compatibility decompositions and then
1078 * normalizing the order of combining characters.
e5a5a138 1079 *
56fd389c 1080 * Returns NULL if the string is not valid for either of the following reasons:
e5a5a138
RK
1081 * - it codes for a UTF-16 surrogate
1082 * - it codes for a value outside the unicode code space
16506c9d
RK
1083 *
1084 * See also:
1085 * - utf32_decompose_canon()
1086 * - utf32_compose_compat()
e5a5a138
RK
1087 */
1088uint32_t *utf32_decompose_compat(const uint32_t *s, size_t ns, size_t *ndp) {
16506c9d
RK
1089 utf32__decompose_generic(compat, 0);
1090}
1091
1092/** @brief Canonically compose @p [s,s+ns)
1093 * @param s Pointer to string
1094 * @param ns Length of string
1095 * @param ndp Where to store length of result
1096 * @return Pointer to result string, or NULL on error
1097 *
1098 * Computes NFC (Normalization Form C) of the string at @p s. This implies
1099 * performing all canonical decompositions, normalizing the order of combining
1100 * characters and then composing all unblocked primary compositables.
1101 *
1102 * Returns NULL if the string is not valid for either of the following reasons:
1103 * - it codes for a UTF-16 surrogate
1104 * - it codes for a value outside the unicode code space
1105 *
1106 * See also:
1107 * - utf32_compose_compat()
1108 * - utf32_decompose_canon()
1109 */
1110uint32_t *utf32_compose_canon(const uint32_t *s, size_t ns, size_t *ndp) {
1111 utf32__decompose_generic(canon, 1);
1112}
1113
1114/** @brief Compatibility compose @p [s,s+ns)
1115 * @param s Pointer to string
1116 * @param ns Length of string
1117 * @param ndp Where to store length of result
1118 * @return Pointer to result string, or NULL on error
1119 *
1120 * Computes NFKC (Normalization Form KC) of the string at @p s. This implies
1121 * performing all canonical and compatibility decompositions, normalizing the
1122 * order of combining characters and then composing all unblocked primary
1123 * compositables.
1124 *
1125 * Returns NULL if the string is not valid for either of the following reasons:
1126 * - it codes for a UTF-16 surrogate
1127 * - it codes for a value outside the unicode code space
1128 *
1129 * See also:
1130 * - utf32_compose_canon()
1131 * - utf32_decompose_compat()
1132 */
1133uint32_t *utf32_compose_compat(const uint32_t *s, size_t ns, size_t *ndp) {
1134 utf32__decompose_generic(compat, 1);
e5a5a138
RK
1135}
1136
56fd389c
RK
1137/** @brief Single-character case-fold and decompose operation */
1138#define utf32__casefold_one(WHICH) do { \
bcf9ed7f 1139 const uint32_t *cf = utf32__unidata(c)->casefold; \
56fd389c
RK
1140 if(cf) { \
1141 /* Found a case-fold mapping in the table */ \
1142 while(*cf) \
1143 utf32__decompose_one_##WHICH(&d, *cf++); \
1144 } else \
1145 utf32__decompose_one_##WHICH(&d, c); \
1146} while(0)
e5a5a138
RK
1147
1148/** @brief Case-fold @p [s,s+ns)
1149 * @param s Pointer to string
1150 * @param ns Length of string
1151 * @param ndp Where to store length of result
f98fcddb 1152 * @return Pointer to result string, or NULL on error
e5a5a138
RK
1153 *
1154 * Case-fold the string at @p s according to full default case-folding rules
56fd389c 1155 * (s3.13) for caseless matching. The result will be in NFD.
e5a5a138 1156 *
56fd389c 1157 * Returns NULL if the string is not valid for either of the following reasons:
e5a5a138
RK
1158 * - it codes for a UTF-16 surrogate
1159 * - it codes for a value outside the unicode code space
1160 */
1161uint32_t *utf32_casefold_canon(const uint32_t *s, size_t ns, size_t *ndp) {
1162 struct dynstr_ucs4 d;
1163 uint32_t c;
1164 size_t n;
1165 uint32_t *ss = 0;
1166
1167 /* If the canonical decomposition of the string includes any combining
1168 * character that case-folds to a non-combining character then we must
1169 * normalize before we fold. In Unicode 5.0.0 this means 0345 COMBINING
1170 * GREEK YPOGEGRAMMENI in its decomposition and the various characters that
1171 * canonically decompose to it. */
bcf9ed7f
RK
1172 for(n = 0; n < ns; ++n)
1173 if(utf32__unidata(s[n])->flags & unicode_normalize_before_casefold)
e5a5a138 1174 break;
e5a5a138
RK
1175 if(n < ns) {
1176 /* We need a preliminary decomposition */
1177 if(!(ss = utf32_decompose_canon(s, ns, &ns)))
1178 return 0;
1179 s = ss;
1180 }
1181 dynstr_ucs4_init(&d);
1182 while(ns) {
1183 c = *s++;
56fd389c 1184 if((c >= 0xD800 && c <= 0xDFFF) || c > 0x10FFFF)
e5a5a138 1185 goto error;
56fd389c 1186 utf32__casefold_one(canon);
e5a5a138
RK
1187 --ns;
1188 }
1189 if(utf32__canonical_ordering(d.vec, d.nvec))
1190 goto error;
1191 dynstr_ucs4_terminate(&d);
1192 if(ndp)
1193 *ndp = d.nvec;
1194 return d.vec;
1195error:
1196 xfree(d.vec);
1197 xfree(ss);
1198 return 0;
1199}
1200
f98fcddb 1201/** @brief Compatibility case-fold @p [s,s+ns)
56fd389c
RK
1202 * @param s Pointer to string
1203 * @param ns Length of string
1204 * @param ndp Where to store length of result
f98fcddb 1205 * @return Pointer to result string, or NULL on error
56fd389c
RK
1206 *
1207 * Case-fold the string at @p s according to full default case-folding rules
1208 * (s3.13) for compatibility caseless matching. The result will be in NFKD.
1209 *
1210 * Returns NULL if the string is not valid for either of the following reasons:
1211 * - it codes for a UTF-16 surrogate
1212 * - it codes for a value outside the unicode code space
1213 */
1214uint32_t *utf32_casefold_compat(const uint32_t *s, size_t ns, size_t *ndp) {
1215 struct dynstr_ucs4 d;
1216 uint32_t c;
1217 size_t n;
1218 uint32_t *ss = 0;
1219
bcf9ed7f
RK
1220 for(n = 0; n < ns; ++n)
1221 if(utf32__unidata(s[n])->flags & unicode_normalize_before_casefold)
56fd389c 1222 break;
56fd389c
RK
1223 if(n < ns) {
1224 /* We need a preliminary _canonical_ decomposition */
1225 if(!(ss = utf32_decompose_canon(s, ns, &ns)))
1226 return 0;
1227 s = ss;
1228 }
1229 /* This computes NFKD(toCaseFold(s)) */
1230#define compat_casefold_middle() do { \
1231 dynstr_ucs4_init(&d); \
1232 while(ns) { \
1233 c = *s++; \
1234 if((c >= 0xD800 && c <= 0xDFFF) || c > 0x10FFFF) \
1235 goto error; \
1236 utf32__casefold_one(compat); \
1237 --ns; \
1238 } \
1239 if(utf32__canonical_ordering(d.vec, d.nvec)) \
1240 goto error; \
1241} while(0)
1242 /* Do the inner (NFKD o toCaseFold) */
1243 compat_casefold_middle();
1244 /* We can do away with the NFD'd copy of the input now */
1245 xfree(ss);
1246 s = ss = d.vec;
1247 ns = d.nvec;
1248 /* Do the outer (NFKD o toCaseFold) */
1249 compat_casefold_middle();
1250 /* That's all */
1251 dynstr_ucs4_terminate(&d);
1252 if(ndp)
1253 *ndp = d.nvec;
1254 return d.vec;
1255error:
1256 xfree(d.vec);
1257 xfree(ss);
1258 return 0;
1259}
1260
e5a5a138
RK
1261/** @brief Order a pair of UTF-32 strings
1262 * @param a First 0-terminated string
1263 * @param b Second 0-terminated string
1264 * @return -1, 0 or 1 for a less than, equal to or greater than b
1265 *
1266 * "Comparable to strcmp() at its best."
1267 */
1268int utf32_cmp(const uint32_t *a, const uint32_t *b) {
1269 while(*a && *b && *a == *b) {
1270 ++a;
1271 ++b;
1272 }
1273 return *a < *b ? -1 : (*a > *b ? 1 : 0);
1274}
1275
35b651f0
RK
1276/** @brief Identify a grapheme cluster boundary
1277 * @param s Start of string (must be NFD)
1278 * @param ns Length of string
1279 * @param n Index within string (in [0,ns].)
1280 * @return 1 at a grapheme cluster boundary, 0 otherwise
1281 *
1282 * This function identifies default grapheme cluster boundaries as described in
f98fcddb 1283 * UAX #29 s3. It returns non-0 if @p n points at the code point just after a
35b651f0
RK
1284 * grapheme cluster boundary (including the hypothetical code point just after
1285 * the end of the string).
f98fcddb
RK
1286 *
1287 * This function uses utf32_iterator_set() internally; see that function for
1288 * remarks on performance.
35b651f0 1289 */
1625e11a 1290int utf32_is_grapheme_boundary(const uint32_t *s, size_t ns, size_t n) {
092f426f 1291 struct utf32_iterator_data it[1];
35b651f0 1292
092f426f
RK
1293 utf32__iterator_init(it, s, ns, n);
1294 return utf32_iterator_grapheme_boundary(it);
0b7052da
RK
1295}
1296
1297/** @brief Identify a word boundary
1298 * @param s Start of string (must be NFD)
1299 * @param ns Length of string
1300 * @param n Index within string (in [0,ns].)
1301 * @return 1 at a word boundary, 0 otherwise
1302 *
1303 * This function identifies default word boundaries as described in UAX #29 s4.
f98fcddb 1304 * It returns non-0 if @p n points at the code point just after a word boundary
0b7052da 1305 * (including the hypothetical code point just after the end of the string).
f98fcddb
RK
1306 *
1307 * This function uses utf32_iterator_set() internally; see that function for
1308 * remarks on performance.
0b7052da
RK
1309 */
1310int utf32_is_word_boundary(const uint32_t *s, size_t ns, size_t n) {
092f426f 1311 struct utf32_iterator_data it[1];
0b7052da 1312
092f426f
RK
1313 utf32__iterator_init(it, s, ns, n);
1314 return utf32_iterator_word_boundary(it);
0b7052da
RK
1315}
1316
8818b7fc
RK
1317/** @brief Split [s,ns) into multiple words
1318 * @param s Pointer to start of string
1319 * @param ns Length of string
1320 * @param nwp Where to store word count, or NULL
c85b7022 1321 * @param wbreak Word_Break property tailor, or NULL
8818b7fc
RK
1322 * @return Pointer to array of pointers to words
1323 *
1324 * The returned array is terminated by a NULL pointer and individual
1325 * strings are 0-terminated.
1326 */
c85b7022
RK
1327uint32_t **utf32_word_split(const uint32_t *s, size_t ns, size_t *nwp,
1328 unicode_property_tailor *wbreak) {
8818b7fc
RK
1329 struct utf32_iterator_data it[1];
1330 size_t b1 = 0, b2 = 0 ,i;
1331 int isword;
1332 struct vector32 v32[1];
1333 uint32_t *w;
1334
1335 vector32_init(v32);
1336 utf32__iterator_init(it, s, ns, 0);
c85b7022 1337 it->word_break = wbreak;
8818b7fc
RK
1338 /* Work our way through the string stopping at each word break. */
1339 do {
1340 if(utf32_iterator_word_boundary(it)) {
1341 /* We've found a new boundary */
1342 b1 = b2;
1343 b2 = it->n;
1344 /*fprintf(stderr, "[%zu, %zu) is a candidate word\n", b1, b2);*/
1345 /* Inspect the characters between the boundary and form an opinion as to
1346 * whether they are a word or not */
1347 isword = 0;
1348 for(i = b1; i < b2; ++i) {
c85b7022 1349 switch(utf32__iterator_word_break(it, it->s[i])) {
8818b7fc
RK
1350 case unicode_Word_Break_ALetter:
1351 case unicode_Word_Break_Numeric:
1352 case unicode_Word_Break_Katakana:
1353 isword = 1;
1354 break;
1355 default:
1356 break;
1357 }
1358 }
1359 /* If it's a word add it to the list of results */
1360 if(isword) {
1361 w = xcalloc(b2 - b1 + 1, sizeof(uint32_t));
1362 memcpy(w, it->s + b1, (b2 - b1) * sizeof (uint32_t));
1363 vector32_append(v32, w);
1364 }
1365 }
1366 } while(!utf32_iterator_advance(it, 1));
1367 vector32_terminate(v32);
1368 if(nwp)
1369 *nwp = v32->nvec;
1370 return v32->vec;
1371}
1372
e5a5a138 1373/*@}*/
349b7b74 1374/** @defgroup utf8 Functions that operate on UTF-8 strings */
e5a5a138
RK
1375/*@{*/
1376
1377/** @brief Wrapper to transform a UTF-8 string using the UTF-32 function */
1378#define utf8__transform(FN) do { \
1379 uint32_t *to32 = 0, *decomp32 = 0; \
1380 size_t nto32, ndecomp32; \
1381 char *decomp8 = 0; \
1382 \
1383 if(!(to32 = utf8_to_utf32(s, ns, &nto32))) goto error; \
1384 if(!(decomp32 = FN(to32, nto32, &ndecomp32))) goto error; \
1385 decomp8 = utf32_to_utf8(decomp32, ndecomp32, ndp); \
1386error: \
1387 xfree(to32); \
1388 xfree(decomp32); \
1389 return decomp8; \
1390} while(0)
1391
1392/** @brief Canonically decompose @p [s,s+ns)
1393 * @param s Pointer to string
1394 * @param ns Length of string
1395 * @param ndp Where to store length of result
f98fcddb 1396 * @return Pointer to result string, or NULL on error
e5a5a138 1397 *
0ae60b83
RK
1398 * Computes NFD (Normalization Form D) of the string at @p s. This implies
1399 * performing all canonical decompositions and then normalizing the order of
1400 * combining characters.
e5a5a138
RK
1401 *
1402 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1403 * this might be.
1404 *
0ae60b83
RK
1405 * See also:
1406 * - utf32_decompose_canon().
1407 * - utf8_decompose_compat()
1408 * - utf8_compose_canon()
e5a5a138
RK
1409 */
1410char *utf8_decompose_canon(const char *s, size_t ns, size_t *ndp) {
1411 utf8__transform(utf32_decompose_canon);
1412}
1413
1414/** @brief Compatibility decompose @p [s,s+ns)
1415 * @param s Pointer to string
1416 * @param ns Length of string
1417 * @param ndp Where to store length of result
f98fcddb 1418 * @return Pointer to result string, or NULL on error
e5a5a138 1419 *
0ae60b83
RK
1420 * Computes NFKD (Normalization Form KD) of the string at @p s. This implies
1421 * performing all canonical and compatibility decompositions and then
1422 * normalizing the order of combining characters.
e5a5a138
RK
1423 *
1424 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1425 * this might be.
1426 *
0ae60b83
RK
1427 * See also:
1428 * - utf32_decompose_compat().
1429 * - utf8_decompose_canon()
1430 * - utf8_compose_compat()
e5a5a138
RK
1431 */
1432char *utf8_decompose_compat(const char *s, size_t ns, size_t *ndp) {
1433 utf8__transform(utf32_decompose_compat);
1434}
1435
0ae60b83
RK
1436/** @brief Canonically compose @p [s,s+ns)
1437 * @param s Pointer to string
1438 * @param ns Length of string
1439 * @param ndp Where to store length of result
1440 * @return Pointer to result string, or NULL on error
1441 *
1442 * Computes NFC (Normalization Form C) of the string at @p s. This implies
1443 * performing all canonical decompositions, normalizing the order of combining
1444 * characters and then composing all unblocked primary compositables.
1445 *
1446 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1447 * this might be.
1448 *
1449 * See also:
1450 * - utf32_compose_canon()
1451 * - utf8_compose_compat()
1452 * - utf8_decompose_canon()
1453 */
1454char *utf8_compose_canon(const char *s, size_t ns, size_t *ndp) {
1455 utf8__transform(utf32_compose_canon);
1456}
1457
1458/** @brief Compatibility compose @p [s,s+ns)
1459 * @param s Pointer to string
1460 * @param ns Length of string
1461 * @param ndp Where to store length of result
1462 * @return Pointer to result string, or NULL on error
1463 *
1464 * Computes NFKC (Normalization Form KC) of the string at @p s. This implies
1465 * performing all canonical and compatibility decompositions, normalizing the
1466 * order of combining characters and then composing all unblocked primary
1467 * compositables.
1468 *
1469 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1470 * this might be.
1471 *
1472 * See also:
1473 * - utf32_compose_compat()
1474 * - utf8_compose_canon()
1475 * - utf8_decompose_compat()
1476 */
1477char *utf8_compose_compat(const char *s, size_t ns, size_t *ndp) {
1478 utf8__transform(utf32_compose_compat);
1479}
1480
e5a5a138
RK
1481/** @brief Case-fold @p [s,s+ns)
1482 * @param s Pointer to string
1483 * @param ns Length of string
1484 * @param ndp Where to store length of result
f98fcddb 1485 * @return Pointer to result string, or NULL on error
e5a5a138
RK
1486 *
1487 * Case-fold the string at @p s according to full default case-folding rules
1488 * (s3.13). The result will be in NFD.
1489 *
1490 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1491 * this might be.
1492 */
1493char *utf8_casefold_canon(const char *s, size_t ns, size_t *ndp) {
1494 utf8__transform(utf32_casefold_canon);
1495}
1496
1497/** @brief Compatibility case-fold @p [s,s+ns)
1498 * @param s Pointer to string
1499 * @param ns Length of string
1500 * @param ndp Where to store length of result
f98fcddb 1501 * @return Pointer to result string, or NULL on error
e5a5a138
RK
1502 *
1503 * Case-fold the string at @p s according to full default case-folding rules
1504 * (s3.13). The result will be in NFKD.
1505 *
1506 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1507 * this might be.
1508 */
e5a5a138
RK
1509char *utf8_casefold_compat(const char *s, size_t ns, size_t *ndp) {
1510 utf8__transform(utf32_casefold_compat);
1511}
e5a5a138 1512
8818b7fc
RK
1513/** @brief Split [s,ns) into multiple words
1514 * @param s Pointer to start of string
1515 * @param ns Length of string
1516 * @param nwp Where to store word count, or NULL
c85b7022 1517 * @param wbreak Word_Break property tailor, or NULL
8818b7fc
RK
1518 * @return Pointer to array of pointers to words
1519 *
1520 * The returned array is terminated by a NULL pointer and individual
1521 * strings are 0-terminated.
1522 */
c85b7022
RK
1523char **utf8_word_split(const char *s, size_t ns, size_t *nwp,
1524 unicode_property_tailor *wbreak) {
8818b7fc
RK
1525 uint32_t *to32 = 0, **v32 = 0;
1526 size_t nto32, nv, n;
1527 char **v8 = 0, **ret = 0;
c85b7022 1528
8818b7fc 1529 if(!(to32 = utf8_to_utf32(s, ns, &nto32))) goto error;
c85b7022 1530 if(!(v32 = utf32_word_split(to32, nto32, &nv, wbreak))) goto error;
8818b7fc
RK
1531 v8 = xcalloc(sizeof (char *), nv + 1);
1532 for(n = 0; n < nv; ++n)
1533 if(!(v8[n] = utf32_to_utf8(v32[n], utf32_len(v32[n]), 0)))
1534 goto error;
1535 ret = v8;
1536 *nwp = nv;
1537 v8 = 0; /* don't free */
c85b7022 1538error:
8818b7fc
RK
1539 if(v8) {
1540 for(n = 0; n < nv; ++n)
1541 xfree(v8[n]);
1542 xfree(v8);
1543 }
1544 if(v32) {
1545 for(n = 0; n < nv; ++n)
1546 xfree(v32[n]);
1547 xfree(v32);
1548 }
1549 xfree(to32);
1550 return ret;
1551}
1552
1553
e5a5a138
RK
1554/*@}*/
1555
1556/*
1557Local Variables:
1558c-basic-offset:2
1559comment-column:40
1560fill-column:79
1561indent-tabs-mode:nil
1562End:
1563*/