chiark / gitweb /
partial tests for kvp.c
[disorder] / lib / unicode.c
CommitLineData
e5a5a138
RK
1/*
2 * This file is part of DisOrder
3 * Copyright (C) 2007 Richard Kettlewell
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
18 * USA
19 */
20/** @file lib/unicode.c
21 * @brief Unicode support functions
22 *
23 * Here by UTF-8 and UTF-8 we mean the encoding forms of those names (not the
35b651f0
RK
24 * encoding schemes). The primary encoding form is UTF-32 but convenience
25 * wrappers using UTF-8 are provided for a number of functions.
e5a5a138
RK
26 *
27 * The idea is that all the strings that hit the database will be in a
28 * particular normalization form, and for the search and tags database
29 * in case-folded form, so they can be naively compared within the
30 * database code.
31 *
32 * As the code stands this guarantee is not well met!
0ae60b83
RK
33 *
34 * Subpages:
35 * - @ref utf32props
36 * - @ref utftransform
37 * - @ref utf32iterator
38 * - @ref utf32
39 * - @ref utf8
e5a5a138
RK
40 */
41
42#include <config.h>
43#include "types.h"
44
45#include <string.h>
46#include <stdio.h> /* TODO */
47
48#include "mem.h"
49#include "vector.h"
50#include "unicode.h"
51#include "unidata.h"
52
092f426f
RK
53/** @defgroup utf32props Unicode Code Point Properties */
54/*@{*/
55
56static const struct unidata *utf32__unidata_hard(uint32_t c);
57
58/** @brief Find definition of code point @p c
59 * @param c Code point
60 * @return Pointer to @ref unidata structure for @p c
61 *
62 * @p c can be any 32-bit value, a sensible value will be returned regardless.
63 * The returned pointer is NOT guaranteed to be unique to @p c.
64 */
65static inline const struct unidata *utf32__unidata(uint32_t c) {
66 /* The bottom half of the table contains almost everything of interest
67 * and we can just return the right thing straight away */
68 if(c < UNICODE_BREAK_START)
69 return &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
70 else
71 return utf32__unidata_hard(c);
72}
73
74/** @brief Find definition of code point @p c
75 * @param c Code point
76 * @return Pointer to @ref unidata structure for @p c
77 *
78 * @p c can be any 32-bit value, a sensible value will be returned regardless.
79 * The returned pointer is NOT guaranteed to be unique to @p c.
80 *
81 * Don't use this function (although it will work fine) - use utf32__unidata()
82 * instead.
83 */
84static const struct unidata *utf32__unidata_hard(uint32_t c) {
85 if(c < UNICODE_BREAK_START)
86 return &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
87 /* Within the break everything is unassigned */
88 if(c < UNICODE_BREAK_END)
89 return utf32__unidata(0xFFFF); /* guaranteed to be Cn */
90 /* Planes 15 and 16 are (mostly) private use */
91 if((c >= 0xF0000 && c <= 0xFFFFD)
92 || (c >= 0x100000 && c <= 0x10FFFD))
93 return utf32__unidata(0xE000); /* first Co code point */
94 /* Everything else above the break top is unassigned */
95 if(c >= UNICODE_BREAK_TOP)
96 return utf32__unidata(0xFFFF); /* guaranteed to be Cn */
97 /* Currently the rest is language tags and variation selectors */
98 c -= (UNICODE_BREAK_END - UNICODE_BREAK_START);
99 return &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
100}
101
102/** @brief Return the combining class of @p c
103 * @param c Code point
104 * @return Combining class of @p c
105 *
106 * @p c can be any 32-bit value, a sensible value will be returned regardless.
107 */
108static inline int utf32__combining_class(uint32_t c) {
109 return utf32__unidata(c)->ccc;
110}
111
3c82b504
RK
112/** @brief Return the combining class of @p c
113 * @param c Code point
114 * @return Combining class of @p c
115 *
116 * @p c can be any 32-bit value, a sensible value will be returned regardless.
117 */
118int utf32_combining_class(uint32_t c) {
119 return utf32__combining_class(c);
120}
121
092f426f 122/** @brief Return the General_Category value for @p c
0ae60b83 123 * @param c Code point
092f426f
RK
124 * @return General_Category property value
125 *
126 * @p c can be any 32-bit value, a sensible value will be returned regardless.
127 */
128static inline enum unicode_General_Category utf32__general_category(uint32_t c) {
129 return utf32__unidata(c)->general_category;
130}
131
132/** @brief Determine Grapheme_Break property
133 * @param c Code point
134 * @return Grapheme_Break property value of @p c
135 *
136 * @p c can be any 32-bit value, a sensible value will be returned regardless.
137 */
138static inline enum unicode_Grapheme_Break utf32__grapheme_break(uint32_t c) {
139 return utf32__unidata(c)->grapheme_break;
140}
141
142/** @brief Determine Word_Break property
143 * @param c Code point
144 * @return Word_Break property value of @p c
145 *
146 * @p c can be any 32-bit value, a sensible value will be returned regardless.
147 */
148static inline enum unicode_Word_Break utf32__word_break(uint32_t c) {
149 return utf32__unidata(c)->word_break;
150}
151
152/** @brief Determine Sentence_Break property
153 * @param c Code point
154 * @return Word_Break property value of @p c
155 *
156 * @p c can be any 32-bit value, a sensible value will be returned regardless.
157 */
158static inline enum unicode_Sentence_Break utf32__sentence_break(uint32_t c) {
159 return utf32__unidata(c)->sentence_break;
160}
161
162/** @brief Return true if @p c is ignorable for boundary specifications
163 * @param wb Word break property value
164 * @return non-0 if @p wb is unicode_Word_Break_Extend or unicode_Word_Break_Format
165 */
166static inline int utf32__boundary_ignorable(enum unicode_Word_Break wb) {
167 return (wb == unicode_Word_Break_Extend
168 || wb == unicode_Word_Break_Format);
169}
170
f98fcddb
RK
171/** @brief Return the canonical decomposition of @p c
172 * @param c Code point
173 * @return 0-terminated canonical decomposition, or 0
174 */
175static inline const uint32_t *utf32__decomposition_canon(uint32_t c) {
176 const struct unidata *const data = utf32__unidata(c);
177 const uint32_t *const decomp = data->decomp;
178
179 if(decomp && !(data->flags & unicode_compatibility_decomposition))
180 return decomp;
181 else
182 return 0;
183}
184
185/** @brief Return the compatibility decomposition of @p c
186 * @param c Code point
187 * @return 0-terminated decomposition, or 0
188 */
189static inline const uint32_t *utf32__decomposition_compat(uint32_t c) {
190 return utf32__unidata(c)->decomp;
191}
192
092f426f 193/*@}*/
e5a5a138
RK
194/** @defgroup utftransform Functions that transform between different Unicode encoding forms */
195/*@{*/
196
197/** @brief Convert UTF-32 to UTF-8
198 * @param s Source string
199 * @param ns Length of source string in code points
200 * @param ndp Where to store length of destination string (or NULL)
201 * @return Newly allocated destination string or NULL on error
202 *
56fd389c
RK
203 * If the UTF-32 is not valid then NULL is returned. A UTF-32 code point is
204 * invalid if:
e5a5a138
RK
205 * - it codes for a UTF-16 surrogate
206 * - it codes for a value outside the unicode code space
207 *
56fd389c
RK
208 * The return value is always 0-terminated. The value returned via @p *ndp
209 * does not include the terminator.
e5a5a138
RK
210 */
211char *utf32_to_utf8(const uint32_t *s, size_t ns, size_t *ndp) {
212 struct dynstr d;
213 uint32_t c;
214
215 dynstr_init(&d);
216 while(ns > 0) {
217 c = *s++;
218 if(c < 0x80)
219 dynstr_append(&d, c);
220 else if(c < 0x0800) {
221 dynstr_append(&d, 0xC0 | (c >> 6));
222 dynstr_append(&d, 0x80 | (c & 0x3F));
223 } else if(c < 0x10000) {
56fd389c 224 if(c >= 0xD800 && c <= 0xDFFF)
e5a5a138
RK
225 goto error;
226 dynstr_append(&d, 0xE0 | (c >> 12));
227 dynstr_append(&d, 0x80 | ((c >> 6) & 0x3F));
228 dynstr_append(&d, 0x80 | (c & 0x3F));
229 } else if(c < 0x110000) {
230 dynstr_append(&d, 0xF0 | (c >> 18));
231 dynstr_append(&d, 0x80 | ((c >> 12) & 0x3F));
232 dynstr_append(&d, 0x80 | ((c >> 6) & 0x3F));
233 dynstr_append(&d, 0x80 | (c & 0x3F));
234 } else
235 goto error;
236 --ns;
237 }
238 dynstr_terminate(&d);
239 if(ndp)
240 *ndp = d.nvec;
241 return d.vec;
242error:
243 xfree(d.vec);
244 return 0;
245}
246
247/** @brief Convert UTF-8 to UTF-32
248 * @param s Source string
249 * @param ns Length of source string in code points
250 * @param ndp Where to store length of destination string (or NULL)
f98fcddb 251 * @return Newly allocated destination string or NULL on error
e5a5a138 252 *
56fd389c
RK
253 * The return value is always 0-terminated. The value returned via @p *ndp
254 * does not include the terminator.
e5a5a138
RK
255 *
256 * If the UTF-8 is not valid then NULL is returned. A UTF-8 sequence
257 * for a code point is invalid if:
258 * - it is not the shortest possible sequence for the code point
259 * - it codes for a UTF-16 surrogate
260 * - it codes for a value outside the unicode code space
261 */
262uint32_t *utf8_to_utf32(const char *s, size_t ns, size_t *ndp) {
263 struct dynstr_ucs4 d;
32b158f2 264 uint32_t c32;
e5a5a138 265 const uint8_t *ss = (const uint8_t *)s;
32b158f2 266 int n;
e5a5a138
RK
267
268 dynstr_ucs4_init(&d);
269 while(ns > 0) {
32b158f2
RK
270 const struct unicode_utf8_row *const r = &unicode_utf8_valid[*ss];
271 if(r->count <= ns) {
272 switch(r->count) {
273 case 1:
274 c32 = *ss;
275 break;
276 case 2:
277 if(ss[1] < r->min2 || ss[1] > r->max2)
278 goto error;
279 c32 = *ss & 0x1F;
280 break;
281 case 3:
282 if(ss[1] < r->min2 || ss[1] > r->max2)
283 goto error;
284 c32 = *ss & 0x0F;
285 break;
286 case 4:
287 if(ss[1] < r->min2 || ss[1] > r->max2)
288 goto error;
289 c32 = *ss & 0x07;
290 break;
291 default:
292 goto error;
293 }
e5a5a138
RK
294 } else
295 goto error;
32b158f2
RK
296 for(n = 1; n < r->count; ++n) {
297 if(ss[n] < 0x80 || ss[n] > 0xBF)
298 goto error;
299 c32 = (c32 << 6) | (ss[n] & 0x3F);
300 }
e5a5a138 301 dynstr_ucs4_append(&d, c32);
32b158f2
RK
302 ss += r->count;
303 ns -= r->count;
e5a5a138
RK
304 }
305 dynstr_ucs4_terminate(&d);
306 if(ndp)
307 *ndp = d.nvec;
308 return d.vec;
309error:
310 xfree(d.vec);
311 return 0;
312}
313
18cda350
RK
314/** @brief Test whether [s,s+ns) is valid UTF-8
315 * @param s Start of string
316 * @param ns Length of string
317 * @return non-0 if @p s is valid UTF-8, 0 if it is not valid
318 *
319 * This function is intended to be much faster than calling utf8_to_utf32() and
320 * throwing away the result.
321 */
322int utf8_valid(const char *s, size_t ns) {
323 const uint8_t *ss = (const uint8_t *)s;
324 while(ns > 0) {
325 const struct unicode_utf8_row *const r = &unicode_utf8_valid[*ss];
326 if(r->count <= ns) {
327 switch(r->count) {
328 case 1:
329 break;
330 case 2:
331 if(ss[1] < r->min2 || ss[1] > r->max2)
332 return 0;
333 break;
334 case 3:
335 if(ss[1] < r->min2 || ss[1] > r->max2)
336 return 0;
337 if(ss[2] < 0x80 || ss[2] > 0xBF)
338 return 0;
339 break;
340 case 4:
341 if(ss[1] < r->min2 || ss[1] > r->max2)
342 return 0;
343 if(ss[2] < 0x80 || ss[2] > 0xBF)
344 return 0;
345 if(ss[3] < 0x80 || ss[3] > 0xBF)
346 return 0;
347 break;
348 default:
349 return 0;
350 }
351 } else
352 return 0;
353 ss += r->count;
354 ns -= r->count;
355 }
356 return 1;
357}
358
e5a5a138 359/*@}*/
092f426f
RK
360/** @defgroup utf32iterator UTF-32 string iterators */
361/*@{*/
362
363struct utf32_iterator_data {
364 /** @brief Start of string */
365 const uint32_t *s;
366
367 /** @brief Length of string */
368 size_t ns;
369
370 /** @brief Current position */
371 size_t n;
372
373 /** @brief Last two non-ignorable characters or (uint32_t)-1
374 *
375 * last[1] is the non-Extend/Format character just before position @p n;
376 * last[0] is the one just before that.
377 *
378 * Exception 1: if there is no such non-Extend/Format character then an
379 * Extend/Format character is accepted instead.
380 *
381 * Exception 2: if there is no such character even taking that into account
382 * the value is (uint32_t)-1.
383 */
384 uint32_t last[2];
092f426f 385
c85b7022
RK
386 /** @brief Tailoring for Word_Break */
387 unicode_property_tailor *word_break;
388};
092f426f
RK
389
390/** @brief Initialize an internal private iterator
391 * @param it Iterator
392 * @param s Start of string
393 * @param ns Length of string
394 * @param n Absolute position
395 */
396static void utf32__iterator_init(utf32_iterator it,
397 const uint32_t *s, size_t ns, size_t n) {
398 it->s = s;
399 it->ns = ns;
400 it->n = 0;
401 it->last[0] = it->last[1] = -1;
c85b7022 402 it->word_break = 0;
b21a155c 403 utf32_iterator_set(it, n);
092f426f
RK
404}
405
c85b7022
RK
406/** @brief Create a new iterator pointing at the start of a string
407 * @param s Start of string
408 * @param ns Length of string
409 * @return New iterator
410 */
411utf32_iterator utf32_iterator_new(const uint32_t *s, size_t ns) {
412 utf32_iterator it = xmalloc(sizeof *it);
413 utf32__iterator_init(it, s, ns, 0);
414 return it;
415}
416
417/** @brief Tailor this iterator's interpretation of the Word_Break property.
418 * @param it Iterator
419 * @param pt Property tailor function or NULL
420 *
421 * After calling this the iterator will call @p pt to determine the Word_Break
422 * property of each code point. If it returns -1 the default value will be
423 * used otherwise the returned value will be used.
424 *
425 * @p pt can be NULL to revert to the default value of the property.
426 *
427 * It is safe to call this function at any time; the iterator's internal state
428 * will be reset to suit the new tailoring.
429 */
430void utf32_iterator_tailor_word_break(utf32_iterator it,
431 unicode_property_tailor *pt) {
432 it->word_break = pt;
433 utf32_iterator_set(it, it->n);
434}
435
436static inline enum unicode_Word_Break utf32__iterator_word_break(utf32_iterator it,
437 uint32_t c) {
438 if(!it->word_break)
439 return utf32__word_break(c);
440 else {
441 const int t = it->word_break(c);
442
443 if(t < 0)
444 return utf32__word_break(c);
445 else
446 return t;
447 }
448}
449
092f426f
RK
450/** @brief Destroy an iterator
451 * @param it Iterator
452 */
453void utf32_iterator_destroy(utf32_iterator it) {
454 xfree(it);
455}
456
457/** @brief Find the current position of an interator
458 * @param it Iterator
459 */
460size_t utf32_iterator_where(utf32_iterator it) {
461 return it->n;
462}
463
464/** @brief Set an iterator's absolute position
465 * @param it Iterator
466 * @param n Absolute position
467 * @return 0 on success, non-0 on error
468 *
469 * It is an error to position the iterator outside the string (but acceptable
470 * to point it at the hypothetical post-final character). If an invalid value
471 * of @p n is specified then the iterator is not changed.
f98fcddb
RK
472 *
473 * This function works by backing up and then advancing to reconstruct the
474 * iterator's internal state for position @p n. The worst case will be O(n)
475 * time complexity (with a worse constant factor that utf32_iterator_advance())
476 * but the typical case is essentially constant-time.
092f426f
RK
477 */
478int utf32_iterator_set(utf32_iterator it, size_t n) {
5617aaff
RK
479 /* We can't just jump to position @p n; the @p last[] values will be wrong.
480 * What we need is to jump a bit behind @p n and then advance forward,
481 * updating @p last[] along the way. How far back? We need to cross two
482 * non-ignorable code points as we advance forwards, so we'd better pass two
483 * such characters on the way back (if such are available).
484 */
b21a155c 485 size_t m;
5617aaff
RK
486
487 if(n > it->ns) /* range check */
092f426f 488 return -1;
b21a155c
RK
489 /* Walk backwards skipping ignorable code points */
490 m = n;
c85b7022
RK
491 while(m > 0
492 && (utf32__boundary_ignorable(utf32__iterator_word_break(it,
493 it->s[m-1]))))
b21a155c
RK
494 --m;
495 /* Either m=0 or s[m-1] is not ignorable */
496 if(m > 0) {
497 --m;
498 /* s[m] is our first non-ignorable code; look for a second in the same
499 way **/
c85b7022
RK
500 while(m > 0
501 && (utf32__boundary_ignorable(utf32__iterator_word_break(it,
502 it->s[m-1]))))
5617aaff 503 --m;
b21a155c
RK
504 /* Either m=0 or s[m-1] is not ignorable */
505 if(m > 0)
506 --m;
507 }
508 it->last[0] = it->last[1] = -1;
5617aaff
RK
509 it->n = m;
510 return utf32_iterator_advance(it, n - m);
092f426f
RK
511}
512
513/** @brief Advance an iterator
514 * @param it Iterator
515 * @param count Number of code points to advance by
516 * @return 0 on success, non-0 on error
517 *
518 * It is an error to advance an iterator beyond the hypothetical post-final
519 * character of the string. If an invalid value of @p n is specified then the
520 * iterator is not changed.
521 *
522 * This function has O(n) time complexity: it works by advancing naively
523 * forwards through the string.
524 */
525int utf32_iterator_advance(utf32_iterator it, size_t count) {
526 if(count <= it->ns - it->n) {
527 while(count > 0) {
528 const uint32_t c = it->s[it->n];
c85b7022 529 const enum unicode_Word_Break wb = utf32__iterator_word_break(it, c);
092f426f
RK
530 if(it->last[1] == (uint32_t)-1
531 || !utf32__boundary_ignorable(wb)) {
532 it->last[0] = it->last[1];
533 it->last[1] = c;
534 }
535 ++it->n;
536 --count;
537 }
538 return 0;
539 } else
540 return -1;
541}
542
543/** @brief Find the current code point
544 * @param it Iterator
545 * @return Current code point or 0
546 *
547 * If the iterator points at the hypothetical post-final character of the
548 * string then 0 is returned. NB that this doesn't mean that there aren't any
549 * 0 code points inside the string!
550 */
551uint32_t utf32_iterator_code(utf32_iterator it) {
552 if(it->n < it->ns)
553 return it->s[it->n];
554 else
555 return 0;
556}
557
558/** @brief Test for a grapheme boundary
559 * @param it Iterator
560 * @return Non-0 if pointing just after a grapheme boundary, otherwise 0
f98fcddb
RK
561 *
562 * This function identifies default grapheme cluster boundaries as described in
563 * UAX #29 s3. It returns non-0 if @p it points at the code point just after a
564 * grapheme cluster boundary (including the hypothetical code point just after
565 * the end of the string).
092f426f
RK
566 */
567int utf32_iterator_grapheme_boundary(utf32_iterator it) {
568 uint32_t before, after;
569 enum unicode_Grapheme_Break gbbefore, gbafter;
570 /* GB1 and GB2 */
571 if(it->n == 0 || it->n == it->ns)
572 return 1;
573 /* Now we know that s[n-1] and s[n] are safe to inspect */
574 /* GB3 */
575 before = it->s[it->n-1];
576 after = it->s[it->n];
577 if(before == 0x000D && after == 0x000A)
578 return 0;
579 gbbefore = utf32__grapheme_break(before);
580 gbafter = utf32__grapheme_break(after);
581 /* GB4 */
582 if(gbbefore == unicode_Grapheme_Break_Control
583 || before == 0x000D
584 || before == 0x000A)
585 return 1;
586 /* GB5 */
587 if(gbafter == unicode_Grapheme_Break_Control
588 || after == 0x000D
589 || after == 0x000A)
590 return 1;
591 /* GB6 */
592 if(gbbefore == unicode_Grapheme_Break_L
593 && (gbafter == unicode_Grapheme_Break_L
594 || gbafter == unicode_Grapheme_Break_V
595 || gbafter == unicode_Grapheme_Break_LV
596 || gbafter == unicode_Grapheme_Break_LVT))
597 return 0;
598 /* GB7 */
599 if((gbbefore == unicode_Grapheme_Break_LV
600 || gbbefore == unicode_Grapheme_Break_V)
601 && (gbafter == unicode_Grapheme_Break_V
602 || gbafter == unicode_Grapheme_Break_T))
603 return 0;
604 /* GB8 */
605 if((gbbefore == unicode_Grapheme_Break_LVT
606 || gbbefore == unicode_Grapheme_Break_T)
607 && gbafter == unicode_Grapheme_Break_T)
608 return 0;
609 /* GB9 */
610 if(gbafter == unicode_Grapheme_Break_Extend)
611 return 0;
612 /* GB10 */
613 return 1;
614
615}
616
617/** @brief Test for a word boundary
618 * @param it Iterator
619 * @return Non-0 if pointing just after a word boundary, otherwise 0
f98fcddb
RK
620 *
621 * This function identifies default word boundaries as described in UAX #29 s4.
622 * It returns non-0 if @p it points at the code point just after a word
623 * boundary (including the hypothetical code point just after the end of the
624 * string) and 0 otherwise.
092f426f
RK
625 */
626int utf32_iterator_word_boundary(utf32_iterator it) {
627 enum unicode_Word_Break twobefore, before, after, twoafter;
628 size_t nn;
629
630 /* WB1 and WB2 */
631 if(it->n == 0 || it->n == it->ns)
632 return 1;
633 /* WB3 */
634 if(it->s[it->n-1] == 0x000D && it->s[it->n] == 0x000A)
635 return 0;
636 /* WB4 */
637 /* (!Sep) x (Extend|Format) as in UAX #29 s6.2 */
638 if(utf32__sentence_break(it->s[it->n-1]) != unicode_Sentence_Break_Sep
c85b7022 639 && utf32__boundary_ignorable(utf32__iterator_word_break(it, it->s[it->n])))
092f426f
RK
640 return 0;
641 /* Gather the property values we'll need for the rest of the test taking the
642 * s6.2 changes into account */
643 /* First we look at the code points after the proposed boundary */
644 nn = it->n; /* <it->ns */
c85b7022 645 after = utf32__iterator_word_break(it, it->s[nn++]);
092f426f
RK
646 if(!utf32__boundary_ignorable(after)) {
647 /* X (Extend|Format)* -> X */
648 while(nn < it->ns
c85b7022
RK
649 && utf32__boundary_ignorable(utf32__iterator_word_break(it,
650 it->s[nn])))
092f426f
RK
651 ++nn;
652 }
653 /* It's possible now that nn=ns */
654 if(nn < it->ns)
c85b7022 655 twoafter = utf32__iterator_word_break(it, it->s[nn]);
092f426f
RK
656 else
657 twoafter = unicode_Word_Break_Other;
658
659 /* We've already recorded the non-ignorable code points before the proposed
660 * boundary */
c85b7022
RK
661 before = utf32__iterator_word_break(it, it->last[1]);
662 twobefore = utf32__iterator_word_break(it, it->last[0]);
092f426f
RK
663
664 /* WB5 */
665 if(before == unicode_Word_Break_ALetter
666 && after == unicode_Word_Break_ALetter)
667 return 0;
668 /* WB6 */
669 if(before == unicode_Word_Break_ALetter
670 && after == unicode_Word_Break_MidLetter
671 && twoafter == unicode_Word_Break_ALetter)
672 return 0;
673 /* WB7 */
674 if(twobefore == unicode_Word_Break_ALetter
675 && before == unicode_Word_Break_MidLetter
676 && after == unicode_Word_Break_ALetter)
677 return 0;
c85b7022 678 /* WB8 */
092f426f
RK
679 if(before == unicode_Word_Break_Numeric
680 && after == unicode_Word_Break_Numeric)
681 return 0;
682 /* WB9 */
683 if(before == unicode_Word_Break_ALetter
684 && after == unicode_Word_Break_Numeric)
685 return 0;
686 /* WB10 */
687 if(before == unicode_Word_Break_Numeric
688 && after == unicode_Word_Break_ALetter)
689 return 0;
690 /* WB11 */
691 if(twobefore == unicode_Word_Break_Numeric
692 && before == unicode_Word_Break_MidNum
693 && after == unicode_Word_Break_Numeric)
694 return 0;
695 /* WB12 */
696 if(before == unicode_Word_Break_Numeric
697 && after == unicode_Word_Break_MidNum
698 && twoafter == unicode_Word_Break_Numeric)
699 return 0;
700 /* WB13 */
701 if(before == unicode_Word_Break_Katakana
702 && after == unicode_Word_Break_Katakana)
703 return 0;
704 /* WB13a */
705 if((before == unicode_Word_Break_ALetter
706 || before == unicode_Word_Break_Numeric
707 || before == unicode_Word_Break_Katakana
708 || before == unicode_Word_Break_ExtendNumLet)
709 && after == unicode_Word_Break_ExtendNumLet)
710 return 0;
711 /* WB13b */
712 if(before == unicode_Word_Break_ExtendNumLet
713 && (after == unicode_Word_Break_ALetter
714 || after == unicode_Word_Break_Numeric
715 || after == unicode_Word_Break_Katakana))
716 return 0;
717 /* WB14 */
718 return 1;
719}
720
721/*@}*/
e5a5a138
RK
722/** @defgroup utf32 Functions that operate on UTF-32 strings */
723/*@{*/
724
725/** @brief Return the length of a 0-terminated UTF-32 string
726 * @param s Pointer to 0-terminated string
727 * @return Length of string in code points (excluding terminator)
728 *
56fd389c 729 * Unlike the conversion functions no validity checking is done on the string.
e5a5a138
RK
730 */
731size_t utf32_len(const uint32_t *s) {
732 const uint32_t *t = s;
733
734 while(*t)
735 ++t;
736 return (size_t)(t - s);
737}
738
e5a5a138
RK
739/** @brief Stably sort [s,s+ns) into descending order of combining class
740 * @param s Start of array
741 * @param ns Number of elements, must be at least 1
742 * @param buffer Buffer of at least @p ns elements
743 */
744static void utf32__sort_ccc(uint32_t *s, size_t ns, uint32_t *buffer) {
745 uint32_t *a, *b, *bp;
746 size_t na, nb;
747
748 switch(ns) {
749 case 1: /* 1-element array is always sorted */
750 return;
751 case 2: /* 2-element arrays are trivial to sort */
752 if(utf32__combining_class(s[0]) > utf32__combining_class(s[1])) {
753 uint32_t tmp = s[0];
754 s[0] = s[1];
755 s[1] = tmp;
756 }
757 return;
758 default:
759 /* Partition the array */
760 na = ns / 2;
761 nb = ns - na;
762 a = s;
763 b = s + na;
764 /* Sort the two halves of the array */
765 utf32__sort_ccc(a, na, buffer);
766 utf32__sort_ccc(b, nb, buffer);
767 /* Merge them back into one, via the buffer */
768 bp = buffer;
769 while(na > 0 && nb > 0) {
16506c9d 770 /* We want ascending order of combining class (hence <)
e5a5a138
RK
771 * and we want stability within combining classes (hence <=)
772 */
773 if(utf32__combining_class(*a) <= utf32__combining_class(*b)) {
774 *bp++ = *a++;
775 --na;
776 } else {
777 *bp++ = *b++;
778 --nb;
779 }
780 }
781 while(na > 0) {
782 *bp++ = *a++;
783 --na;
784 }
785 while(nb > 0) {
786 *bp++ = *b++;
787 --nb;
788 }
789 memcpy(s, buffer, ns * sizeof(uint32_t));
790 return;
791 }
792}
793
794/** @brief Put combining characters into canonical order
795 * @param s Pointer to UTF-32 string
796 * @param ns Length of @p s
f98fcddb 797 * @return 0 on success, non-0 on error
e5a5a138 798 *
56fd389c
RK
799 * @p s is modified in-place. See Unicode 5.0 s3.11 for details of the
800 * ordering.
e5a5a138 801 *
56fd389c 802 * Currently we only support a maximum of 1024 combining characters after each
f98fcddb 803 * base character. If this limit is exceeded then a non-0 value is returned.
e5a5a138
RK
804 */
805static int utf32__canonical_ordering(uint32_t *s, size_t ns) {
806 size_t nc;
807 uint32_t buffer[1024];
808
809 /* The ordering amounts to a stable sort of each contiguous group of
810 * characters with non-0 combining class. */
811 while(ns > 0) {
812 /* Skip non-combining characters */
813 if(utf32__combining_class(*s) == 0) {
814 ++s;
815 --ns;
816 continue;
817 }
818 /* We must now have at least one combining character; see how many
819 * there are */
820 for(nc = 1; nc < ns && utf32__combining_class(s[nc]) != 0; ++nc)
821 ;
822 if(nc > 1024)
823 return -1;
824 /* Sort the array */
825 utf32__sort_ccc(s, nc, buffer);
826 s += nc;
827 ns -= nc;
828 }
829 return 0;
830}
831
832/* Magic numbers from UAX #15 s16 */
833#define SBase 0xAC00
834#define LBase 0x1100
835#define VBase 0x1161
836#define TBase 0x11A7
837#define LCount 19
838#define VCount 21
839#define TCount 28
840#define NCount (VCount * TCount)
841#define SCount (LCount * NCount)
842
843/** @brief Guts of the decomposition lookup functions */
844#define utf32__decompose_one_generic(WHICH) do { \
f98fcddb 845 const uint32_t *dc = utf32__decomposition_##WHICH(c); \
e5a5a138
RK
846 if(dc) { \
847 /* Found a canonical decomposition in the table */ \
848 while(*dc) \
849 utf32__decompose_one_##WHICH(d, *dc++); \
850 } else if(c >= SBase && c < SBase + SCount) { \
851 /* Mechanically decomposable Hangul syllable (UAX #15 s16) */ \
852 const uint32_t SIndex = c - SBase; \
853 const uint32_t L = LBase + SIndex / NCount; \
854 const uint32_t V = VBase + (SIndex % NCount) / TCount; \
855 const uint32_t T = TBase + SIndex % TCount; \
856 dynstr_ucs4_append(d, L); \
857 dynstr_ucs4_append(d, V); \
858 if(T != TBase) \
859 dynstr_ucs4_append(d, T); \
860 } else \
861 /* Equal to own canonical decomposition */ \
862 dynstr_ucs4_append(d, c); \
863} while(0)
864
865/** @brief Recursively compute the canonical decomposition of @p c
866 * @param d Dynamic string to store decomposition in
867 * @param c Code point to decompose (must be a valid!)
f98fcddb 868 * @return 0 on success, non-0 on error
e5a5a138
RK
869 */
870static void utf32__decompose_one_canon(struct dynstr_ucs4 *d, uint32_t c) {
871 utf32__decompose_one_generic(canon);
872}
873
874/** @brief Recursively compute the compatibility decomposition of @p c
875 * @param d Dynamic string to store decomposition in
876 * @param c Code point to decompose (must be a valid!)
f98fcddb 877 * @return 0 on success, non-0 on error
e5a5a138
RK
878 */
879static void utf32__decompose_one_compat(struct dynstr_ucs4 *d, uint32_t c) {
880 utf32__decompose_one_generic(compat);
881}
882
16506c9d
RK
883/** @brief Magic utf32__compositions() return value for Hangul Choseong */
884static const uint32_t utf32__hangul_L[1];
885
886/** @brief Return the list of compositions that @p c starts
887 * @param c Starter code point
888 * @return Composition list or NULL
889 *
890 * For Hangul leading (Choseong) jamo we return the special value
891 * utf32__hangul_L. These code points are not listed as the targets of
892 * canonical decompositions (make-unidata checks) so there is no confusion with
893 * real decompositions here.
894 */
895static const uint32_t *utf32__compositions(uint32_t c) {
896 const uint32_t *compositions = utf32__unidata(c)->composed;
897
898 if(compositions)
899 return compositions;
900 /* Special-casing for Hangul */
901 switch(utf32__grapheme_break(c)) {
902 default:
903 return 0;
904 case unicode_Grapheme_Break_L:
905 return utf32__hangul_L;
906 }
907}
908
909/** @brief Composition step
910 * @param s Start of string
911 * @param ns Length of string
912 * @return New length of string
913 *
914 * This is called from utf32__decompose_generic() to compose the result string
915 * in place.
916 */
917static size_t utf32__compose(uint32_t *s, size_t ns) {
918 const uint32_t *compositions;
919 uint32_t *start = s, *t = s, *tt, cc;
920
921 while(ns > 0) {
922 uint32_t starter = *s++;
923 int block_starters = 0;
924 --ns;
925 /* We don't attempt to compose the following things:
926 * - final characters whatever kind they are
927 * - non-starter characters
928 * - starters that don't take part in a canonical decomposition mapping
929 */
930 if(ns == 0
931 || utf32__combining_class(starter)
932 || !(compositions = utf32__compositions(starter))) {
933 *t++ = starter;
934 continue;
935 }
936 if(compositions != utf32__hangul_L) {
937 /* Where we'll put the eventual starter */
938 tt = t++;
939 do {
940 /* See if we can find composition of starter+*s */
941 const uint32_t cchar = *s, *cp = compositions;
942 while((cc = *cp++)) {
943 const uint32_t *decomp = utf32__decomposition_canon(cc);
944 /* We know decomp[0] == starter */
945 if(decomp[1] == cchar)
946 break;
947 }
948 if(cc) {
949 /* Found a composition: cc decomposes to starter,*s */
950 starter = cc;
951 compositions = utf32__compositions(starter);
952 ++s;
953 --ns;
954 } else {
955 /* No composition found. */
956 const int class = utf32__combining_class(*s);
957 if(class) {
958 /* Transfer the uncomposable combining character to the output */
959 *t++ = *s++;
960 --ns;
961 /* All the combining characters of the same class of the
962 * uncomposable character are blocked by it, but there may be
963 * others of higher class later. We eat the uncomposable and
964 * blocked characters and go back round the loop for that higher
965 * class. */
966 while(ns > 0 && utf32__combining_class(*s) == class) {
967 *t++ = *s++;
968 --ns;
969 }
970 /* Block any subsequent starters */
971 block_starters = 1;
972 } else {
973 /* The uncombinable character is itself a starter, so we don't
974 * transfer it to the output but instead go back round the main
975 * loop. */
976 break;
977 }
978 }
979 /* Keep going while there are still characters and the starter takes
980 * part in some composition */
981 } while(ns > 0 && compositions
982 && (!block_starters || utf32__combining_class(*s)));
983 /* Store any remaining combining characters */
984 while(ns > 0 && utf32__combining_class(*s)) {
985 *t++ = *s++;
986 --ns;
987 }
988 /* Store the resulting starter */
989 *tt = starter;
990 } else {
991 /* Special-casing for Hangul
992 *
993 * If there are combining characters between the L and the V then they
994 * will block the V and so no composition happens. Similarly combining
995 * characters between V and T will block the T and so we only get as far
996 * as LV.
997 */
998 if(utf32__grapheme_break(*s) == unicode_Grapheme_Break_V) {
999 const uint32_t V = *s++;
1000 const uint32_t LIndex = starter - LBase;
1001 const uint32_t VIndex = V - VBase;
1002 uint32_t TIndex;
1003 --ns;
1004 if(ns > 0
1005 && utf32__grapheme_break(*s) == unicode_Grapheme_Break_T) {
1006 /* We have an L V T sequence */
1007 const uint32_t T = *s++;
1008 TIndex = T - TBase;
1009 --ns;
1010 } else
1011 /* It's just L V */
1012 TIndex = 0;
1013 /* Compose to LVT or LV as appropriate */
1014 starter = (LIndex * VCount + VIndex) * TCount + TIndex + SBase;
1015 } /* else we only have L or LV and no V or T */
1016 *t++ = starter;
1017 /* There could be some combining characters that belong to the V or T.
1018 * These will be treated as non-starter characters at the top of the loop
1019 * and thuss transferred to the output. */
1020 }
1021 }
1022 return t - start;
1023}
1024
1025/** @brief Guts of the composition and decomposition functions
1026 * @param WHICH @c canon or @c compat to choose decomposition
1027 * @param COMPOSE @c 0 or @c 1 to compose
1028 */
1029#define utf32__decompose_generic(WHICH, COMPOSE) do { \
e5a5a138
RK
1030 struct dynstr_ucs4 d; \
1031 uint32_t c; \
1032 \
1033 dynstr_ucs4_init(&d); \
1034 while(ns) { \
1035 c = *s++; \
56fd389c 1036 if((c >= 0xD800 && c <= 0xDFFF) || c > 0x10FFFF) \
e5a5a138
RK
1037 goto error; \
1038 utf32__decompose_one_##WHICH(&d, c); \
1039 --ns; \
1040 } \
1041 if(utf32__canonical_ordering(d.vec, d.nvec)) \
1042 goto error; \
16506c9d
RK
1043 if(COMPOSE) \
1044 d.nvec = utf32__compose(d.vec, d.nvec); \
e5a5a138
RK
1045 dynstr_ucs4_terminate(&d); \
1046 if(ndp) \
1047 *ndp = d.nvec; \
1048 return d.vec; \
1049error: \
1050 xfree(d.vec); \
1051 return 0; \
1052} while(0)
1053
1054/** @brief Canonically decompose @p [s,s+ns)
1055 * @param s Pointer to string
1056 * @param ns Length of string
1057 * @param ndp Where to store length of result
f98fcddb 1058 * @return Pointer to result string, or NULL on error
e5a5a138 1059 *
16506c9d
RK
1060 * Computes NFD (Normalization Form D) of the string at @p s. This implies
1061 * performing all canonical decompositions and then normalizing the order of
1062 * combining characters.
e5a5a138 1063 *
56fd389c 1064 * Returns NULL if the string is not valid for either of the following reasons:
e5a5a138
RK
1065 * - it codes for a UTF-16 surrogate
1066 * - it codes for a value outside the unicode code space
16506c9d
RK
1067 *
1068 * See also:
1069 * - utf32_decompose_compat()
1070 * - utf32_compose_canon()
e5a5a138
RK
1071 */
1072uint32_t *utf32_decompose_canon(const uint32_t *s, size_t ns, size_t *ndp) {
16506c9d 1073 utf32__decompose_generic(canon, 0);
e5a5a138
RK
1074}
1075
1076/** @brief Compatibility decompose @p [s,s+ns)
1077 * @param s Pointer to string
1078 * @param ns Length of string
1079 * @param ndp Where to store length of result
f98fcddb 1080 * @return Pointer to result string, or NULL on error
e5a5a138 1081 *
16506c9d
RK
1082 * Computes NFKD (Normalization Form KD) of the string at @p s. This implies
1083 * performing all canonical and compatibility decompositions and then
1084 * normalizing the order of combining characters.
e5a5a138 1085 *
56fd389c 1086 * Returns NULL if the string is not valid for either of the following reasons:
e5a5a138
RK
1087 * - it codes for a UTF-16 surrogate
1088 * - it codes for a value outside the unicode code space
16506c9d
RK
1089 *
1090 * See also:
1091 * - utf32_decompose_canon()
1092 * - utf32_compose_compat()
e5a5a138
RK
1093 */
1094uint32_t *utf32_decompose_compat(const uint32_t *s, size_t ns, size_t *ndp) {
16506c9d
RK
1095 utf32__decompose_generic(compat, 0);
1096}
1097
1098/** @brief Canonically compose @p [s,s+ns)
1099 * @param s Pointer to string
1100 * @param ns Length of string
1101 * @param ndp Where to store length of result
1102 * @return Pointer to result string, or NULL on error
1103 *
1104 * Computes NFC (Normalization Form C) of the string at @p s. This implies
1105 * performing all canonical decompositions, normalizing the order of combining
1106 * characters and then composing all unblocked primary compositables.
1107 *
1108 * Returns NULL if the string is not valid for either of the following reasons:
1109 * - it codes for a UTF-16 surrogate
1110 * - it codes for a value outside the unicode code space
1111 *
1112 * See also:
1113 * - utf32_compose_compat()
1114 * - utf32_decompose_canon()
1115 */
1116uint32_t *utf32_compose_canon(const uint32_t *s, size_t ns, size_t *ndp) {
1117 utf32__decompose_generic(canon, 1);
1118}
1119
1120/** @brief Compatibility compose @p [s,s+ns)
1121 * @param s Pointer to string
1122 * @param ns Length of string
1123 * @param ndp Where to store length of result
1124 * @return Pointer to result string, or NULL on error
1125 *
1126 * Computes NFKC (Normalization Form KC) of the string at @p s. This implies
1127 * performing all canonical and compatibility decompositions, normalizing the
1128 * order of combining characters and then composing all unblocked primary
1129 * compositables.
1130 *
1131 * Returns NULL if the string is not valid for either of the following reasons:
1132 * - it codes for a UTF-16 surrogate
1133 * - it codes for a value outside the unicode code space
1134 *
1135 * See also:
1136 * - utf32_compose_canon()
1137 * - utf32_decompose_compat()
1138 */
1139uint32_t *utf32_compose_compat(const uint32_t *s, size_t ns, size_t *ndp) {
1140 utf32__decompose_generic(compat, 1);
e5a5a138
RK
1141}
1142
56fd389c
RK
1143/** @brief Single-character case-fold and decompose operation */
1144#define utf32__casefold_one(WHICH) do { \
bcf9ed7f 1145 const uint32_t *cf = utf32__unidata(c)->casefold; \
56fd389c
RK
1146 if(cf) { \
1147 /* Found a case-fold mapping in the table */ \
1148 while(*cf) \
1149 utf32__decompose_one_##WHICH(&d, *cf++); \
1150 } else \
1151 utf32__decompose_one_##WHICH(&d, c); \
1152} while(0)
e5a5a138
RK
1153
1154/** @brief Case-fold @p [s,s+ns)
1155 * @param s Pointer to string
1156 * @param ns Length of string
1157 * @param ndp Where to store length of result
f98fcddb 1158 * @return Pointer to result string, or NULL on error
e5a5a138
RK
1159 *
1160 * Case-fold the string at @p s according to full default case-folding rules
56fd389c 1161 * (s3.13) for caseless matching. The result will be in NFD.
e5a5a138 1162 *
56fd389c 1163 * Returns NULL if the string is not valid for either of the following reasons:
e5a5a138
RK
1164 * - it codes for a UTF-16 surrogate
1165 * - it codes for a value outside the unicode code space
1166 */
1167uint32_t *utf32_casefold_canon(const uint32_t *s, size_t ns, size_t *ndp) {
1168 struct dynstr_ucs4 d;
1169 uint32_t c;
1170 size_t n;
1171 uint32_t *ss = 0;
1172
1173 /* If the canonical decomposition of the string includes any combining
1174 * character that case-folds to a non-combining character then we must
1175 * normalize before we fold. In Unicode 5.0.0 this means 0345 COMBINING
1176 * GREEK YPOGEGRAMMENI in its decomposition and the various characters that
1177 * canonically decompose to it. */
bcf9ed7f
RK
1178 for(n = 0; n < ns; ++n)
1179 if(utf32__unidata(s[n])->flags & unicode_normalize_before_casefold)
e5a5a138 1180 break;
e5a5a138
RK
1181 if(n < ns) {
1182 /* We need a preliminary decomposition */
1183 if(!(ss = utf32_decompose_canon(s, ns, &ns)))
1184 return 0;
1185 s = ss;
1186 }
1187 dynstr_ucs4_init(&d);
1188 while(ns) {
1189 c = *s++;
56fd389c 1190 if((c >= 0xD800 && c <= 0xDFFF) || c > 0x10FFFF)
e5a5a138 1191 goto error;
56fd389c 1192 utf32__casefold_one(canon);
e5a5a138
RK
1193 --ns;
1194 }
1195 if(utf32__canonical_ordering(d.vec, d.nvec))
1196 goto error;
1197 dynstr_ucs4_terminate(&d);
1198 if(ndp)
1199 *ndp = d.nvec;
1200 return d.vec;
1201error:
1202 xfree(d.vec);
1203 xfree(ss);
1204 return 0;
1205}
1206
f98fcddb 1207/** @brief Compatibility case-fold @p [s,s+ns)
56fd389c
RK
1208 * @param s Pointer to string
1209 * @param ns Length of string
1210 * @param ndp Where to store length of result
f98fcddb 1211 * @return Pointer to result string, or NULL on error
56fd389c
RK
1212 *
1213 * Case-fold the string at @p s according to full default case-folding rules
1214 * (s3.13) for compatibility caseless matching. The result will be in NFKD.
1215 *
1216 * Returns NULL if the string is not valid for either of the following reasons:
1217 * - it codes for a UTF-16 surrogate
1218 * - it codes for a value outside the unicode code space
1219 */
1220uint32_t *utf32_casefold_compat(const uint32_t *s, size_t ns, size_t *ndp) {
1221 struct dynstr_ucs4 d;
1222 uint32_t c;
1223 size_t n;
1224 uint32_t *ss = 0;
1225
bcf9ed7f
RK
1226 for(n = 0; n < ns; ++n)
1227 if(utf32__unidata(s[n])->flags & unicode_normalize_before_casefold)
56fd389c 1228 break;
56fd389c
RK
1229 if(n < ns) {
1230 /* We need a preliminary _canonical_ decomposition */
1231 if(!(ss = utf32_decompose_canon(s, ns, &ns)))
1232 return 0;
1233 s = ss;
1234 }
1235 /* This computes NFKD(toCaseFold(s)) */
1236#define compat_casefold_middle() do { \
1237 dynstr_ucs4_init(&d); \
1238 while(ns) { \
1239 c = *s++; \
1240 if((c >= 0xD800 && c <= 0xDFFF) || c > 0x10FFFF) \
1241 goto error; \
1242 utf32__casefold_one(compat); \
1243 --ns; \
1244 } \
1245 if(utf32__canonical_ordering(d.vec, d.nvec)) \
1246 goto error; \
1247} while(0)
1248 /* Do the inner (NFKD o toCaseFold) */
1249 compat_casefold_middle();
1250 /* We can do away with the NFD'd copy of the input now */
1251 xfree(ss);
1252 s = ss = d.vec;
1253 ns = d.nvec;
1254 /* Do the outer (NFKD o toCaseFold) */
1255 compat_casefold_middle();
1256 /* That's all */
1257 dynstr_ucs4_terminate(&d);
1258 if(ndp)
1259 *ndp = d.nvec;
1260 return d.vec;
1261error:
1262 xfree(d.vec);
1263 xfree(ss);
1264 return 0;
1265}
1266
e5a5a138
RK
1267/** @brief Order a pair of UTF-32 strings
1268 * @param a First 0-terminated string
1269 * @param b Second 0-terminated string
1270 * @return -1, 0 or 1 for a less than, equal to or greater than b
1271 *
1272 * "Comparable to strcmp() at its best."
1273 */
1274int utf32_cmp(const uint32_t *a, const uint32_t *b) {
1275 while(*a && *b && *a == *b) {
1276 ++a;
1277 ++b;
1278 }
1279 return *a < *b ? -1 : (*a > *b ? 1 : 0);
1280}
1281
35b651f0
RK
1282/** @brief Identify a grapheme cluster boundary
1283 * @param s Start of string (must be NFD)
1284 * @param ns Length of string
1285 * @param n Index within string (in [0,ns].)
1286 * @return 1 at a grapheme cluster boundary, 0 otherwise
1287 *
1288 * This function identifies default grapheme cluster boundaries as described in
f98fcddb 1289 * UAX #29 s3. It returns non-0 if @p n points at the code point just after a
35b651f0
RK
1290 * grapheme cluster boundary (including the hypothetical code point just after
1291 * the end of the string).
f98fcddb
RK
1292 *
1293 * This function uses utf32_iterator_set() internally; see that function for
1294 * remarks on performance.
35b651f0 1295 */
1625e11a 1296int utf32_is_grapheme_boundary(const uint32_t *s, size_t ns, size_t n) {
092f426f 1297 struct utf32_iterator_data it[1];
35b651f0 1298
092f426f
RK
1299 utf32__iterator_init(it, s, ns, n);
1300 return utf32_iterator_grapheme_boundary(it);
0b7052da
RK
1301}
1302
1303/** @brief Identify a word boundary
1304 * @param s Start of string (must be NFD)
1305 * @param ns Length of string
1306 * @param n Index within string (in [0,ns].)
1307 * @return 1 at a word boundary, 0 otherwise
1308 *
1309 * This function identifies default word boundaries as described in UAX #29 s4.
f98fcddb 1310 * It returns non-0 if @p n points at the code point just after a word boundary
0b7052da 1311 * (including the hypothetical code point just after the end of the string).
f98fcddb
RK
1312 *
1313 * This function uses utf32_iterator_set() internally; see that function for
1314 * remarks on performance.
0b7052da
RK
1315 */
1316int utf32_is_word_boundary(const uint32_t *s, size_t ns, size_t n) {
092f426f 1317 struct utf32_iterator_data it[1];
0b7052da 1318
092f426f
RK
1319 utf32__iterator_init(it, s, ns, n);
1320 return utf32_iterator_word_boundary(it);
0b7052da
RK
1321}
1322
8818b7fc
RK
1323/** @brief Split [s,ns) into multiple words
1324 * @param s Pointer to start of string
1325 * @param ns Length of string
1326 * @param nwp Where to store word count, or NULL
c85b7022 1327 * @param wbreak Word_Break property tailor, or NULL
8818b7fc
RK
1328 * @return Pointer to array of pointers to words
1329 *
1330 * The returned array is terminated by a NULL pointer and individual
1331 * strings are 0-terminated.
1332 */
c85b7022
RK
1333uint32_t **utf32_word_split(const uint32_t *s, size_t ns, size_t *nwp,
1334 unicode_property_tailor *wbreak) {
8818b7fc
RK
1335 struct utf32_iterator_data it[1];
1336 size_t b1 = 0, b2 = 0 ,i;
1337 int isword;
1338 struct vector32 v32[1];
1339 uint32_t *w;
1340
1341 vector32_init(v32);
1342 utf32__iterator_init(it, s, ns, 0);
c85b7022 1343 it->word_break = wbreak;
8818b7fc
RK
1344 /* Work our way through the string stopping at each word break. */
1345 do {
1346 if(utf32_iterator_word_boundary(it)) {
1347 /* We've found a new boundary */
1348 b1 = b2;
1349 b2 = it->n;
1350 /*fprintf(stderr, "[%zu, %zu) is a candidate word\n", b1, b2);*/
1351 /* Inspect the characters between the boundary and form an opinion as to
1352 * whether they are a word or not */
1353 isword = 0;
1354 for(i = b1; i < b2; ++i) {
c85b7022 1355 switch(utf32__iterator_word_break(it, it->s[i])) {
8818b7fc
RK
1356 case unicode_Word_Break_ALetter:
1357 case unicode_Word_Break_Numeric:
1358 case unicode_Word_Break_Katakana:
1359 isword = 1;
1360 break;
1361 default:
1362 break;
1363 }
1364 }
1365 /* If it's a word add it to the list of results */
1366 if(isword) {
1367 w = xcalloc(b2 - b1 + 1, sizeof(uint32_t));
1368 memcpy(w, it->s + b1, (b2 - b1) * sizeof (uint32_t));
1369 vector32_append(v32, w);
1370 }
1371 }
1372 } while(!utf32_iterator_advance(it, 1));
1373 vector32_terminate(v32);
1374 if(nwp)
1375 *nwp = v32->nvec;
1376 return v32->vec;
1377}
1378
e5a5a138 1379/*@}*/
349b7b74 1380/** @defgroup utf8 Functions that operate on UTF-8 strings */
e5a5a138
RK
1381/*@{*/
1382
1383/** @brief Wrapper to transform a UTF-8 string using the UTF-32 function */
1384#define utf8__transform(FN) do { \
1385 uint32_t *to32 = 0, *decomp32 = 0; \
1386 size_t nto32, ndecomp32; \
1387 char *decomp8 = 0; \
1388 \
1389 if(!(to32 = utf8_to_utf32(s, ns, &nto32))) goto error; \
1390 if(!(decomp32 = FN(to32, nto32, &ndecomp32))) goto error; \
1391 decomp8 = utf32_to_utf8(decomp32, ndecomp32, ndp); \
1392error: \
1393 xfree(to32); \
1394 xfree(decomp32); \
1395 return decomp8; \
1396} while(0)
1397
1398/** @brief Canonically decompose @p [s,s+ns)
1399 * @param s Pointer to string
1400 * @param ns Length of string
1401 * @param ndp Where to store length of result
f98fcddb 1402 * @return Pointer to result string, or NULL on error
e5a5a138 1403 *
0ae60b83
RK
1404 * Computes NFD (Normalization Form D) of the string at @p s. This implies
1405 * performing all canonical decompositions and then normalizing the order of
1406 * combining characters.
e5a5a138
RK
1407 *
1408 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1409 * this might be.
1410 *
0ae60b83
RK
1411 * See also:
1412 * - utf32_decompose_canon().
1413 * - utf8_decompose_compat()
1414 * - utf8_compose_canon()
e5a5a138
RK
1415 */
1416char *utf8_decompose_canon(const char *s, size_t ns, size_t *ndp) {
1417 utf8__transform(utf32_decompose_canon);
1418}
1419
1420/** @brief Compatibility decompose @p [s,s+ns)
1421 * @param s Pointer to string
1422 * @param ns Length of string
1423 * @param ndp Where to store length of result
f98fcddb 1424 * @return Pointer to result string, or NULL on error
e5a5a138 1425 *
0ae60b83
RK
1426 * Computes NFKD (Normalization Form KD) of the string at @p s. This implies
1427 * performing all canonical and compatibility decompositions and then
1428 * normalizing the order of combining characters.
e5a5a138
RK
1429 *
1430 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1431 * this might be.
1432 *
0ae60b83
RK
1433 * See also:
1434 * - utf32_decompose_compat().
1435 * - utf8_decompose_canon()
1436 * - utf8_compose_compat()
e5a5a138
RK
1437 */
1438char *utf8_decompose_compat(const char *s, size_t ns, size_t *ndp) {
1439 utf8__transform(utf32_decompose_compat);
1440}
1441
0ae60b83
RK
1442/** @brief Canonically compose @p [s,s+ns)
1443 * @param s Pointer to string
1444 * @param ns Length of string
1445 * @param ndp Where to store length of result
1446 * @return Pointer to result string, or NULL on error
1447 *
1448 * Computes NFC (Normalization Form C) of the string at @p s. This implies
1449 * performing all canonical decompositions, normalizing the order of combining
1450 * characters and then composing all unblocked primary compositables.
1451 *
1452 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1453 * this might be.
1454 *
1455 * See also:
1456 * - utf32_compose_canon()
1457 * - utf8_compose_compat()
1458 * - utf8_decompose_canon()
1459 */
1460char *utf8_compose_canon(const char *s, size_t ns, size_t *ndp) {
1461 utf8__transform(utf32_compose_canon);
1462}
1463
1464/** @brief Compatibility compose @p [s,s+ns)
1465 * @param s Pointer to string
1466 * @param ns Length of string
1467 * @param ndp Where to store length of result
1468 * @return Pointer to result string, or NULL on error
1469 *
1470 * Computes NFKC (Normalization Form KC) of the string at @p s. This implies
1471 * performing all canonical and compatibility decompositions, normalizing the
1472 * order of combining characters and then composing all unblocked primary
1473 * compositables.
1474 *
1475 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1476 * this might be.
1477 *
1478 * See also:
1479 * - utf32_compose_compat()
1480 * - utf8_compose_canon()
1481 * - utf8_decompose_compat()
1482 */
1483char *utf8_compose_compat(const char *s, size_t ns, size_t *ndp) {
1484 utf8__transform(utf32_compose_compat);
1485}
1486
e5a5a138
RK
1487/** @brief Case-fold @p [s,s+ns)
1488 * @param s Pointer to string
1489 * @param ns Length of string
1490 * @param ndp Where to store length of result
f98fcddb 1491 * @return Pointer to result string, or NULL on error
e5a5a138
RK
1492 *
1493 * Case-fold the string at @p s according to full default case-folding rules
1494 * (s3.13). The result will be in NFD.
1495 *
1496 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1497 * this might be.
1498 */
1499char *utf8_casefold_canon(const char *s, size_t ns, size_t *ndp) {
1500 utf8__transform(utf32_casefold_canon);
1501}
1502
1503/** @brief Compatibility case-fold @p [s,s+ns)
1504 * @param s Pointer to string
1505 * @param ns Length of string
1506 * @param ndp Where to store length of result
f98fcddb 1507 * @return Pointer to result string, or NULL on error
e5a5a138
RK
1508 *
1509 * Case-fold the string at @p s according to full default case-folding rules
1510 * (s3.13). The result will be in NFKD.
1511 *
1512 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1513 * this might be.
1514 */
e5a5a138
RK
1515char *utf8_casefold_compat(const char *s, size_t ns, size_t *ndp) {
1516 utf8__transform(utf32_casefold_compat);
1517}
e5a5a138 1518
8818b7fc
RK
1519/** @brief Split [s,ns) into multiple words
1520 * @param s Pointer to start of string
1521 * @param ns Length of string
1522 * @param nwp Where to store word count, or NULL
c85b7022 1523 * @param wbreak Word_Break property tailor, or NULL
8818b7fc
RK
1524 * @return Pointer to array of pointers to words
1525 *
1526 * The returned array is terminated by a NULL pointer and individual
1527 * strings are 0-terminated.
1528 */
c85b7022
RK
1529char **utf8_word_split(const char *s, size_t ns, size_t *nwp,
1530 unicode_property_tailor *wbreak) {
8818b7fc
RK
1531 uint32_t *to32 = 0, **v32 = 0;
1532 size_t nto32, nv, n;
1533 char **v8 = 0, **ret = 0;
c85b7022 1534
8818b7fc 1535 if(!(to32 = utf8_to_utf32(s, ns, &nto32))) goto error;
c85b7022 1536 if(!(v32 = utf32_word_split(to32, nto32, &nv, wbreak))) goto error;
8818b7fc
RK
1537 v8 = xcalloc(sizeof (char *), nv + 1);
1538 for(n = 0; n < nv; ++n)
1539 if(!(v8[n] = utf32_to_utf8(v32[n], utf32_len(v32[n]), 0)))
1540 goto error;
1541 ret = v8;
1542 *nwp = nv;
1543 v8 = 0; /* don't free */
c85b7022 1544error:
8818b7fc
RK
1545 if(v8) {
1546 for(n = 0; n < nv; ++n)
1547 xfree(v8[n]);
1548 xfree(v8);
1549 }
1550 if(v32) {
1551 for(n = 0; n < nv; ++n)
1552 xfree(v32[n]);
1553 xfree(v32);
1554 }
1555 xfree(to32);
1556 return ret;
1557}
1558
1559
e5a5a138
RK
1560/*@}*/
1561
1562/*
1563Local Variables:
1564c-basic-offset:2
1565comment-column:40
1566fill-column:79
1567indent-tabs-mode:nil
1568End:
1569*/