/*
* This file is part of DisOrder
- * Copyright (C) 2007 Richard Kettlewell
+ * Copyright (C) 2007, 2009, 2013 Richard Kettlewell
*
- * This program is free software; you can redistribute it and/or modify
+ * This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
+ * the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- *
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
* You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- * USA
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/** @file lib/unicode.c
* @brief Unicode support functions
* - @ref utf8
*/
-#include <config.h>
-#include "types.h"
-
-#include <string.h>
-#include <stdio.h> /* TODO */
+#include "common.h"
#include "mem.h"
#include "vector.h"
return utf32__unidata(c)->ccc;
}
+/** @brief Return the combining class of @p c
+ * @param c Code point
+ * @return Combining class of @p c
+ *
+ * @p c can be any 32-bit value, a sensible value will be returned regardless.
+ */
+int utf32_combining_class(uint32_t c) {
+ return utf32__combining_class(c);
+}
+
/** @brief Return the General_Category value for @p c
* @param c Code point
* @return General_Category property value
return 0;
}
+/** @brief Convert UTF-16 to UTF-8
+ * @param s Source string
+ * @param ns Length of source string in code points
+ * @param ndp Where to store length of destination string (or NULL)
+ * @return Newly allocated destination string or NULL on error
+ *
+ * If the UTF-16 is not valid then NULL is returned. A UTF-16 sequence t is
+ * invalid if it contains an incomplete surrogate.
+ *
+ * The return value is always 0-terminated. The value returned via @p *ndp
+ * does not include the terminator.
+ */
+char *utf16_to_utf8(const uint16_t *s, size_t ns, size_t *ndp) {
+ struct dynstr d;
+ uint32_t c;
+
+ dynstr_init(&d);
+ while(ns > 0) {
+ c = *s++;
+ --ns;
+ if(c >= 0xD800 && c <= 0xDBFF) {
+ if(ns && *s >= 0xDC00 && c <= 0xDFFF)
+ c = ((c - 0xD800) << 10) + (*s++ - 0xDC00) + 0x10000;
+ else
+ goto error;
+ } else if(c >= 0xDC00 && c <= 0xDFFF)
+ goto error;
+ if(c < 0x80)
+ dynstr_append(&d, c);
+ else if(c < 0x0800) {
+ dynstr_append(&d, 0xC0 | (c >> 6));
+ dynstr_append(&d, 0x80 | (c & 0x3F));
+ } else if(c < 0x10000) {
+ if(c >= 0xD800 && c <= 0xDFFF)
+ goto error;
+ dynstr_append(&d, 0xE0 | (c >> 12));
+ dynstr_append(&d, 0x80 | ((c >> 6) & 0x3F));
+ dynstr_append(&d, 0x80 | (c & 0x3F));
+ } else if(c < 0x110000) {
+ dynstr_append(&d, 0xF0 | (c >> 18));
+ dynstr_append(&d, 0x80 | ((c >> 12) & 0x3F));
+ dynstr_append(&d, 0x80 | ((c >> 6) & 0x3F));
+ dynstr_append(&d, 0x80 | (c & 0x3F));
+ } else
+ goto error;
+ }
+ dynstr_terminate(&d);
+ if(ndp)
+ *ndp = d.nvec;
+ return d.vec;
+error:
+ xfree(d.vec);
+ return 0;
+}
+
+/** @brief Convert UTF-8 to UTF-16
+ * @param s Source string
+ * @param ns Length of source string in code points
+ * @param ndp Where to store length of destination string (or NULL)
+ * @return Newly allocated destination string or NULL on error
+ *
+ * The return value is always 0-terminated. The value returned via @p *ndp
+ * does not include the terminator.
+ *
+ * If the UTF-8 is not valid then NULL is returned. A UTF-8 sequence
+ * for a code point is invalid if:
+ * - it is not the shortest possible sequence for the code point
+ * - it codes for a UTF-16 surrogate
+ * - it codes for a value outside the unicode code space
+ */
+uint16_t *utf8_to_utf16(const char *s, size_t ns, size_t *ndp) {
+ struct dynstr_utf16 d;
+ uint32_t c32;
+ const uint8_t *ss = (const uint8_t *)s;
+ int n;
+
+ dynstr_utf16_init(&d);
+ while(ns > 0) {
+ const struct unicode_utf8_row *const r = &unicode_utf8_valid[*ss];
+ if(r->count <= ns) {
+ switch(r->count) {
+ case 1:
+ c32 = *ss;
+ break;
+ case 2:
+ if(ss[1] < r->min2 || ss[1] > r->max2)
+ goto error;
+ c32 = *ss & 0x1F;
+ break;
+ case 3:
+ if(ss[1] < r->min2 || ss[1] > r->max2)
+ goto error;
+ c32 = *ss & 0x0F;
+ break;
+ case 4:
+ if(ss[1] < r->min2 || ss[1] > r->max2)
+ goto error;
+ c32 = *ss & 0x07;
+ break;
+ default:
+ goto error;
+ }
+ } else
+ goto error;
+ for(n = 1; n < r->count; ++n) {
+ if(ss[n] < 0x80 || ss[n] > 0xBF)
+ goto error;
+ c32 = (c32 << 6) | (ss[n] & 0x3F);
+ }
+ if(c32 >= 0x10000) {
+ c32 -= 0x10000;
+ dynstr_utf16_append(&d, 0xD800 + (c32 >> 10));
+ dynstr_utf16_append(&d, 0xDC00 + (c32 & 0x03FF));
+ } else
+ dynstr_utf16_append(&d, c32);
+ ss += r->count;
+ ns -= r->count;
+ }
+ dynstr_utf16_terminate(&d);
+ if(ndp)
+ *ndp = d.nvec;
+ return d.vec;
+error:
+ xfree(d.vec);
+ return 0;
+}
+
/** @brief Test whether [s,s+ns) is valid UTF-8
* @param s Start of string
* @param ns Length of string
* the value is (uint32_t)-1.
*/
uint32_t last[2];
-};
-/** @brief Create a new iterator pointing at the start of a string
- * @param s Start of string
- * @param ns Length of string
- * @return New iterator
- */
-utf32_iterator utf32_iterator_new(const uint32_t *s, size_t ns) {
- utf32_iterator it = xmalloc(sizeof *it);
- it->s = s;
- it->ns = ns;
- it->n = 0;
- it->last[0] = it->last[1] = -1;
- return it;
-}
+ /** @brief Tailoring for Word_Break */
+ unicode_property_tailor *word_break;
+};
/** @brief Initialize an internal private iterator
* @param it Iterator
it->ns = ns;
it->n = 0;
it->last[0] = it->last[1] = -1;
+ it->word_break = 0;
utf32_iterator_set(it, n);
}
+/** @brief Create a new iterator pointing at the start of a string
+ * @param s Start of string
+ * @param ns Length of string
+ * @return New iterator
+ */
+utf32_iterator utf32_iterator_new(const uint32_t *s, size_t ns) {
+ utf32_iterator it = xmalloc(sizeof *it);
+ utf32__iterator_init(it, s, ns, 0);
+ return it;
+}
+
+/** @brief Tailor this iterator's interpretation of the Word_Break property.
+ * @param it Iterator
+ * @param pt Property tailor function or NULL
+ *
+ * After calling this the iterator will call @p pt to determine the Word_Break
+ * property of each code point. If it returns -1 the default value will be
+ * used otherwise the returned value will be used.
+ *
+ * @p pt can be NULL to revert to the default value of the property.
+ *
+ * It is safe to call this function at any time; the iterator's internal state
+ * will be reset to suit the new tailoring.
+ */
+void utf32_iterator_tailor_word_break(utf32_iterator it,
+ unicode_property_tailor *pt) {
+ it->word_break = pt;
+ utf32_iterator_set(it, it->n);
+}
+
+static inline enum unicode_Word_Break utf32__iterator_word_break(utf32_iterator it,
+ uint32_t c) {
+ if(!it->word_break)
+ return utf32__word_break(c);
+ else {
+ const int t = it->word_break(c);
+
+ if(t < 0)
+ return utf32__word_break(c);
+ else
+ return t;
+ }
+}
+
/** @brief Destroy an iterator
* @param it Iterator
*/
return -1;
/* Walk backwards skipping ignorable code points */
m = n;
- while(m > 0 && (utf32__boundary_ignorable(utf32__word_break(it->s[m-1]))))
+ while(m > 0
+ && (utf32__boundary_ignorable(utf32__iterator_word_break(it,
+ it->s[m-1]))))
--m;
/* Either m=0 or s[m-1] is not ignorable */
if(m > 0) {
--m;
/* s[m] is our first non-ignorable code; look for a second in the same
way **/
- while(m > 0 && (utf32__boundary_ignorable(utf32__word_break(it->s[m-1]))))
+ while(m > 0
+ && (utf32__boundary_ignorable(utf32__iterator_word_break(it,
+ it->s[m-1]))))
--m;
/* Either m=0 or s[m-1] is not ignorable */
if(m > 0)
if(count <= it->ns - it->n) {
while(count > 0) {
const uint32_t c = it->s[it->n];
- const enum unicode_Word_Break wb = utf32__word_break(c);
+ const enum unicode_Word_Break wb = utf32__iterator_word_break(it, c);
if(it->last[1] == (uint32_t)-1
|| !utf32__boundary_ignorable(wb)) {
it->last[0] = it->last[1];
/* GB9 */
if(gbafter == unicode_Grapheme_Break_Extend)
return 0;
+ /* GB9a */
+ if(gbafter == unicode_Grapheme_Break_SpacingMark)
+ return 0;
+ /* GB9b */
+ if(gbbefore == unicode_Grapheme_Break_Prepend)
+ return 0;
/* GB10 */
return 1;
* string) and 0 otherwise.
*/
int utf32_iterator_word_boundary(utf32_iterator it) {
- enum unicode_Word_Break twobefore, before, after, twoafter;
+ uint32_t before, after;
+ enum unicode_Word_Break wbtwobefore, wbbefore, wbafter, wbtwoafter;
size_t nn;
/* WB1 and WB2 */
if(it->n == 0 || it->n == it->ns)
return 1;
+ before = it->s[it->n-1];
+ after = it->s[it->n];
/* WB3 */
- if(it->s[it->n-1] == 0x000D && it->s[it->n] == 0x000A)
+ if(before == 0x000D && after == 0x000A)
return 0;
+ /* WB3a */
+ if(utf32__iterator_word_break(it, before) == unicode_Word_Break_Newline
+ || before == 0x000D
+ || before == 0x000A)
+ return 1;
+ /* WB3b */
+ if(utf32__iterator_word_break(it, after) == unicode_Word_Break_Newline
+ || after == 0x000D
+ || after == 0x000A)
+ return 1;
/* WB4 */
/* (!Sep) x (Extend|Format) as in UAX #29 s6.2 */
- if(utf32__sentence_break(it->s[it->n-1]) != unicode_Sentence_Break_Sep
- && utf32__boundary_ignorable(utf32__word_break(it->s[it->n])))
+ if(utf32__sentence_break(before) != unicode_Sentence_Break_Sep
+ && utf32__boundary_ignorable(utf32__iterator_word_break(it, after)))
return 0;
/* Gather the property values we'll need for the rest of the test taking the
* s6.2 changes into account */
/* First we look at the code points after the proposed boundary */
nn = it->n; /* <it->ns */
- after = utf32__word_break(it->s[nn++]);
- if(!utf32__boundary_ignorable(after)) {
+ wbafter = utf32__iterator_word_break(it, it->s[nn++]);
+ if(!utf32__boundary_ignorable(wbafter)) {
/* X (Extend|Format)* -> X */
while(nn < it->ns
- && utf32__boundary_ignorable(utf32__word_break(it->s[nn])))
+ && utf32__boundary_ignorable(utf32__iterator_word_break(it,
+ it->s[nn])))
++nn;
}
/* It's possible now that nn=ns */
if(nn < it->ns)
- twoafter = utf32__word_break(it->s[nn]);
+ wbtwoafter = utf32__iterator_word_break(it, it->s[nn]);
else
- twoafter = unicode_Word_Break_Other;
+ wbtwoafter = unicode_Word_Break_Other;
/* We've already recorded the non-ignorable code points before the proposed
* boundary */
- before = utf32__word_break(it->last[1]);
- twobefore = utf32__word_break(it->last[0]);
+ wbbefore = utf32__iterator_word_break(it, it->last[1]);
+ wbtwobefore = utf32__iterator_word_break(it, it->last[0]);
/* WB5 */
- if(before == unicode_Word_Break_ALetter
- && after == unicode_Word_Break_ALetter)
+ if(wbbefore == unicode_Word_Break_ALetter
+ && wbafter == unicode_Word_Break_ALetter)
return 0;
/* WB6 */
- if(before == unicode_Word_Break_ALetter
- && after == unicode_Word_Break_MidLetter
- && twoafter == unicode_Word_Break_ALetter)
+ if(wbbefore == unicode_Word_Break_ALetter
+ && (wbafter == unicode_Word_Break_MidLetter
+ || wbafter == unicode_Word_Break_MidNumLet)
+ && wbtwoafter == unicode_Word_Break_ALetter)
return 0;
/* WB7 */
- if(twobefore == unicode_Word_Break_ALetter
- && before == unicode_Word_Break_MidLetter
- && after == unicode_Word_Break_ALetter)
+ if(wbtwobefore == unicode_Word_Break_ALetter
+ && (wbbefore == unicode_Word_Break_MidLetter
+ || wbbefore == unicode_Word_Break_MidNumLet)
+ && wbafter == unicode_Word_Break_ALetter)
return 0;
- /* WB8 */
- if(before == unicode_Word_Break_Numeric
- && after == unicode_Word_Break_Numeric)
+ /* WB8 */
+ if(wbbefore == unicode_Word_Break_Numeric
+ && wbafter == unicode_Word_Break_Numeric)
return 0;
/* WB9 */
- if(before == unicode_Word_Break_ALetter
- && after == unicode_Word_Break_Numeric)
+ if(wbbefore == unicode_Word_Break_ALetter
+ && wbafter == unicode_Word_Break_Numeric)
return 0;
/* WB10 */
- if(before == unicode_Word_Break_Numeric
- && after == unicode_Word_Break_ALetter)
+ if(wbbefore == unicode_Word_Break_Numeric
+ && wbafter == unicode_Word_Break_ALetter)
return 0;
/* WB11 */
- if(twobefore == unicode_Word_Break_Numeric
- && before == unicode_Word_Break_MidNum
- && after == unicode_Word_Break_Numeric)
+ if(wbtwobefore == unicode_Word_Break_Numeric
+ && (wbbefore == unicode_Word_Break_MidNum
+ || wbbefore == unicode_Word_Break_MidNumLet)
+ && wbafter == unicode_Word_Break_Numeric)
return 0;
/* WB12 */
- if(before == unicode_Word_Break_Numeric
- && after == unicode_Word_Break_MidNum
- && twoafter == unicode_Word_Break_Numeric)
+ if(wbbefore == unicode_Word_Break_Numeric
+ && (wbafter == unicode_Word_Break_MidNum
+ || wbafter == unicode_Word_Break_MidNumLet)
+ && wbtwoafter == unicode_Word_Break_Numeric)
return 0;
/* WB13 */
- if(before == unicode_Word_Break_Katakana
- && after == unicode_Word_Break_Katakana)
+ if(wbbefore == unicode_Word_Break_Katakana
+ && wbafter == unicode_Word_Break_Katakana)
return 0;
/* WB13a */
- if((before == unicode_Word_Break_ALetter
- || before == unicode_Word_Break_Numeric
- || before == unicode_Word_Break_Katakana
- || before == unicode_Word_Break_ExtendNumLet)
- && after == unicode_Word_Break_ExtendNumLet)
+ if((wbbefore == unicode_Word_Break_ALetter
+ || wbbefore == unicode_Word_Break_Numeric
+ || wbbefore == unicode_Word_Break_Katakana
+ || wbbefore == unicode_Word_Break_ExtendNumLet)
+ && wbafter == unicode_Word_Break_ExtendNumLet)
return 0;
/* WB13b */
- if(before == unicode_Word_Break_ExtendNumLet
- && (after == unicode_Word_Break_ALetter
- || after == unicode_Word_Break_Numeric
- || after == unicode_Word_Break_Katakana))
+ if(wbbefore == unicode_Word_Break_ExtendNumLet
+ && (wbafter == unicode_Word_Break_ALetter
+ || wbafter == unicode_Word_Break_Numeric
+ || wbafter == unicode_Word_Break_Katakana))
return 0;
/* WB14 */
return 1;
return utf32_iterator_word_boundary(it);
}
+/** @brief Split [s,ns) into multiple words
+ * @param s Pointer to start of string
+ * @param ns Length of string
+ * @param nwp Where to store word count, or NULL
+ * @param wbreak Word_Break property tailor, or NULL
+ * @return Pointer to array of pointers to words
+ *
+ * The returned array is terminated by a NULL pointer and individual
+ * strings are 0-terminated.
+ */
+uint32_t **utf32_word_split(const uint32_t *s, size_t ns, size_t *nwp,
+ unicode_property_tailor *wbreak) {
+ struct utf32_iterator_data it[1];
+ size_t b1 = 0, b2 = 0 ,i;
+ int isword;
+ struct vector32 v32[1];
+ uint32_t *w;
+
+ vector32_init(v32);
+ utf32__iterator_init(it, s, ns, 0);
+ it->word_break = wbreak;
+ /* Work our way through the string stopping at each word break. */
+ do {
+ if(utf32_iterator_word_boundary(it)) {
+ /* We've found a new boundary */
+ b1 = b2;
+ b2 = it->n;
+ /*fprintf(stderr, "[%zu, %zu) is a candidate word\n", b1, b2);*/
+ /* Inspect the characters between the boundary and form an opinion as to
+ * whether they are a word or not */
+ isword = 0;
+ for(i = b1; i < b2; ++i) {
+ switch(utf32__iterator_word_break(it, it->s[i])) {
+ case unicode_Word_Break_ALetter:
+ case unicode_Word_Break_Numeric:
+ case unicode_Word_Break_Katakana:
+ isword = 1;
+ break;
+ default:
+ break;
+ }
+ }
+ /* If it's a word add it to the list of results */
+ if(isword) {
+ const size_t len = b2 - b1;
+ w = xcalloc_noptr(len + 1, sizeof(uint32_t));
+ memcpy(w, it->s + b1, len * sizeof (uint32_t));
+ w[len] = 0;
+ vector32_append(v32, w);
+ }
+ }
+ } while(!utf32_iterator_advance(it, 1));
+ vector32_terminate(v32);
+ if(nwp)
+ *nwp = v32->nvec;
+ return v32->vec;
+}
+
/*@}*/
/** @defgroup utf8 Functions that operate on UTF-8 strings */
/*@{*/
utf8__transform(utf32_casefold_compat);
}
+/** @brief Split [s,ns) into multiple words
+ * @param s Pointer to start of string
+ * @param ns Length of string
+ * @param nwp Where to store word count, or NULL
+ * @param wbreak Word_Break property tailor, or NULL
+ * @return Pointer to array of pointers to words
+ *
+ * The returned array is terminated by a NULL pointer and individual
+ * strings are 0-terminated.
+ */
+char **utf8_word_split(const char *s, size_t ns, size_t *nwp,
+ unicode_property_tailor *wbreak) {
+ uint32_t *to32 = 0, **v32 = 0;
+ size_t nto32, nv, n;
+ char **v8 = 0, **ret = 0;
+
+ if(!(to32 = utf8_to_utf32(s, ns, &nto32))) goto error;
+ if(!(v32 = utf32_word_split(to32, nto32, &nv, wbreak))) goto error;
+ v8 = xcalloc(sizeof (char *), nv + 1);
+ for(n = 0; n < nv; ++n)
+ if(!(v8[n] = utf32_to_utf8(v32[n], utf32_len(v32[n]), 0)))
+ goto error;
+ ret = v8;
+ *nwp = nv;
+ v8 = 0; /* don't free */
+error:
+ if(v8) {
+ for(n = 0; n < nv; ++n)
+ xfree(v8[n]);
+ xfree(v8);
+ }
+ if(v32) {
+ for(n = 0; n < nv; ++n)
+ xfree(v32[n]);
+ xfree(v32);
+ }
+ xfree(to32);
+ return ret;
+}
+
+
/*@}*/
+/** @brief Return the length of a 0-terminated UTF-16 string
+ * @param s Pointer to 0-terminated string
+ * @return Length of string in code points (excluding terminator)
+ *
+ * Unlike the conversion functions no validity checking is done on the string.
+ */
+size_t utf16_len(const uint16_t *s) {
+ const uint16_t *t = s;
+
+ while(*t)
+ ++t;
+ return (size_t)(t - s);
+}
+
/*
Local Variables:
c-basic-offset:2
~mdw / disorder / blobdiff