X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~mdw/git/disorder/blobdiff_plain/0db97c0d2f4568c2b5b5f5f9b791b8025b4732aa..16b0fea8ae1a581d568dbee2efa2932aa4e6fcce:/lib/unicode.c

diff --git a/lib/unicode.c b/lib/unicode.c
index 40b9854..27f82b7 100644
--- a/lib/unicode.c
+++ b/lib/unicode.c
@@ -1,21 +1,19 @@
 /*
  * This file is part of DisOrder
- * Copyright (C) 2007 Richard Kettlewell
+ * Copyright (C) 2007, 2009, 2013 Richard Kettlewell
  *
- * This program is free software; you can redistribute it and/or modify
+ * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
+ * the Free Software Foundation, either version 3 of the License, or
  * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
+ * 
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
  * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- * USA
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 /** @file lib/unicode.c
  * @brief Unicode support functions
@@ -39,11 +37,7 @@
  * - @ref utf8
  */
 
-#include <config.h>
-#include "types.h"
-
-#include <string.h>
-#include <stdio.h>		/* TODO */
+#include "common.h"
 
 #include "mem.h"
 #include "vector.h"
@@ -109,6 +103,16 @@ static inline int utf32__combining_class(uint32_t c) {
   return utf32__unidata(c)->ccc;
 }
 
+/** @brief Return the combining class of @p c
+ * @param c Code point
+ * @return Combining class of @p c
+ *
+ * @p c can be any 32-bit value, a sensible value will be returned regardless.
+ */
+int utf32_combining_class(uint32_t c) {
+  return utf32__combining_class(c);
+}
+
 /** @brief Return the General_Category value for @p c
  * @param c Code point
  * @return General_Category property value
@@ -301,6 +305,133 @@ error:
   return 0;
 }
 
+/** @brief Convert UTF-16 to UTF-8
+ * @param s Source string
+ * @param ns Length of source string in code points
+ * @param ndp Where to store length of destination string (or NULL)
+ * @return Newly allocated destination string or NULL on error
+ *
+ * If the UTF-16 is not valid then NULL is returned.  A UTF-16 sequence t is
+ * invalid if it contains an incomplete surrogate.
+ *
+ * The return value is always 0-terminated.  The value returned via @p *ndp
+ * does not include the terminator.
+ */
+char *utf16_to_utf8(const uint16_t *s, size_t ns, size_t *ndp) {
+  struct dynstr d;
+  uint32_t c;
+
+  dynstr_init(&d);
+  while(ns > 0) {
+    c = *s++;
+    --ns;
+    if(c >= 0xD800 && c <= 0xDBFF) {
+      if(ns && *s >= 0xDC00 && c <= 0xDFFF)
+        c = ((c - 0xD800) << 10) + (*s++ - 0xDC00) + 0x10000;
+      else
+        goto error;
+    } else if(c >= 0xDC00 && c <= 0xDFFF)
+      goto error;
+    if(c < 0x80)
+      dynstr_append(&d, c);
+    else if(c < 0x0800) {
+      dynstr_append(&d, 0xC0 | (c >> 6));
+      dynstr_append(&d, 0x80 | (c & 0x3F));
+    } else if(c < 0x10000) {
+      if(c >= 0xD800 && c <= 0xDFFF)
+	goto error;
+      dynstr_append(&d, 0xE0 | (c >> 12));
+      dynstr_append(&d, 0x80 | ((c >> 6) & 0x3F));
+      dynstr_append(&d, 0x80 | (c & 0x3F));
+    } else if(c < 0x110000) {
+      dynstr_append(&d, 0xF0 | (c >> 18));
+      dynstr_append(&d, 0x80 | ((c >> 12) & 0x3F));
+      dynstr_append(&d, 0x80 | ((c >> 6) & 0x3F));
+      dynstr_append(&d, 0x80 | (c & 0x3F));
+    } else
+      goto error;
+  }
+  dynstr_terminate(&d);
+  if(ndp)
+    *ndp = d.nvec;
+  return d.vec;
+error:
+  xfree(d.vec);
+  return 0;
+}
+
+/** @brief Convert UTF-8 to UTF-16
+ * @param s Source string
+ * @param ns Length of source string in code points
+ * @param ndp Where to store length of destination string (or NULL)
+ * @return Newly allocated destination string or NULL on error
+ *
+ * The return value is always 0-terminated.  The value returned via @p *ndp
+ * does not include the terminator.
+ *
+ * If the UTF-8 is not valid then NULL is returned.  A UTF-8 sequence
+ * for a code point is invalid if:
+ * - it is not the shortest possible sequence for the code point
+ * - it codes for a UTF-16 surrogate
+ * - it codes for a value outside the unicode code space
+ */
+uint16_t *utf8_to_utf16(const char *s, size_t ns, size_t *ndp) {
+  struct dynstr_utf16 d;
+  uint32_t c32;
+  const uint8_t *ss = (const uint8_t *)s;
+  int n;
+
+  dynstr_utf16_init(&d);
+  while(ns > 0) {
+    const struct unicode_utf8_row *const r = &unicode_utf8_valid[*ss];
+    if(r->count <= ns) {
+      switch(r->count) {
+      case 1:
+        c32 = *ss;
+        break;
+      case 2:
+        if(ss[1] < r->min2 || ss[1] > r->max2)
+          goto error;
+        c32 = *ss & 0x1F;
+        break;
+      case 3:
+        if(ss[1] < r->min2 || ss[1] > r->max2)
+          goto error;
+        c32 = *ss & 0x0F;
+        break;
+      case 4:
+        if(ss[1] < r->min2 || ss[1] > r->max2)
+          goto error;
+        c32 = *ss & 0x07;
+        break;
+      default:
+        goto error;
+      }
+    } else
+      goto error;
+    for(n = 1; n < r->count; ++n) {
+      if(ss[n] < 0x80 || ss[n] > 0xBF)
+        goto error;
+      c32 = (c32 << 6) | (ss[n] & 0x3F);
+    }
+    if(c32 >= 0x10000) {
+      c32 -= 0x10000;
+      dynstr_utf16_append(&d, 0xD800 + (c32 >> 10));
+      dynstr_utf16_append(&d, 0xDC00 + (c32 & 0x03FF));
+    } else
+      dynstr_utf16_append(&d, c32);
+    ss += r->count;
+    ns -= r->count;
+  }
+  dynstr_utf16_terminate(&d);
+  if(ndp)
+    *ndp = d.nvec;
+  return d.vec;
+error:
+  xfree(d.vec);
+  return 0;
+}
+
 /** @brief Test whether [s,s+ns) is valid UTF-8
  * @param s Start of string
  * @param ns Length of string
@@ -599,6 +730,12 @@ int utf32_iterator_grapheme_boundary(utf32_iterator it) {
   /* GB9 */
   if(gbafter == unicode_Grapheme_Break_Extend)
     return 0;
+  /* GB9a */
+  if(gbafter == unicode_Grapheme_Break_SpacingMark)
+    return 0;
+  /* GB9b */
+  if(gbbefore == unicode_Grapheme_Break_Prepend)
+    return 0;
   /* GB10 */
   return 1;
 
@@ -614,26 +751,39 @@ int utf32_iterator_grapheme_boundary(utf32_iterator it) {
  * string) and 0 otherwise.
  */
 int utf32_iterator_word_boundary(utf32_iterator it) {
-  enum unicode_Word_Break twobefore, before, after, twoafter;
+  uint32_t before, after;
+  enum unicode_Word_Break wbtwobefore, wbbefore, wbafter, wbtwoafter;
   size_t nn;
 
   /* WB1 and WB2 */
   if(it->n == 0 || it->n == it->ns)
     return 1;
+  before = it->s[it->n-1];
+  after = it->s[it->n];
   /* WB3 */
-  if(it->s[it->n-1] == 0x000D && it->s[it->n] == 0x000A)
+  if(before == 0x000D && after == 0x000A)
     return 0;
+  /* WB3a */
+  if(utf32__iterator_word_break(it, before) == unicode_Word_Break_Newline
+     || before == 0x000D
+     || before == 0x000A)
+    return 1;
+  /* WB3b */
+  if(utf32__iterator_word_break(it, after) == unicode_Word_Break_Newline
+     || after == 0x000D
+     || after == 0x000A)
+    return 1;
   /* WB4 */
   /* (!Sep) x (Extend|Format) as in UAX #29 s6.2 */
-  if(utf32__sentence_break(it->s[it->n-1]) != unicode_Sentence_Break_Sep
-     && utf32__boundary_ignorable(utf32__iterator_word_break(it, it->s[it->n])))
+  if(utf32__sentence_break(before) != unicode_Sentence_Break_Sep
+     && utf32__boundary_ignorable(utf32__iterator_word_break(it, after)))
     return 0;
   /* Gather the property values we'll need for the rest of the test taking the
    * s6.2 changes into account */
   /* First we look at the code points after the proposed boundary */
   nn = it->n;                           /* <it->ns */
-  after = utf32__iterator_word_break(it, it->s[nn++]);
-  if(!utf32__boundary_ignorable(after)) {
+  wbafter = utf32__iterator_word_break(it, it->s[nn++]);
+  if(!utf32__boundary_ignorable(wbafter)) {
     /* X (Extend|Format)* -> X */
     while(nn < it->ns
           && utf32__boundary_ignorable(utf32__iterator_word_break(it,
@@ -642,67 +792,71 @@ int utf32_iterator_word_boundary(utf32_iterator it) {
   }
   /* It's possible now that nn=ns */
   if(nn < it->ns)
-    twoafter = utf32__iterator_word_break(it, it->s[nn]);
+    wbtwoafter = utf32__iterator_word_break(it, it->s[nn]);
   else
-    twoafter = unicode_Word_Break_Other;
+    wbtwoafter = unicode_Word_Break_Other;
 
   /* We've already recorded the non-ignorable code points before the proposed
    * boundary */
-  before = utf32__iterator_word_break(it, it->last[1]);
-  twobefore = utf32__iterator_word_break(it, it->last[0]);
+  wbbefore = utf32__iterator_word_break(it, it->last[1]);
+  wbtwobefore = utf32__iterator_word_break(it, it->last[0]);
 
   /* WB5 */
-  if(before == unicode_Word_Break_ALetter
-     && after == unicode_Word_Break_ALetter)
+  if(wbbefore == unicode_Word_Break_ALetter
+     && wbafter == unicode_Word_Break_ALetter)
     return 0;
   /* WB6 */
-  if(before == unicode_Word_Break_ALetter
-     && after == unicode_Word_Break_MidLetter
-     && twoafter == unicode_Word_Break_ALetter)
+  if(wbbefore == unicode_Word_Break_ALetter
+     && (wbafter == unicode_Word_Break_MidLetter
+         || wbafter == unicode_Word_Break_MidNumLet)
+     && wbtwoafter == unicode_Word_Break_ALetter)
     return 0;
   /* WB7 */
-  if(twobefore == unicode_Word_Break_ALetter
-     && before == unicode_Word_Break_MidLetter
-     && after == unicode_Word_Break_ALetter)
+  if(wbtwobefore == unicode_Word_Break_ALetter
+     && (wbbefore == unicode_Word_Break_MidLetter
+         || wbbefore == unicode_Word_Break_MidNumLet)
+     && wbafter == unicode_Word_Break_ALetter)
     return 0;
   /* WB8 */
-  if(before == unicode_Word_Break_Numeric
-     && after == unicode_Word_Break_Numeric)
+  if(wbbefore == unicode_Word_Break_Numeric
+     && wbafter == unicode_Word_Break_Numeric)
     return 0;
   /* WB9 */
-  if(before == unicode_Word_Break_ALetter
-     && after == unicode_Word_Break_Numeric)
+  if(wbbefore == unicode_Word_Break_ALetter
+     && wbafter == unicode_Word_Break_Numeric)
     return 0;
   /* WB10 */
-  if(before == unicode_Word_Break_Numeric
-     && after == unicode_Word_Break_ALetter)
+  if(wbbefore == unicode_Word_Break_Numeric
+     && wbafter == unicode_Word_Break_ALetter)
     return 0;
    /* WB11 */
-  if(twobefore == unicode_Word_Break_Numeric
-     && before == unicode_Word_Break_MidNum
-     && after == unicode_Word_Break_Numeric)
+  if(wbtwobefore == unicode_Word_Break_Numeric
+     && (wbbefore == unicode_Word_Break_MidNum
+         || wbbefore == unicode_Word_Break_MidNumLet)
+     && wbafter == unicode_Word_Break_Numeric)
     return 0;
   /* WB12 */
-  if(before == unicode_Word_Break_Numeric
-     && after == unicode_Word_Break_MidNum
-     && twoafter == unicode_Word_Break_Numeric)
+  if(wbbefore == unicode_Word_Break_Numeric
+     && (wbafter == unicode_Word_Break_MidNum
+         || wbafter == unicode_Word_Break_MidNumLet)
+     && wbtwoafter == unicode_Word_Break_Numeric)
     return 0;
   /* WB13 */
-  if(before == unicode_Word_Break_Katakana
-     && after == unicode_Word_Break_Katakana)
+  if(wbbefore == unicode_Word_Break_Katakana
+     && wbafter == unicode_Word_Break_Katakana)
     return 0;
   /* WB13a */
-  if((before == unicode_Word_Break_ALetter
-      || before == unicode_Word_Break_Numeric
-      || before == unicode_Word_Break_Katakana
-      || before == unicode_Word_Break_ExtendNumLet)
-     && after == unicode_Word_Break_ExtendNumLet)
+  if((wbbefore == unicode_Word_Break_ALetter
+      || wbbefore == unicode_Word_Break_Numeric
+      || wbbefore == unicode_Word_Break_Katakana
+      || wbbefore == unicode_Word_Break_ExtendNumLet)
+     && wbafter == unicode_Word_Break_ExtendNumLet)
     return 0;
   /* WB13b */
-  if(before == unicode_Word_Break_ExtendNumLet
-     && (after == unicode_Word_Break_ALetter
-         || after == unicode_Word_Break_Numeric
-         || after == unicode_Word_Break_Katakana))
+  if(wbbefore == unicode_Word_Break_ExtendNumLet
+     && (wbafter == unicode_Word_Break_ALetter
+         || wbafter == unicode_Word_Break_Numeric
+         || wbafter == unicode_Word_Break_Katakana))
     return 0;
   /* WB14 */
   return 1;
@@ -1354,8 +1508,10 @@ uint32_t **utf32_word_split(const uint32_t *s, size_t ns, size_t *nwp,
       }
       /* If it's a word add it to the list of results */
       if(isword) {
-        w = xcalloc(b2 - b1 + 1, sizeof(uint32_t));
-        memcpy(w, it->s + b1, (b2 - b1) * sizeof (uint32_t));
+        const size_t len = b2 - b1;
+        w = xcalloc_noptr(len + 1, sizeof(uint32_t));
+        memcpy(w, it->s + b1, len * sizeof (uint32_t));
+        w[len] = 0;
         vector32_append(v32, w);
       }
     }
@@ -1549,6 +1705,20 @@ error:
 
 /*@}*/
 
+/** @brief Return the length of a 0-terminated UTF-16 string
+ * @param s Pointer to 0-terminated string
+ * @return Length of string in code points (excluding terminator)
+ *
+ * Unlike the conversion functions no validity checking is done on the string.
+ */
+size_t utf16_len(const uint16_t *s) {
+  const uint16_t *t = s;
+
+  while(*t)
+    ++t;
+  return (size_t)(t - s);
+}
+
 /*
 Local Variables:
 c-basic-offset:2