utf32_word_split() and utf8_word_split() splits a string into words

[disorder] / lib / words.c

diff --git a/lib/words.c b/lib/words.c
index e6e4087b444ad4e0d7b5db8526212b1584920da6..89174cd8e5048ff9047dd1c652fcc8de760d1810 100644 (file)
--- a/lib/words.c
+++ b/lib/words.c
@@ -33,125 +33,19 @@
 #include "charset.h"
 
 #include "unidata.h"
+#include "unicode.h"
 
 const char *casefold(const char *ptr) {
-  struct dynstr d;
-  uint32_t c;
-  const char *s = ptr;
-
-  dynstr_init(&d);
-  while(*s) {
-    /* Convert UTF-8 to UCS-32 */
-    PARSE_UTF8(s, c, return ptr);
-    /* Normalize */
-    if(c < UNICODE_NCHARS) {
-      /* If this a known character, convert it to lower case */
-      const struct unidata *const ud = &unidata[c / 256][c % 256];
-      c += ud->lower_offset;
-    }
-    /* Convert UCS-4 back to UTF-8 */
-    one_ucs42utf8(c, &d);
-  }
-  dynstr_terminate(&d);
-  return d.vec;
-}
-
-static enum unicode_gc_cat cat(uint32_t c) {
-  if(c < UNICODE_NCHARS) {
-    /* If this a known character, convert it to lower case */
-    const struct unidata *const ud = &unidata[c / 256][c % 256];
-    return ud->gc;
-  } else
-    return unicode_gc_Cn;
+  return utf8_casefold_compat(ptr, strlen(ptr), 0);
 }
 
-/* XXX this is a bit kludgy */
-
 char **words(const char *s, int *nvecp) {
-  struct vector v;
-  struct dynstr d;
-  const char *start;
-  uint32_t c;
-  int in_word = 0;
-
-  vector_init(&v);
-  while(*s) {
-    start = s;
-    PARSE_UTF8(s, c, return 0);
-    /* special cases first */
-    switch(c) {
-    case '/':
-    case '.':
-    case '+':
-    case '&':
-    case ':':
-    case '_':
-    case '-':
-      goto separator;
-    }
-    /* do the rest on category */
-    switch(cat(c)) {
-    case unicode_gc_Ll:
-    case unicode_gc_Lm:
-    case unicode_gc_Lo:
-    case unicode_gc_Lt:
-    case unicode_gc_Lu:
-    case unicode_gc_Nd:
-    case unicode_gc_Nl:
-    case unicode_gc_No:
-    case unicode_gc_Sc:
-    case unicode_gc_Sk:
-    case unicode_gc_Sm:
-    case unicode_gc_So:
-      /* letters, digits and symbols are considered to be part of
-       * words */
-      if(!in_word) {
-       dynstr_init(&d);
-       in_word = 1;
-      }
-      dynstr_append_bytes(&d, start, s - start);
-      break;
-
-    case unicode_gc_Cc:
-    case unicode_gc_Cf:
-    case unicode_gc_Co:
-    case unicode_gc_Cs:
-    case unicode_gc_Zl:
-    case unicode_gc_Zp:
-    case unicode_gc_Zs:
-    case unicode_gc_Pe:
-    case unicode_gc_Ps:
-    separator:
-      if(in_word) {
-       dynstr_terminate(&d);
-       vector_append(&v, d.vec);
-       in_word = 0;
-      }
-      break;
-
-    case unicode_gc_Mc:
-    case unicode_gc_Me:
-    case unicode_gc_Mn:
-    case unicode_gc_Pc:
-    case unicode_gc_Pd:
-    case unicode_gc_Pf:
-    case unicode_gc_Pi:
-    case unicode_gc_Po:
-    case unicode_gc_Cn:
-      /* control and punctuation is completely ignored */
-      break;
+  size_t nv;
+  char **v;
 
-    }
-  }
-  if(in_word) {
-    /* pick up the final word */
-    dynstr_terminate(&d);
-    vector_append(&v, d.vec);
-  }
-  vector_terminate(&v);
-  if(nvecp)
-    *nvecp = v.nvec;
-  return v.vec;
+  v = utf8_word_split(s, strlen(s), &nv);
+  *nvecp = nv;
+  return v;
 }
 
 /*