d8feabb353b3c2650facea4afd08c86bb56e5549
kent
  Fri Apr 6 16:51:54 2012 -0700
Moving autoSql and autoDtd and autoXml back to just under hg.  A little autoSql -django fix.
diff --git src/hg/autoSql/eztok.c src/hg/autoSql/eztok.c
new file mode 100644
index 0000000..c2fadd5
--- /dev/null
+++ src/hg/autoSql/eztok.c
@@ -0,0 +1,71 @@
+/* Currently unused... */
+
+/* eztok - simple but often effective tokenizer.  Separates
+ * things into space or punctuation delimited words.  Punctuation
+ * (anything non-alphanumeric) is returned as a single character
+ * token.  It will handle quotes, but not allow escapes within
+ * quotes. */
+
+#include "common.h"
+#include "eztok.h"
+
+
+struct ezTok *ezNewTok(char *string, int stringSize)
+/* Allocate and initialize a new token. */
+{
+struct ezTok *tok;
+int totalSize = stringSize + sizeof(*tok);
+tok = needMem(totalSize);
+memcpy(tok->string, string, stringSize);
+return tok;
+}
+
+struct ezTok *ezTokenize(char *text)
+/* Convert text to list of tokens. */
+{
+struct ezTok *tokList = NULL, *tok;
+char c, *s, *start = NULL, *end = NULL;
+
+s = text;
+for (;;)
+    {
+    if ((c = *s) == 0)
+        break;
+    start = s++;
+    if (isspace(c))
+        {
+        continue;
+        }
+    else if (isalnum(c) || (c = '_'))
+        {
+        for (;;)
+            {
+            c = *s;
+            if (isalnum(c) || (c = '_'))
+                ++s;
+            else
+                break;
+            }
+        end = s;
+        }
+    else if (c == '"')
+        {
+        start = s;
+        for (;;)
+            {
+            c = *s++;
+            if (c == '"')
+                break;
+            }
+        end = s-1;
+        }
+    else
+        {
+        end = s;
+        }
+    tok = ezNewTok(start, end-start);
+    slAddHead(&tokList, tok);
+    }
+slReverse(&tokList);
+return tokList;
+}