45872bfa8b5701e2da983a5052c9f5c18c661621
kent
  Thu Dec 9 12:54:18 2021 -0800
Adding option for tokenizer to handle a pretty good spectrum of two character operations.

diff --git src/lib/tokenizer.c src/lib/tokenizer.c
index 7fc9e41..0a5c5de 100644
--- src/lib/tokenizer.c
+++ src/lib/tokenizer.c
@@ -148,32 +148,59 @@
 	    else
 		break;
 	    }
 	else if (c == 0)
 	    {
 	    break;
 	    }
 	++s;
 	}
     end = s;
     if (c != 0)
 	++s;
     if (tkz->leaveQuotes)
 	end += 1;
     }
-else
+else /* case for punctuation etc. */
+    {
+    if (tkz->twoCharOps)
+        {
+	int opSize = 1;
+	switch (c)
 	   {
+	   case '>':
+	   case '<':
+	   case '=':
+	   case '!':
+	       if (start[1] == '=')
+		   opSize = 2;
+	       break;
+	   case '+':
+	   case '-':
+	   case '|':
+	   case '&':
+	       if (start[1] == c)
+		   opSize = 2;
+	       break;
+	   default:
+	       opSize = 1;
+	       break;
+	   }
+	s += opSize;
+	end = s;
+	}
+    else
 	end = ++s;
     }
 tkz->linePt = s;
 size = end - start;
 if (size >= tkz->sAlloc)
     {
     tkz->sAlloc = size+128;
     tkz->string = needMoreMem(tkz->string, 0, tkz->sAlloc);
     }
 memcpy(tkz->string, start, size);
 tkz->string[size] = 0;
 return tkz->string;
 }