6e7b8bdf6aeb604219a24f35cf1d303a0cb6edef
kent
  Wed Aug 7 18:08:33 2019 -0700
Librarifying strex string expression module that is going in for tabToTabDir to help sort through some of HCA madness.

diff --git src/hg/oneShot/freen/freen.c src/hg/oneShot/freen/freen.c
index ce09600..de06c01 100644
--- src/hg/oneShot/freen/freen.c
+++ src/hg/oneShot/freen/freen.c
@@ -1,1095 +1,81 @@
 /* freen - My Pet Freen.  A pet freen is actually more dangerous than a wild one. */
 
 /* Copyright (C) 2014 The Regents of the University of California 
  * See README in this or parent directory for licensing information. */
 
 #include "common.h"
 #include "linefile.h"
 #include "hash.h"
 #include "options.h"
 #include "dystring.h"
 #include "cheapcgi.h"
 #include "jksql.h"
 #include "portable.h"
 #include "obscure.h"
 #include "localmem.h"
 #include "csv.h"
 #include "tokenizer.h"
 #include "strex.h"
 #include "hmac.h"
 
 /* Command line validation table. */
 static struct optionSpec options[] = {
    {NULL, 0},
 };
 
 void usage()
 {
 errAbort("freen - test some hairbrained thing.\n"
          "usage:  freen input\n");
 }
 
 /*  strex -String Expression - stuff to implement a relatively simple string
  *  processing expression parser. */
 
-
-enum strexType
-/* A type */
-    {
-    strexTypeBoolean = 1,
-    strexTypeString = 2,
-    strexTypeInt = 3,
-    strexTypeDouble = 4,
-    };
-
-enum strexBuiltInFunc
-/* One of these for each builtIn.  We'll just do a switch to implement */
-    {
-    strexBuiltInTrim,
-    strexBuiltInBetween,
-    strexBuiltInSpaced,
-    strexBuiltInNow,
-    strexBuiltInMd5,
-    strexBuiltInSplit,
-    };
-
-struct strexBuiltIn
-/* A built in function */
-    {
-    char *name;
-    enum strexBuiltInFunc func;
-    int paramCount;
-    enum strexType *paramTypes;
-    };
-
-
-union strexVal
-/* Some value of arbirary type that can be of any type corresponding to strexType */
-    {
-    boolean b;
-    char *s;
-    long long i;
-    double x;
-    struct strexBuiltIn *builtIn;
-    };
-
-struct strexEval
-/* Result of evaluation of parse tree. */
-    {
-    enum strexType type;
-    union strexVal val;
-    };
-
-enum strexOp
-/* An operation in the parse tree. */
-    {
-    strexOpUnknown,	/* Should not occur */
-    strexOpLiteral,        /* Literal string or number. */
-    strexOpSymbol,	/* A symbol name. */
-
-    strexOpBuiltInCall,	/* Call a built in function */
-    strexOpArrayIx,	/* An array with an index. */
-
-    strexOpUnaryMinusInt,
-    strexOpUnaryMinusDouble,
-
-    /* Binary operations. */
-    strexOpAdd,
-
-    /* Type conversions */
-    strexOpStringToBoolean,
-    strexOpIntToBoolean,
-    strexOpDoubleToBoolean,
-    strexOpStringToInt,
-    strexOpDoubleToInt,
-    strexOpBooleanToInt,
-    strexOpStringToDouble,
-    strexOpBooleanToDouble,
-    strexOpIntToDouble,
-    strexOpBooleanToString,
-    strexOpIntToString,
-    strexOpDoubleToString,
-    };
-
-
-struct strexParse
-/* A strex parse-tree. */
-    {
-    struct strexParse *next;	/* Points to younger sibling if any. */
-    struct strexParse *children;	/* Points to oldest child if any. */
-    enum strexOp op;		/* Operation at this node. */
-    enum strexType type;		/* Return type of this operation. */
-    union strexVal val;		/* Return value of this operation. */
-    };
-
-struct strexIn
-/* Input to the strex parser */
-    {
-    struct tokenizer *tkz;  /* Get next text input from here */
-    struct hash *builtInHash;  /* Hash of built in functions */
-    };
-
-enum strexType oneString[] = {strexTypeString};
-enum strexType twoStrings[] = {strexTypeString, strexTypeString};
-enum strexType threeStrings[] = {strexTypeString, strexTypeString, strexTypeString};
-enum strexType stringInt[] = {strexTypeString, strexTypeInt};
-enum strexType stringStringInt[] = {strexTypeString, strexTypeString, strexTypeInt};
-
-static struct strexBuiltIn builtins[] = {
-    { "trim", strexBuiltInTrim, 1, oneString, },
-    { "between", strexBuiltInBetween, 3, threeStrings, },
-    { "spaced", strexBuiltInSpaced, 2, stringInt },
-    { "now", strexBuiltInNow, 0, NULL },
-    { "md5", strexBuiltInMd5, 1, oneString },
-    { "split", strexBuiltInSplit, 3, stringStringInt },
-};
-
-
-
-static struct hash *hashBuiltIns()
-/* Build a hash of builtins keyed by name */
-{
-struct hash *hash = hashNew(0);
-int i;
-for (i=0; i<ArraySize(builtins); ++i)
-    hashAdd(hash, builtins[i].name, &builtins[i]);
-return hash;
-}
-
-static struct strexIn *strexInNew(char *expression)
-/* Return a new strexIn structure wrapped around expression */
-{
-struct lineFile *lf = lineFileOnString(expression, TRUE, cloneString(expression));
-struct tokenizer *tkz = tokenizerOnLineFile(lf);
-tkz->leaveQuotes = TRUE;
-struct strexIn *si;
-AllocVar(si);
-si->tkz = tkz;
-si->builtInHash = hashBuiltIns();
-return si;
-}
-
-void strexInFree(struct strexIn **pSi)
-/* Free up memory associated with strexIn structure */
-{
-struct strexIn *si = *pSi;
-if (si != NULL)
-    {
-    hashFree(&si->builtInHash);
-    tokenizerFree(&si->tkz);
-    freez(pSi);
-    }
-}
-
-static void strexValDump(union strexVal val, enum strexType type, FILE *f)
-/* Dump out value to file. */
-{
-switch (type)
-    {
-    case strexTypeBoolean:
-        fprintf(f, "%s", (val.b ? "true" : "false") );
-	break;
-    case strexTypeString:
-        fprintf(f, "%s", (val.s == NULL ? "(null)" : val.s));
-	break;
-    case strexTypeInt:
-        fprintf(f, "%lld", val.i);
-	break;
-    case strexTypeDouble:
-        fprintf(f, "%f", val.x);
-	break;
-    }
-}
-
-static char *strexTypeToString(enum strexType type)
-/* Return a string representation of type */
-{
-switch (type)
-    {
-    case strexTypeBoolean:
-	return "boolean";
-	break;
-    case strexTypeString:
-	return "string";
-	break;
-    case strexTypeInt:
-	return "integer";
-	break;
-    case strexTypeDouble:
-	return "floating point";
-	break;
-    default:
-        internalErr();
-	return NULL;
-    }
-}
-
-static char *strexOpToString(enum strexOp op)
-/* Return string representation of parse op. */
-{
-switch (op)
-    {
-    case strexOpLiteral:
-	return "strexOpLiteral";
-    case strexOpSymbol:
-	return "strexOpSymbol";
-    
-    case strexOpStringToBoolean:
-        return "strexOpStringToBoolean";
-    case strexOpIntToBoolean:
-        return "strexOpIntToBoolean";
-    case strexOpDoubleToBoolean:
-        return "strexOpDoubleToBoolean";
-    case strexOpStringToInt:
-        return "strexOpStringToInt";
-    case strexOpDoubleToInt:
-        return "strexOpDoubleToInt";
-    case strexOpBooleanToInt:
-        return "strexOpBooleanToInt";
-    case strexOpStringToDouble:
-        return "strexOpStringToDouble";
-    case strexOpBooleanToDouble:
-        return "strexOpBooleanToDouble";
-    case strexOpIntToDouble:
-        return "strexOpIntToDouble";
-    case strexOpBooleanToString:
-        return "strexOpBooleanToString";
-    case strexOpIntToString:
-        return "strexOpIntToString";
-    case strexOpDoubleToString:
-        return "strexOpDoubleToString";
-
-    case strexOpUnaryMinusInt:
-        return "strexOpUnaryMinusInt";
-    case strexOpUnaryMinusDouble:
-        return "strexOpUnaryMinusDouble";
-
-
-    case strexOpAdd:
-	return "strexOpAdd";
-
-    case strexOpBuiltInCall:
-        return "strexOpBuiltInCall";
-
-    case strexOpArrayIx:
-        return "strexOpArrayIx";
-
-    default:
-	return "strexOpUnknown";
-    }
-}
-
-void strexParseDump(struct strexParse *p, int depth, FILE *f)
-/* Dump out strexParse tree and children. */
-{
-spaceOut(f, 3*depth);
-fprintf(f, "%s ", strexOpToString(p->op));
-strexValDump(p->val, p->type,  f);
-fprintf(f, "\n");
-struct strexParse *child;
-for (child = p->children; child != NULL; child= child->next)
-    strexParseDump(child, depth+1, f);
-}
-
-static void expectingGot(struct strexIn *in, char *expecting, char *got)
-/* Print out error message about unexpected input. */
-{
-errAbort("Expecting %s, got %s, line %d of %s", expecting, got, in->tkz->lf->lineIx,
-	in->tkz->lf->fileName);
-}
-
-static void skipOverRequired(struct strexIn *in, char *expecting)
-/* Make sure that next token is tok, and skip over it. */
-{
-tokenizerMustHaveNext(in->tkz);
-if (!sameWord(in->tkz->string, expecting))
-    expectingGot(in, expecting, in->tkz->string);
-}
-
-
-struct strexParse *strexParseExpression(struct strexIn *in);
-/* Parse out an expression with a single value */
-
-static struct strexParse *strexParseAtom(struct strexIn *in)
-/* Return low level (symbol or literal) */
-{
-struct tokenizer *tkz = in->tkz;
-char *tok = tokenizerMustHaveNext(tkz);
-struct strexParse *p;
-AllocVar(p);
-char c = tok[0];
-if (c == '\'' || c == '"')
-    {
-    p->op = strexOpLiteral;
-    p->type = strexTypeString;
-    int len = strlen(tok+1);
-    p->val.s = cloneStringZ(tok+1, len-1);
-    }
-else if (isalpha(c) || c == '_')
-    {
-    p->op = strexOpSymbol;
-    p->type = strexTypeString;	/* String until promoted at least. */
-    struct dyString *dy = dyStringNew(64);
-    for (;;)  // Join together . separated things into single symbol */
-	{
-	dyStringAppend(dy, tok);
-	if ((tok = tokenizerNext(tkz)) == NULL)
-	    break;
-	if (tok[0] != '.')
-	    {
-	    tokenizerReuse(tkz);
-	    break;
-	    }
-	dyStringAppend(dy, tok);
-	if ((tok = tokenizerNext(tkz)) == NULL)
-	    break;
-	}
-    p->val.s = dyStringCannibalize(&dy);
-    }
-else if (isdigit(c))
-    {
-    p->op = strexOpLiteral;
-    p->type = strexTypeInt;
-    p->val.i = sqlUnsigned(tok);
-    if ((tok = tokenizerNext(tkz)) != NULL)
-	{
-	if (tok[0] == '.')
-	    {
-	    char buf[32];
-	    tok = tokenizerMustHaveNext(tkz);
-	    safef(buf, sizeof(buf), "%lld.%s", p->val.i, tok);
-	    p->type = strexTypeDouble;
-	    p->val.x = sqlDouble(buf);
-	    }
-	else
-	    tokenizerReuse(tkz);
-	}
-    }
-else if (c == '(')
-    {
-    p = strexParseExpression(in);
-    skipOverRequired(in, ")");
-    }
-else
-    {
-    errAbort("Unexpected %s line %d of %s", tok, tkz->lf->lineIx, tkz->lf->fileName);
-    }
-return p;
-}
-
-static enum strexType commonTypeForBop(enum strexType left, enum strexType right)
-/* Return type that will work for a binary operation. */
-{
-if (left == right)
-    return left;
-else if (left == strexTypeString || right == strexTypeString)
-    return strexTypeString;
-else if (left == strexTypeDouble || right == strexTypeDouble)
-    return strexTypeDouble;
-else if (left == strexTypeInt || right == strexTypeInt)
-    return strexTypeInt;
-else if (left == strexTypeBoolean || right == strexTypeBoolean)
-    return strexTypeBoolean;
-else
-    {
-    errAbort("Can't find commonTypeForBop");
-    return strexTypeString;
-    }
-}
-
-
-static enum strexOp booleanCastOp(enum strexType oldType)
-/* Return op to convert oldType to boolean. */
-{
-switch (oldType)
-    {
-    case strexTypeString:
-        return strexOpStringToBoolean;
-    case strexTypeInt:
-        return strexOpIntToBoolean;
-    case strexTypeDouble:
-        return strexOpDoubleToBoolean;
-    default:
-        internalErr();
-	return strexOpUnknown;
-    }
-}
-
-static enum strexOp intCastOp(enum strexType oldType)
-/* Return op to convert oldType to int. */
-{
-switch (oldType)
-    {
-    case strexTypeString:
-        return strexOpStringToInt;
-    case strexTypeBoolean:
-        return strexOpBooleanToInt;
-    case strexTypeDouble:
-        return strexOpDoubleToInt;
-    default:
-        internalErr();
-	return strexOpUnknown;
-    }
-}
-
-static enum strexOp doubleCastOp(enum strexType oldType)
-/* Return op to convert oldType to double. */
-{
-switch (oldType)
-    {
-    case strexTypeString:
-        return strexOpStringToDouble;
-    case strexTypeBoolean:
-        return strexOpBooleanToDouble;
-    case strexTypeInt:
-        return strexOpIntToDouble;
-    default:
-        internalErr();
-	return strexOpUnknown;
-    }
-}
-
-static enum strexOp stringCastOp(enum strexType oldType)
-/* Return op to convert oldType to double. */
-{
-switch (oldType)
-    {
-    case strexTypeDouble:
-        return strexOpDoubleToString;
-    case strexTypeBoolean:
-        return strexOpBooleanToString;
-    case strexTypeInt:
-        return strexOpIntToString;
-    default:
-        internalErr();
-	return strexOpUnknown;
-    }
-}
-
-
-
-static struct strexParse *strexParseCoerce(struct strexParse *p, enum strexType type)
-/* If p is not of correct type, wrap type conversion node around it. */
-{
-if (p->type == type)
-    return p;
-else
-    {
-    struct strexParse *cast;
-    AllocVar(cast);
-    cast->children = p;
-    cast->type = type;
-    switch (type)
-        {
-	case strexTypeString:
-	    cast->op = stringCastOp(p->type);
-	    break;
-	case strexTypeBoolean:
-	    cast->op = booleanCastOp(p->type);
-	    break;
-	case strexTypeInt:
-	    cast->op = intCastOp(p->type);
-	    break;
-	case strexTypeDouble:
-	    cast->op = doubleCastOp(p->type);
-	    break;
-	default:
-	    internalErr();
-	    break;
-	}
-    return cast;
-    }
-}
-
-static struct strexParse *strexParseFunction(struct strexIn *in)
-/* Handle the (a,b,c) in funcCall(a,b,c).  Convert it into tree:
-*         strexOpBuiltInCall
-*            strexParse(a)
-*            strexParse(b)
-*            strexParse(c)
-* or something like that.  With no parameters 
-*            strexParseFunction */
-{
-struct tokenizer *tkz = in->tkz;
-struct strexParse *function = strexParseAtom(in);
-char *tok = tokenizerNext(tkz);
-if (tok == NULL)
-    tokenizerReuse(tkz);
-else if (tok[0] == '(')
-    {
-    /* Check that the current op, is a pure symbol. */
-    if (function->op != strexOpSymbol)
-        errAbort("Unexpected '(' line %d of %s", tkz->lf->lineIx, tkz->lf->fileName);
-
-    /* Look up function to call and complain if it doesn't exist */
-    char *functionName = function->val.s;
-    struct strexBuiltIn *builtIn = hashFindVal(in->builtInHash, functionName);
-    if (builtIn == NULL)
-        errAbort("No built in function %s exists line %d of %s", functionName, tkz->lf->lineIx,
-	    tkz->lf->fileName);
-
-    /* We're going to reuse this current op */
-    function->op = strexOpBuiltInCall;
-    function->type = strexTypeString;
-    function->val.builtIn = builtIn;
-
-    tok = tokenizerMustHaveNext(tkz);
-    if (tok[0] != ')')
-        {
-	tokenizerReuse(tkz);
-	for (;;)
-	    {
-	    struct strexParse *param = strexParseExpression(in);
-	    slAddHead(&function->children, param);
-	    tok = tokenizerMustHaveNext(tkz);
-	    if (tok[0] == ')')
-	        break;
-	    else if (tok[0] != ',')
-	        errAbort("Error in parameter list for %s line %d of %s", function->val.s, 
-		    tkz->lf->lineIx, tkz->lf->fileName);
-	    }
-	slReverse(&function->children);
-	}
-
-    /* Check function parameter count */
-    int childCount = slCount(function->children);
-    if (childCount != builtIn->paramCount)
-        errAbort("Function %s has %d parameters but needs %d line %d of %s",
-	    builtIn->name, childCount, builtIn->paramCount, tkz->lf->lineIx, tkz->lf->fileName);
-	    
-    /* Check function parameter types */
-    int i;
-    struct strexParse *p;
-    for (i=0, p=function->children; i<childCount; ++i, p = p->next)
-        {
-	if (p->type != builtIn->paramTypes[i])
-	    {
-	    errAbort("Parameter #%d to %s needs to be type %s not %s line %d of %s",
-		i, builtIn->name,  strexTypeToString(builtIn->paramTypes[i]), 
-		strexTypeToString(p->type), tkz->lf->lineIx, tkz->lf->fileName);
-	    }
-	}
-    }
-else
-    tokenizerReuse(tkz);
-return function;
-}
-
-static struct strexParse *strexParseIndex(struct strexIn *in)
-/* Handle the [] in this[6].  Convert it into tree:
-*         strexOpArrayIx
-*            strexParseFunction
-*            strexParseFunction */
-{
-struct tokenizer *tkz = in->tkz;
-struct strexParse *collection = strexParseFunction(in);
-struct strexParse *p = collection;
-char *tok = tokenizerNext(tkz);
-if (tok == NULL)
-    tokenizerReuse(tkz);
-else if (tok[0] == '[')
-    {
-    struct strexParse *index = strexParseExpression(in);
-    // struct strexParse *index = strexParseFunction(in);
-    index = strexParseCoerce(index, strexTypeInt);
-    skipOverRequired(in, "]");
-    AllocVar(p);
-    p->op = strexOpArrayIx;
-    p->type = strexTypeString;
-    p->children = collection;
-    p->val.s = cloneString("");
-    collection->next = index;
-    }
-else
-    tokenizerReuse(tkz);
-return p;
-}
-
-
-static struct strexParse *strexParseUnaryMinus(struct strexIn *in)
-/* Return unary minus sort of parse tree if there's a leading '-' */
-{
-struct tokenizer *tkz = in->tkz;
-char *tok = tokenizerMustHaveNext(tkz);
-if (tok[0] == '-')
-    {
-    struct strexParse *c = strexParseIndex(in);
-    struct strexParse *p;
-    AllocVar(p);
-    if (c->type == strexTypeInt)
-        {
-	p->op = strexOpUnaryMinusInt;
-	p->type = strexTypeInt;
-	}
-    else
-	{
-	c = strexParseCoerce(c, strexTypeDouble);
-	p->op = strexOpUnaryMinusDouble;
-	p->type = strexTypeDouble;
-	}
-    p->children = c;
-    return p;
-    }
-else
-    {
-    tokenizerReuse(tkz);
-    return strexParseIndex(in);
-    }
-}
-
-static struct strexParse *strexParseSum(struct strexIn *in)
-/* Parse out plus or minus. */
-{
-struct tokenizer *tkz = in->tkz;
-struct strexParse *p = strexParseUnaryMinus(in);
-for (;;)
-    {
-    char *tok = tokenizerNext(tkz);
-    if (tok == NULL || differentString(tok, "+"))
-	{
-	tokenizerReuse(tkz);
-	return p;
-	}
-
-    /* What we've parsed so far becomes left side of binary op, next term ends up on right. */
-    struct strexParse *l = p;
-    struct strexParse *r = strexParseUnaryMinus(in);
-
-    /* Make left and right side into a common type */
-    enum strexType childType = commonTypeForBop(l->type, r->type);
-    l = strexParseCoerce(l, childType);
-    r = strexParseCoerce(r, childType);
-
-    /* Create the binary operation */
-    AllocVar(p);
-    p->op = strexOpAdd;
-    p->type = childType;
-
-    /* Now hang children onto node. */
-    p->children = l;
-    l->next = r;
-    }
-}
-
-
-struct strexParse *strexParseExpression(struct strexIn *in)
-/* Parse out an expression. Leaves input at next expression. */
-{
-return strexParseSum(in);
-}
-
-/* ~~~ start of strexEval */
-
-static struct strexEval strexLocalEval(struct strexParse *p, void *record, StrexEvalLookup lookup, 
-	struct lm *lm);
-/* Evaluate self on parse tree, allocating memory if needed from lm. */
-
-
-struct strexEval strexEvalCoerceToString(struct strexEval r, char *buf, int bufSize)
-/* Return a version of r with .val.s filled in with something reasonable even
- * if r input is not a string */
-{
-assert(bufSize >= 32);
-switch (r.type)
-    {
-    case strexTypeBoolean:
-        r.val.s = (r.val.b ? "true" : "false");
-    case strexTypeString:
-	break;	/* It's already done. */
-    case strexTypeInt:
-	safef(buf, bufSize, "%lld", r.val.i);
-	r.val.s = buf;
-	break;
-    case strexTypeDouble:
-	safef(buf, bufSize, "%g", r.val.x);
-	r.val.s = buf;
-	break;
-    default:
-	internalErr();
-	r.val.s = NULL;
-	break;
-    }
-r.type = strexTypeString;
-return r;
-}
-
-static struct strexEval strexEvalAdd(struct strexParse *p, void *record, StrexEvalLookup lookup,
-	struct lm *lm)
-/* Return a + b. */
-{
-struct strexParse *lp = p->children;
-struct strexParse *rp = lp->next;
-struct strexEval lv = strexLocalEval(lp, record, lookup, lm);
-struct strexEval rv = strexLocalEval(rp, record, lookup, lm);
-struct strexEval res;
-switch (lv.type)
-    {
-    case strexTypeInt:
-	res.val.i = (lv.val.i + rv.val.i);
-	break;
-    case strexTypeDouble:
-	res.val.x = (lv.val.x + rv.val.x);
-	break;
-    case strexTypeString:
-	{
-	char numBuf[32];
-	if (rv.type != strexTypeString)  // Perhaps later could coerce to string
-	    {
-	    rv = strexEvalCoerceToString(rv, numBuf, sizeof(numBuf));
-	    }
-	int lLen = strlen(lv.val.s);
-	int rLen = strlen(rv.val.s);
-	char *s = lmAlloc(lm, lLen + rLen + 1);
-	memcpy(s, lv.val.s, lLen);
-	memcpy(s+lLen, rv.val.s, rLen);
-	res.val.s = s;
-	break;
-	}
-    default:
-	internalErr();
-	res.val.b = FALSE;
-	break;
-    }
-res.type = lv.type;
-return res;
-}
-
-
-char *csvParseOneOut(char *csvIn, int ix, struct dyString *scratch)
-/* Return csv value of given index or NULL if at end */
-{
-char *pos = csvIn;
-int i;
-for (i=0; i<ix; ++i)
-    {
-    if (csvParseNext(&pos,scratch) == NULL)
-        return NULL;
-    }
-return csvParseNext(&pos, scratch);
-}
-
-static struct strexEval strexEvalArrayIx(struct strexParse *p, void *record, StrexEvalLookup lookup,
-	struct lm *lm)
-/* Handle parse tree generated by an indexed array. */
-{
-struct strexParse *array = p->children;
-struct strexParse *index = array->next;
-struct strexEval arrayVal = strexLocalEval(array, record, lookup, lm);
-struct strexEval indexVal = strexLocalEval(index, record, lookup, lm);
-struct strexEval res;
-struct dyString *scratch = dyStringNew(0);
-char *val = emptyForNull(csvParseOneOut(arrayVal.val.s, indexVal.val.i, scratch));
-res.val.s = cloneString(val);
-res.type = strexTypeString;
-return res;
-}
-
-static char *wordInString(char *words,  int ix,  struct lm *lm)
-/* Return the word delimited string of index ix as clone into lm */
-{
-char *s = words;
-int i;
-for (i=0; ; ++i)
-    {
-    s = skipLeadingSpaces(s);
-    if (isEmpty(s))
-        errAbort("There aren't %d words in %s", ix+1, words);
-    char *end = skipToSpaces(s);
-    if (i == ix)
-        {
-	if (end == NULL)
-	    return lmCloneString(lm, s);
-	else
-	    return lmCloneMem(lm, s, end - s);
-	}
-    s = end;
-    }
-}
-
-static char *splitString(char *string, char *splitter, int ix, struct lm *lm)
-/* Return the ix'th part of string as split apart by splitter */
-{
-int splitterSize = strlen(splitter);
-if (splitterSize != 1)
-    errAbort("Separator parameter to split must be a single character, not %s", splitter);
-int count = chopByChar(string, splitter[0], NULL, 0);
-if (ix >= count)
-    errAbort("There aren't %d fields separated by %s in %s", ix+1, splitter, string);
-char **row;
-lmAllocArray(lm, row, count);
-char *scratch = lmCloneString(lm, string);
-chopByChar(scratch, splitter[0], row, count);
-return row[ix];
-}
-
-static struct strexEval strexEvalCallBuiltIn(struct strexParse *p, 
-    void *record, StrexEvalLookup lookup, struct lm *lm)
-/* Handle parse tree generated by an indexed array. */
-{
-struct strexBuiltIn *builtIn = p->val.builtIn;
-struct strexEval res;
-res.type = strexTypeString;
-switch (builtIn->func)
-    {
-    case strexBuiltInTrim:
-	{
-        struct strexEval a = strexLocalEval(p->children, record, lookup, lm);
-	res.val.s = trimSpaces(a.val.s);
-	break;
-	}
-    case strexBuiltInBetween:
-	{
-        struct strexEval a = strexLocalEval(p->children, record, lookup, lm);
-        struct strexEval b = strexLocalEval(p->children->next, record, lookup, lm);
-        struct strexEval c = strexLocalEval(p->children->next->next, record, lookup, lm);
-	char *between = stringBetween(a.val.s, c.val.s, b.val.s);
-	res.val.s = lmCloneString(lm, between);
-	freeMem(between);
-        break;
-	}
-    case strexBuiltInSpaced:
-        {
-        struct strexEval a = strexLocalEval(p->children, record, lookup, lm);
-        struct strexEval b = strexLocalEval(p->children->next, record, lookup, lm);
-	res.val.s = wordInString(a.val.s, b.val.i, lm);
-	break;
-	}
-    case strexBuiltInNow:
-        {
-	time_t now = time(NULL);
-	res.val.s = lmCloneString(lm, ctime(&now));
-	eraseTrailingSpaces(res.val.s);
-	break;
-	}
-    case strexBuiltInMd5:
-        {
-        struct strexEval a = strexLocalEval(p->children, record, lookup, lm);
-	char *md5 = hmacMd5("", a.val.s);
-	res.val.s = lmCloneString(lm, md5);
-	freez(&md5);
-	break;
-	}
-    case strexBuiltInSplit:
-        {
-        struct strexEval a = strexLocalEval(p->children, record, lookup, lm);
-        struct strexEval b = strexLocalEval(p->children->next, record, lookup, lm);
-        struct strexEval c = strexLocalEval(p->children->next->next, record, lookup, lm);
-	res.val.s = splitString(a.val.s, b.val.s, c.val.i, lm);
-	break;
-	}
-    }
-return res;
-}
-
-
-/* ~~~ evaluation part of strex */
-
-static struct strexEval strexLocalEval(struct strexParse *p, void *record, StrexEvalLookup lookup, 
-	struct lm *lm)
-/* Evaluate self on parse tree, allocating memory if needed from lm. */
-{
-struct strexEval res;
-switch (p->op)
-    {
-    case strexOpLiteral:
-	res.val = p->val;
-	res.type = p->type;
-	break;
-    case strexOpSymbol:
-	res.type = strexTypeString;
-	char *s = lookup(record, p->val.s);
-	if (s == NULL)
-	    res.val.s = "";
-	else
-	    res.val.s = s;
-	break;
-
-    /* Type casts. */
-    case strexOpStringToBoolean:
-	res = strexLocalEval(p->children, record, lookup, lm);
-	res.type = strexTypeBoolean;
-	res.val.b = (res.val.s[0] != 0);
-	break;
-    case strexOpIntToBoolean:
-        res = strexLocalEval(p->children, record, lookup, lm);
-	res.type = strexTypeBoolean;
-	res.val.b = (res.val.i != 0);
-	break;
-    case strexOpDoubleToBoolean:
-        res = strexLocalEval(p->children, record, lookup, lm);
-	res.type = strexTypeBoolean;
-	res.val.b = (res.val.x != 0.0);
-	break;
-    case strexOpStringToInt:
-	res = strexLocalEval(p->children, record, lookup, lm);
-	res.type = strexTypeInt;
-	res.val.i = atoll(res.val.s);
-	break;
-    case strexOpDoubleToInt:
-	res = strexLocalEval(p->children, record, lookup, lm);
-	res.type = strexTypeInt;
-	res.val.i = res.val.x;
-	break;
-
-    case strexOpStringToDouble:
-	res = strexLocalEval(p->children, record, lookup, lm);
-	res.type = strexTypeDouble;
-	res.val.x = atof(res.val.s);
-	break;
-    case strexOpBooleanToInt:
-	res = strexLocalEval(p->children, record, lookup, lm);
-	res.type = strexTypeInt;
-	res.val.i = res.val.b;
-	break;
-    case strexOpBooleanToDouble:
-	res = strexLocalEval(p->children, record, lookup, lm);
-	res.type = strexTypeDouble;
-	res.val.x = res.val.b;
-	break;
-    case strexOpIntToDouble:
-	res = strexLocalEval(p->children, record, lookup, lm);
-	res.type = strexTypeDouble;
-	res.val.x = res.val.b;
-	break;
-
-    case strexOpIntToString:
-        {
-	res = strexLocalEval(p->children, record, lookup, lm);
-	res.type = strexTypeString;
-	char buf[32];
-	safef(buf, sizeof(buf), "%lld", res.val.i);
-	res.val.s = lmCloneString(lm, buf);
-	break;
-	}
-    case strexOpDoubleToString:
-        {
-	res = strexLocalEval(p->children, record, lookup, lm);
-	res.type = strexTypeString;
-	char buf[32];
-	safef(buf, sizeof(buf), "%g", res.val.x);
-	res.val.s = lmCloneString(lm, buf);
-	break;
-	}
-    case strexOpBooleanToString:
-	res = strexLocalEval(p->children, record, lookup, lm);
-	res.type = strexTypeString;
-        res.val.s = (res.val.b ? "true" : "false");
-	break;
-
-    /* Arithmetical negation. */
-    case strexOpUnaryMinusInt:
-        res = strexLocalEval(p->children, record, lookup, lm);
-	res.val.i = -res.val.i;
-	break;
-    case strexOpUnaryMinusDouble:
-        res = strexLocalEval(p->children, record, lookup, lm);
-	res.val.x = -res.val.x;
-	break;
-
-    case strexOpArrayIx:
-       res = strexEvalArrayIx(p, record, lookup, lm);
-       break;
-
-    case strexOpBuiltInCall:
-       res = strexEvalCallBuiltIn(p, record, lookup, lm);
-       break;
-
-    /* Mathematical ops, simple binary type */
-    case strexOpAdd:
-       res = strexEvalAdd(p, record, lookup, lm);
-       break;
-
-    default:
-        errAbort("Unknown op %s\n", strexOpToString(p->op));
-	res.type = strexTypeInt;	// Keep compiler from complaining.
-	res.val.i = 0;	// Keep compiler from complaining.
-	break;
-    }
-return res;
-}
-
-char *strexEvalAsString(struct strexParse *p, void *record, StrexEvalLookup lookup)
-/* Evaluating a strex expression on a symbol table with a lookup function for variables and
- * return result as a string value. */
-{
-struct lm *lm = lmInit(0);
-struct strexEval res = strexLocalEval(p, record, lookup, lm);
-char numBuf[32];
-struct strexEval strRes = strexEvalCoerceToString(res, numBuf, sizeof(numBuf));
-char *ret = cloneString(strRes.val.s);
-lmCleanup(&lm);
-return ret;
-}
-
-static struct strexEval strexLocalEval(struct strexParse *p, void *record, StrexEvalLookup lookup, 
-	struct lm *lm);
-/* Evaluate self on parse tree, allocating memory if needed from lm. */
-
-/* ~~~ end of strex */
-
 struct hash *hashFromFile(char *fileName)
 /* Given a two column file (key, value) return a hash. */
 {
 struct lineFile *lf = lineFileOpen(fileName, TRUE);
 struct hash *hash = hashNew(16);
 char *line;
 while (lineFileNextReal(lf, &line))
     {
     char *word = nextWord(&line);
     line = trimSpaces(line);
     hashAdd(hash, word, lmCloneString(hash->lm, line));
     }
 lineFileClose(&lf);
 return hash;
 }
 
 static char *symLookup(void *record, char *key)
 /* Lookup symbol in hash */
 {
 struct hash *hash = record;
 return hashFindVal(hash, key);
 }
 
-static void ensureAtEnd(struct strexIn *in)
-/* Make sure that we are at end of input. */
-{
-struct tokenizer *tkz = in->tkz;
-char *leftover = tokenizerNext(tkz);
-if (leftover != NULL)
-    errAbort("Extra input starting with '%s' line %d of %s", leftover, tkz->lf->lineIx,
-	tkz->lf->fileName);
-}
-
-struct strexParse *strexParseString(char *s)
-/* Parse out string expression in s and return root of tree. */
-{
-struct strexIn *si = strexInNew(s);
-struct strexParse *parseTree = strexParseExpression(si);
-ensureAtEnd(si);
-strexInFree(&si);
-return parseTree;
-}
 
 void freen(char *symbols, char *expression)
 /* Test something */
 {
 struct hash *symHash = hashFromFile(symbols);
 verbose(1, "Got %d symbols from %s\n", symHash->elCount, symbols);
 uglyf("Parsing...\n");
 struct strexParse *parseTree = strexParseString(expression);
 strexParseDump(parseTree, 0, uglyOut);
 uglyf("Evaluating...\n");
 uglyf("%s\n", strexEvalAsString(parseTree, symHash, symLookup));
 }
 
 
 int main(int argc, char *argv[])
 /* Process command line. */
 {
 optionInit(&argc, argv, options);
 if (argc != 3)
     usage();
 freen(argv[1], argv[2]);
 return 0;
 }