c622b3f8d3959b7f3eb53f3f4aec9b9e61c76989
kent
  Thu Aug 15 12:03:17 2019 -0700
Added strexParseFile function, so can parse files as well as strings.  Also added import operator, which is going to be so helpful for debugging and reusing complicated expressions.

diff --git src/lib/strex.c src/lib/strex.c
index 36d6a71..577c415 100644
--- src/lib/strex.c
+++ src/lib/strex.c
@@ -141,30 +141,31 @@
     {
     struct strexParse *next;	/* Points to younger sibling if any. */
     struct strexParse *children;	/* Points to oldest child if any. */
     enum strexOp op;		/* Operation at this node. */
     enum strexType type;		/* Return type of this operation. */
     union strexVal val;		/* Return value of this operation. */
     };
 
 struct strexIn
 /* Input to the strex parser - tokenizer and a hash full of built in functions. */
     {
     struct tokenizer *tkz;  /* Get next text input from here */
     struct hash *builtInHash;  /* Hash of built in functions */
     void *symbols;	    /* NULL or pointer to a symbol table to check */
     StrexLookup lookup; /* lookup something in symbol table if we have it */
+    struct hash *importHash;   /* Hash of importex expressions keyed by file name */
     };
 
 /* Some predefined lists of parameter types */
 static enum strexType oneString[] = {strexTypeString};
 static enum strexType twoStrings[] = {strexTypeString, strexTypeString};
 static enum strexType threeStrings[] = {strexTypeString, strexTypeString, strexTypeString};
 static enum strexType stringInt[] = {strexTypeString, strexTypeInt};
 static enum strexType stringStringInt[] = {strexTypeString, strexTypeString, strexTypeInt};
 
 /* There's one element here for each built in function.  There's also a few switches you'll need to
  * fill in if you add a new built in function. */
 static struct strexBuiltIn builtins[] = {
     { "trim", strexBuiltInTrim, strexTypeString, 1, oneString, },
     { "between", strexBuiltInBetween, strexTypeString, 3, threeStrings },
     { "split", strexBuiltInSplit, strexTypeString, 2, stringInt },
@@ -184,54 +185,54 @@
     { "starts", strexBuiltInStarts, strexTypeBoolean, 2, twoStrings}, 
     { "ends", strexBuiltInEnds, strexTypeBoolean, 2, twoStrings}, 
     { "same", strexBuiltInSame, strexTypeBoolean, 2, twoStrings}, 
 };
 
 static struct hash *hashBuiltIns()
 /* Build a hash of builtins keyed by name */
 {
 struct hash *hash = hashNew(0);
 int i;
 for (i=0; i<ArraySize(builtins); ++i)
     hashAdd(hash, builtins[i].name, &builtins[i]);
 return hash;
 }
 
-static struct strexIn *strexInNew(char *expression, char *fileName, int fileLineNumber,
+static struct strexIn *strexInNew(struct lineFile *lf,
     void *symbols, StrexLookup lookup)
-/* Return a new strexIn structure wrapped around expression */
+/* Return a new strexIn structure wrapped around lineFile */
 {
-struct lineFile *lf = lineFileOnString(fileName, TRUE, expression);
-lf->lineIx = fileLineNumber;
 struct tokenizer *tkz = tokenizerOnLineFile(lf);
 tkz->leaveQuotes = TRUE;
 struct strexIn *si;
 AllocVar(si);
 si->tkz = tkz;
 si->builtInHash = hashBuiltIns();
+si->importHash = hashNew(4);
 si->symbols = symbols;
 si->lookup = lookup;
 return si;
 }
 
 static void strexInFree(struct strexIn **pSi)
 /* Free up memory associated with strexIn structure */
 {
 struct strexIn *si = *pSi;
 if (si != NULL)
     {
     hashFree(&si->builtInHash);
+    hashFree(&si->importHash);
     tokenizerFree(&si->tkz);
     freez(pSi);
     }
 }
 
 struct strexParse *strexParseNew(enum strexOp op, enum strexType type)
 /* Return a fresh strexParse of the given op and type with the val set to 0/NULL */
 {
 struct strexParse *p;
 AllocVar(p);
 p->op = op;
 p->type = type;
 return p;
 }
 
@@ -362,30 +363,42 @@
 fprintf(f, "%s ", strexOpToString(p->op));
 strexParseValDump(p,  f);
 fprintf(f, "\n");
 struct strexParse *child;
 for (child = p->children; child != NULL; child= child->next)
     strexParseDump(child, depth+1, f);
 }
 
 static void expectingGot(struct strexIn *in, char *expecting, char *got)
 /* Print out error message about unexpected input. */
 {
 errAbort("Expecting %s, got %s, line %d of %s", expecting, got, in->tkz->lf->lineIx,
 	in->tkz->lf->fileName);
 }
 
+static void strexParseFree(struct strexParse **pParse)
+/* Free up memory resources associeated with a strexParse */
+{
+struct strexParse *p = *pParse;
+if (p != NULL)
+    {
+    if (p->type == strexTypeString)
+	freeMem(p->val.s);
+    freez(pParse);
+    }
+}
+
 static void skipOverRequired(struct strexIn *in, char *expecting)
 /* Make sure that next token is tok, and skip over it. */
 {
 tokenizerMustHaveNext(in->tkz);
 if (!sameWord(in->tkz->string, expecting))
     expectingGot(in, expecting, in->tkz->string);
 }
 
 
 static struct strexParse *strexParseExpression(struct strexIn *in);
 /* Parse out an expression with a single value */
 
 static struct strexParse *strexParseOr(struct strexIn *in);
 /* Parse out logical/string or binary operator. Lowest level logical op, before conditional */
 
@@ -722,30 +735,58 @@
 	if (defaultVal == NULL)
 	    {
 	    /* Need to make up empty default */
 	    defaultVal = strexParseNew(strexOpLiteral, firstVal->type);
 	    defaultVal->val = strexValEmptyForType(firstVal->type);
 	    }
 	slAddHead(&function->children, defaultVal);
 
 	/* Finally put the key expression at the very head */
 	slAddHead(&function->children, keyExp);
 
 	/* Going to reuse current op, turn it into pick */
 	function->op = strexOpPick;
 	function->type = firstVal->type;
 	}
+    else if sameString(functionName, "import")
+        {
+	/* We save away the current op for now.  The function variable is where we'll
+	 * return the expression we import. */
+	struct strexParse *importer = function;
+	function = NULL;
+
+	/* Parse out the file name.  We'll insist it's a constant string */
+	struct strexParse *fileExp = strexParseAtom(in);
+	if (fileExp->op != strexOpLiteral || fileExp->type != strexTypeString)
+	   errAbort("Paramater to import needs to be a quoted file name line %d of %s",
+	       tkz->lf->lineIx,  tkz->lf->fileName);
+	char *fileName = fileExp->val.s;
+	
+	/* Look up imported parse tree in hash,  reading it from file if need be. */
+	function = hashFindVal(in->importHash, fileName);
+	if (function == NULL)
+	    {
+	    function = strexParseFile(fileName, in->symbols, in->lookup);
+	    hashAdd(in->importHash, fileName, function);
+	    }
+
+	skipOverRequired(in, ")");
+
+	/* Clean up */
+	strexParseFree(&fileExp);
+	strexParseFree(&importer);
+	}
     else
 	{
 	/* It's a builtin function as opposed to a special op.  Figure out which one.*/
 	struct strexBuiltIn *builtIn = hashFindVal(in->builtInHash, functionName);
 	if (builtIn == NULL)
 	    errAbort("No built in function %s exists line %d of %s", functionName, tkz->lf->lineIx,
 		tkz->lf->fileName);
 
 	/* We're going to reuse this current op */
 	function->op = strexOpBuiltInCall;
 	function->type = builtIn->returnType;
 	function->val.builtIn = builtIn;
 
 	tok = tokenizerMustHaveNext(tkz);
 	if (tok[0] != ')')
@@ -1047,33 +1088,48 @@
 return strexParseConditional(in);
 }
 
 static void ensureAtEnd(struct strexIn *in)
 /* Make sure that we are at end of input. */
 {
 struct tokenizer *tkz = in->tkz;
 char *leftover = tokenizerNext(tkz);
 if (leftover != NULL)
     errAbort("Extra input starting with '%s' line %d of %s", leftover, tkz->lf->lineIx,
 	tkz->lf->fileName);
 }
 
 struct strexParse *strexParseString(char *s, char *fileName, int fileLineNumber,
     void *symbols, StrexLookup lookup)
-/* Parse out string expression in s and return root of tree. */
+/* Parse out string expression in s and return root of tree.  The fileName and
+ * fileLineNumber should be filled in with where string came from.  */
+{
+struct lineFile *lf = lineFileOnString(fileName, TRUE, s);
+lf->lineIx = fileLineNumber;
+struct strexIn *si = strexInNew(lf, symbols, lookup);
+struct strexParse *parseTree = strexParseExpression(si);
+ensureAtEnd(si);
+strexInFree(&si);
+return parseTree;
+}
+
+struct strexParse *strexParseFile(char *fileName, void *symbols, StrexLookup lookup)
+/* Parse string expression out of a file */
 {
-struct strexIn *si = strexInNew(s, fileName, fileLineNumber, symbols, lookup);
+struct lineFile *lf = lineFileOpen(fileName, TRUE);
+struct strexIn *si = strexInNew(lf, symbols, lookup);
+si->tkz->uncommentShell = TRUE;   // Yay to comments.
 struct strexParse *parseTree = strexParseExpression(si);
 ensureAtEnd(si);
 strexInFree(&si);
 return parseTree;
 }
 
 /************ The parsing section is done, now for the evaluation section. **************/
 
 static struct strexEval strexLocalEval(struct strexParse *p, void *record, StrexLookup lookup, 
 	struct lm *lm);
 /* Evaluate self on parse tree, allocating memory if needed from lm. */
 
 
 static struct strexEval strexEvalCoerceToString(struct strexEval r, char *buf, int bufSize)
 /* Return a version of r with .val.s filled in with something reasonable even