c622b3f8d3959b7f3eb53f3f4aec9b9e61c76989 kent Thu Aug 15 12:03:17 2019 -0700 Added strexParseFile function, so can parse files as well as strings. Also added import operator, which is going to be so helpful for debugging and reusing complicated expressions. diff --git src/lib/strex.c src/lib/strex.c index 36d6a71..577c415 100644 --- src/lib/strex.c +++ src/lib/strex.c @@ -141,30 +141,31 @@ { struct strexParse *next; /* Points to younger sibling if any. */ struct strexParse *children; /* Points to oldest child if any. */ enum strexOp op; /* Operation at this node. */ enum strexType type; /* Return type of this operation. */ union strexVal val; /* Return value of this operation. */ }; struct strexIn /* Input to the strex parser - tokenizer and a hash full of built in functions. */ { struct tokenizer *tkz; /* Get next text input from here */ struct hash *builtInHash; /* Hash of built in functions */ void *symbols; /* NULL or pointer to a symbol table to check */ StrexLookup lookup; /* lookup something in symbol table if we have it */ + struct hash *importHash; /* Hash of importex expressions keyed by file name */ }; /* Some predefined lists of parameter types */ static enum strexType oneString[] = {strexTypeString}; static enum strexType twoStrings[] = {strexTypeString, strexTypeString}; static enum strexType threeStrings[] = {strexTypeString, strexTypeString, strexTypeString}; static enum strexType stringInt[] = {strexTypeString, strexTypeInt}; static enum strexType stringStringInt[] = {strexTypeString, strexTypeString, strexTypeInt}; /* There's one element here for each built in function. There's also a few switches you'll need to * fill in if you add a new built in function. */ static struct strexBuiltIn builtins[] = { { "trim", strexBuiltInTrim, strexTypeString, 1, oneString, }, { "between", strexBuiltInBetween, strexTypeString, 3, threeStrings }, { "split", strexBuiltInSplit, strexTypeString, 2, stringInt }, @@ -184,54 +185,54 @@ { "starts", strexBuiltInStarts, strexTypeBoolean, 2, twoStrings}, { "ends", strexBuiltInEnds, strexTypeBoolean, 2, twoStrings}, { "same", strexBuiltInSame, strexTypeBoolean, 2, twoStrings}, }; static struct hash *hashBuiltIns() /* Build a hash of builtins keyed by name */ { struct hash *hash = hashNew(0); int i; for (i=0; ilineIx = fileLineNumber; struct tokenizer *tkz = tokenizerOnLineFile(lf); tkz->leaveQuotes = TRUE; struct strexIn *si; AllocVar(si); si->tkz = tkz; si->builtInHash = hashBuiltIns(); +si->importHash = hashNew(4); si->symbols = symbols; si->lookup = lookup; return si; } static void strexInFree(struct strexIn **pSi) /* Free up memory associated with strexIn structure */ { struct strexIn *si = *pSi; if (si != NULL) { hashFree(&si->builtInHash); + hashFree(&si->importHash); tokenizerFree(&si->tkz); freez(pSi); } } struct strexParse *strexParseNew(enum strexOp op, enum strexType type) /* Return a fresh strexParse of the given op and type with the val set to 0/NULL */ { struct strexParse *p; AllocVar(p); p->op = op; p->type = type; return p; } @@ -362,30 +363,42 @@ fprintf(f, "%s ", strexOpToString(p->op)); strexParseValDump(p, f); fprintf(f, "\n"); struct strexParse *child; for (child = p->children; child != NULL; child= child->next) strexParseDump(child, depth+1, f); } static void expectingGot(struct strexIn *in, char *expecting, char *got) /* Print out error message about unexpected input. */ { errAbort("Expecting %s, got %s, line %d of %s", expecting, got, in->tkz->lf->lineIx, in->tkz->lf->fileName); } +static void strexParseFree(struct strexParse **pParse) +/* Free up memory resources associeated with a strexParse */ +{ +struct strexParse *p = *pParse; +if (p != NULL) + { + if (p->type == strexTypeString) + freeMem(p->val.s); + freez(pParse); + } +} + static void skipOverRequired(struct strexIn *in, char *expecting) /* Make sure that next token is tok, and skip over it. */ { tokenizerMustHaveNext(in->tkz); if (!sameWord(in->tkz->string, expecting)) expectingGot(in, expecting, in->tkz->string); } static struct strexParse *strexParseExpression(struct strexIn *in); /* Parse out an expression with a single value */ static struct strexParse *strexParseOr(struct strexIn *in); /* Parse out logical/string or binary operator. Lowest level logical op, before conditional */ @@ -722,30 +735,58 @@ if (defaultVal == NULL) { /* Need to make up empty default */ defaultVal = strexParseNew(strexOpLiteral, firstVal->type); defaultVal->val = strexValEmptyForType(firstVal->type); } slAddHead(&function->children, defaultVal); /* Finally put the key expression at the very head */ slAddHead(&function->children, keyExp); /* Going to reuse current op, turn it into pick */ function->op = strexOpPick; function->type = firstVal->type; } + else if sameString(functionName, "import") + { + /* We save away the current op for now. The function variable is where we'll + * return the expression we import. */ + struct strexParse *importer = function; + function = NULL; + + /* Parse out the file name. We'll insist it's a constant string */ + struct strexParse *fileExp = strexParseAtom(in); + if (fileExp->op != strexOpLiteral || fileExp->type != strexTypeString) + errAbort("Paramater to import needs to be a quoted file name line %d of %s", + tkz->lf->lineIx, tkz->lf->fileName); + char *fileName = fileExp->val.s; + + /* Look up imported parse tree in hash, reading it from file if need be. */ + function = hashFindVal(in->importHash, fileName); + if (function == NULL) + { + function = strexParseFile(fileName, in->symbols, in->lookup); + hashAdd(in->importHash, fileName, function); + } + + skipOverRequired(in, ")"); + + /* Clean up */ + strexParseFree(&fileExp); + strexParseFree(&importer); + } else { /* It's a builtin function as opposed to a special op. Figure out which one.*/ struct strexBuiltIn *builtIn = hashFindVal(in->builtInHash, functionName); if (builtIn == NULL) errAbort("No built in function %s exists line %d of %s", functionName, tkz->lf->lineIx, tkz->lf->fileName); /* We're going to reuse this current op */ function->op = strexOpBuiltInCall; function->type = builtIn->returnType; function->val.builtIn = builtIn; tok = tokenizerMustHaveNext(tkz); if (tok[0] != ')') @@ -1047,33 +1088,48 @@ return strexParseConditional(in); } static void ensureAtEnd(struct strexIn *in) /* Make sure that we are at end of input. */ { struct tokenizer *tkz = in->tkz; char *leftover = tokenizerNext(tkz); if (leftover != NULL) errAbort("Extra input starting with '%s' line %d of %s", leftover, tkz->lf->lineIx, tkz->lf->fileName); } struct strexParse *strexParseString(char *s, char *fileName, int fileLineNumber, void *symbols, StrexLookup lookup) -/* Parse out string expression in s and return root of tree. */ +/* Parse out string expression in s and return root of tree. The fileName and + * fileLineNumber should be filled in with where string came from. */ +{ +struct lineFile *lf = lineFileOnString(fileName, TRUE, s); +lf->lineIx = fileLineNumber; +struct strexIn *si = strexInNew(lf, symbols, lookup); +struct strexParse *parseTree = strexParseExpression(si); +ensureAtEnd(si); +strexInFree(&si); +return parseTree; +} + +struct strexParse *strexParseFile(char *fileName, void *symbols, StrexLookup lookup) +/* Parse string expression out of a file */ { -struct strexIn *si = strexInNew(s, fileName, fileLineNumber, symbols, lookup); +struct lineFile *lf = lineFileOpen(fileName, TRUE); +struct strexIn *si = strexInNew(lf, symbols, lookup); +si->tkz->uncommentShell = TRUE; // Yay to comments. struct strexParse *parseTree = strexParseExpression(si); ensureAtEnd(si); strexInFree(&si); return parseTree; } /************ The parsing section is done, now for the evaluation section. **************/ static struct strexEval strexLocalEval(struct strexParse *p, void *record, StrexLookup lookup, struct lm *lm); /* Evaluate self on parse tree, allocating memory if needed from lm. */ static struct strexEval strexEvalCoerceToString(struct strexEval r, char *buf, int bufSize) /* Return a version of r with .val.s filled in with something reasonable even