2d795fa03e0ef57637885b4ddf9d13832143799b
kent
  Fri Aug 16 11:55:57 2019 -0700
Adding tidy builtin function.

diff --git src/lib/strex.c src/lib/strex.c
index 41082c6..81c4c37 100644
--- src/lib/strex.c
+++ src/lib/strex.c
@@ -55,30 +55,31 @@
     strexBuiltInMd5,
     strexBuiltInSeparate,
     strexBuiltInUncsv,
     strexBuiltInUntsv,
     strexBuiltInReplace,
     strexBuiltInFix,
     strexBuiltInStrip,
     strexBuiltInLen,
     strexBuiltInSymbol,
     strexBuiltInLower,
     strexBuiltInUpper,
     strexBuiltInIn, 
     strexBuiltInStarts,
     strexBuiltInEnds,
     strexBuiltInSame,
+    strexBuiltInTidy,
     };
 
 struct strexBuiltIn
 /* Information to describe a built in function */
     {
     char *name;		/* Name in strex language:  trim, split, etc */
     enum strexBuiltInFunc func;  /* enum version: strexBuiltInTrim strexBuiltInSplit etc. */
     enum strexType returnType;	 /* Type of return value */
     int paramCount;	/* Number of parameters, not flexible in this language! */
     enum strexType *paramTypes;  /* Array of types, one for each parameter */
     };
 
 union strexVal
 /* Some value of arbirary type that can be of any type corresponding to strexType */
     {
@@ -173,30 +174,31 @@
     { "md5", strexBuiltInMd5, strexTypeString, 1, oneString },
     { "separate", strexBuiltInSeparate, strexTypeString, 3, stringStringInt },
     { "uncsv", strexBuiltInUncsv, strexTypeString, 2, stringInt },
     { "untsv", strexBuiltInUntsv, strexTypeString, 2, stringInt },
     { "replace", strexBuiltInReplace, strexTypeString, 3, threeStrings },
     { "fix", strexBuiltInFix, strexTypeString, 3, threeStrings },
     { "strip", strexBuiltInStrip, strexTypeString, 2, twoStrings },
     { "len", strexBuiltInLen, strexTypeInt, 1, oneString},
     { "symbol", strexBuiltInSymbol, strexTypeString, 2, twoStrings },
     { "upper", strexBuiltInUpper, strexTypeString, 1, oneString },
     { "lower", strexBuiltInLower, strexTypeString, 1, oneString },
     { "in", strexBuiltInIn, strexTypeBoolean, 2, twoStrings },
     { "starts", strexBuiltInStarts, strexTypeBoolean, 2, twoStrings}, 
     { "ends", strexBuiltInEnds, strexTypeBoolean, 2, twoStrings}, 
     { "same", strexBuiltInSame, strexTypeBoolean, 2, twoStrings}, 
+    { "tidy", strexBuiltInTidy, strexTypeString, 3, threeStrings },
 };
 
 static struct hash *hashBuiltIns()
 /* Build a hash of builtins keyed by name */
 {
 struct hash *hash = hashNew(0);
 int i;
 for (i=0; i<ArraySize(builtins); ++i)
     hashAdd(hash, builtins[i].name, &builtins[i]);
 return hash;
 }
 
 static struct strexIn *strexInNew(struct lineFile *lf,
     void *symbols, StrexLookup lookup)
 /* Return a new strexIn structure wrapped around lineFile */
@@ -1420,42 +1422,49 @@
 static char *stripAll(char *in, char *toRemove, struct lm *lm)
 /* Remove every occurrence of any of the chars in toRemove from in. */
 {
 char *result = lmCloneString(lm, in);  // Move to local memory
 char c, *s = toRemove;
 while ((c = *s++) != 0)
     stripChar(result, c);
 return result;
 }
 
 static char *symbolify(char *prefix, char *original, struct lm *lm)
 /* Convert original to something could use as a C language symbol with dots maybe. */
 {
 int prefixSize = strlen(prefix);
 int originalSize = strlen(original);
-int allocSize = prefixSize + originalSize + 1;
+int allocSize = prefixSize + 2*originalSize + 1;    // worse case all hexed
 char *result = lmAlloc(lm, allocSize);  // Move to local memory
 strcpy(result, prefix);
 char *in = skipLeadingSpaces(original); 
 char *out = result + prefixSize;
-char c;
+unsigned char c;
 while ((c = *in++) != 0)
      {
      if (isspace(c) || c == '-' || c == '.')
 	 *out++ = '_';
      else if (isalnum(c) || c == '_')
          *out++ = c;
+     else
+         {
+	 char hexBuf[8];
+	 safef(hexBuf, sizeof(hexBuf), "%02X", c);
+	 strcpy(out, hexBuf);
+	 out += strlen(hexBuf);
+	 }
      }
 *out++ = 0;
 int len = strlen(result) - prefixSize;
 if (len > 32)
     {
     char *md5 = hmacMd5("", original);
     strcpy(result + prefixSize, md5);
     freeMem(md5);
     }
 
 return result;
 }
 
 static struct strexEval strexEvalCallBuiltIn(struct strexParse *p, 
     void *record, StrexLookup lookup, struct lm *lm)
@@ -1602,30 +1611,66 @@
 	}
     case strexBuiltInEnds:
         {
         struct strexEval string = strexLocalEval(p->children, record, lookup, lm);
         struct strexEval end = strexLocalEval(p->children->next, record, lookup, lm);
 	res.val.b = endsWith(string.val.s, end.val.s);
 	break;
 	}
     case strexBuiltInSame:
         {
         struct strexEval a = strexLocalEval(p->children, record, lookup, lm);
         struct strexEval b = strexLocalEval(p->children->next, record, lookup, lm);
 	res.val.b = (strcmp(a.val.s, b.val.s) == 0);
 	break;
 	}
+    case strexBuiltInTidy:
+	{
+	/* Get parameters */
+        struct strexEval a = strexLocalEval(p->children, record, lookup, lm);
+        struct strexEval b = strexLocalEval(p->children->next, record, lookup, lm);
+        struct strexEval c = strexLocalEval(p->children->next->next, record, lookup, lm);
+	char *before = a.val.s;
+	char *orig = b.val.s;
+	char *after = c.val.s;
+
+	/* Figure out start position - start of string if before string is empty or doesn't match 
+	 * otherwise right after the place where before matches. */
+	char *ourStart = NULL;
+	if (!isEmpty(before))
+	    {
+	    ourStart = strstr(orig, before);
+	    if (ourStart != NULL) 
+		ourStart += strlen(before);
+	    }
+	if (ourStart == NULL)
+	    ourStart = orig;
+
+	/* Figure out end position */
+	char *defaultEnd = ourStart + strlen(ourStart);
+	char *ourEnd = NULL;
+	if (!isEmpty(after))
+	    {
+	    ourEnd = strstr(ourStart, after);
+	    }
+	if (ourEnd == NULL)
+	    ourEnd = defaultEnd;
+
+	int size = ourEnd - ourStart;
+	assert(size >= 0);
+	res.val.s = lmCloneStringZ(lm, ourStart, size);
+	}
     }
 return res;
 }
 
 static struct strexEval strexEvalPick(struct strexParse *pick, void *record, StrexLookup lookup,
     struct lm *lm)
 /* Evaluate a pick operator. */
 {
 /* Evaluate the keyValue */
 struct strexParse *p = pick->children;
 struct strexEval keyVal = strexLocalEval(p, record, lookup, lm);
 p = p->next;
 
 /* Get pointer to default expression but don't evaluate it yet */
 struct strexParse *defaultExp = p;