Index: src/hg/lib/mdb.c
===================================================================
RCS file: /projects/compbio/cvsroot/kent/src/hg/lib/mdb.c,v
retrieving revision 1.6
retrieving revision 1.7
diff -b -B -U 1000000 -r1.6 -r1.7
--- src/hg/lib/mdb.c	11 May 2010 01:43:30 -0000	1.6
+++ src/hg/lib/mdb.c	11 May 2010 16:03:48 -0000	1.7
@@ -1,1881 +1,1882 @@
 /* mdb.c was originally generated by the autoSql program, which also
  * generated mdb.h and mdb.sql.  This module links the database and
  * the RAM representation of objects. */
 
 #include "common.h"
 #include "linefile.h"
 #include "dystring.h"
 #include "jksql.h"
 #include "mdb.h"
 
 static char const rcsid[] = "$Id$";
 
 void mdbStaticLoad(char **row, struct mdb *ret)
 /* Load a row from mdb table into ret.  The contents of ret will
  * be replaced at the next call to this function. */
 {
 
 ret->obj = row[0];
 ret->var = row[1];
 ret->varType = row[2];
 ret->val = row[3];
 }
 
 struct mdb *mdbLoadByQuery(struct sqlConnection *conn, char *query)
 /* Load all mdb from table that satisfy the query given.
  * Where query is of the form 'select * from example where something=something'
  * or 'select example.* from example, anotherTable where example.something =
  * anotherTable.something'.
  * Dispose of this with mdbFreeList(). */
 {
 struct mdb *list = NULL, *el;
 struct sqlResult *sr;
 char **row;
 
 sr = sqlGetResult(conn, query);
 while ((row = sqlNextRow(sr)) != NULL)
     {
     el = mdbLoad(row);
     slAddHead(&list, el);
     }
 slReverse(&list);
 sqlFreeResult(&sr);
 return list;
 }
 
 void mdbSaveToDb(struct sqlConnection *conn, struct mdb *el, char *tableName, int updateSize)
 /* Save mdb as a row to the table specified by tableName.
  * As blob fields may be arbitrary size updateSize specifies the approx size
  * of a string that would contain the entire query. Arrays of native types are
  * converted to comma separated strings and loaded as such, User defined types are
  * inserted as NULL. Note that strings must be escaped to allow insertion into the database.
  * For example "autosql's features include" --> "autosql\'s features include"
  * If worried about this use mdbSaveToDbEscaped() */
 {
 struct dyString *update = newDyString(updateSize);
 dyStringPrintf(update, "insert into %s values ( '%s','%s','%s',%s)",
 	tableName,  el->obj,  el->var,  el->varType,  el->val);
 sqlUpdate(conn, update->string);
 freeDyString(&update);
 }
 
 void mdbSaveToDbEscaped(struct sqlConnection *conn, struct mdb *el, char *tableName, int updateSize)
 /* Save mdb as a row to the table specified by tableName.
  * As blob fields may be arbitrary size updateSize specifies the approx size.
  * of a string that would contain the entire query. Automatically
  * escapes all simple strings (not arrays of string) but may be slower than mdbSaveToDb().
  * For example automatically copies and converts:
  * "autosql's features include" --> "autosql\'s features include"
  * before inserting into database. */
 {
 struct dyString *update = newDyString(updateSize);
 char  *obj, *var, *varType, *val;
 obj = sqlEscapeString(el->obj);
 var = sqlEscapeString(el->var);
 varType = sqlEscapeString(el->varType);
 val = sqlEscapeString(el->val);
 
 dyStringPrintf(update, "insert into %s values ( '%s','%s','%s','%s')",
 	tableName,  obj,  var,  varType,  val);
 sqlUpdate(conn, update->string);
 freeDyString(&update);
 freez(&obj);
 freez(&var);
 freez(&varType);
 freez(&val);
 }
 
 struct mdb *mdbLoad(char **row)
 /* Load a mdb from row fetched with select * from mdb
  * from database.  Dispose of this with mdbFree(). */
 {
 struct mdb *ret;
 
 AllocVar(ret);
 ret->obj = cloneString(row[0]);
 ret->var = cloneString(row[1]);
 ret->varType = cloneString(row[2]);
 ret->val = cloneString(row[3]);
 return ret;
 }
 
 struct mdb *mdbLoadAll(char *fileName)
 /* Load all mdb from a whitespace-separated file.
  * Dispose of this with mdbFreeList(). */
 {
 struct mdb *list = NULL, *el;
 struct lineFile *lf = lineFileOpen(fileName, TRUE);
 char *row[4];
 
 while (lineFileRow(lf, row))
     {
     el = mdbLoad(row);
     slAddHead(&list, el);
     }
 lineFileClose(&lf);
 slReverse(&list);
 return list;
 }
 
 struct mdb *mdbLoadAllByChar(char *fileName, char chopper)
 /* Load all mdb from a chopper separated file.
  * Dispose of this with mdbFreeList(). */
 {
 struct mdb *list = NULL, *el;
 struct lineFile *lf = lineFileOpen(fileName, TRUE);
 char *row[4];
 
 while (lineFileNextCharRow(lf, chopper, row, ArraySize(row)))
     {
     el = mdbLoad(row);
     slAddHead(&list, el);
     }
 lineFileClose(&lf);
 slReverse(&list);
 return list;
 }
 
 struct mdb *mdbCommaIn(char **pS, struct mdb *ret)
 /* Create a mdb out of a comma separated string.
  * This will fill in ret if non-null, otherwise will
  * return a new mdb */
 {
 char *s = *pS;
 
 if (ret == NULL)
     AllocVar(ret);
 ret->obj = sqlStringComma(&s);
 ret->var = sqlStringComma(&s);
 ret->varType = sqlStringComma(&s);
 ret->val = sqlStringComma(&s);
 *pS = s;
 return ret;
 }
 
 void mdbFree(struct mdb **pEl)
 /* Free a single dynamically allocated mdb such as created
  * with mdbLoad(). */
 {
 struct mdb *el;
 
 if ((el = *pEl) == NULL) return;
 freeMem(el->obj);
 freeMem(el->var);
 freeMem(el->varType);
 freeMem(el->val);
 freez(pEl);
 }
 
 void mdbFreeList(struct mdb **pList)
 /* Free a list of dynamically allocated mdb's */
 {
 struct mdb *el, *next;
 
 for (el = *pList; el != NULL; el = next)
     {
     next = el->next;
     mdbFree(&el);
     }
 *pList = NULL;
 }
 
 void mdbOutput(struct mdb *el, FILE *f, char sep, char lastSep)
 /* Print out mdb.  Separate fields with sep. Follow last field with lastSep. */
 {
 if (sep == ',') fputc('"',f);
 fprintf(f, "%s", el->obj);
 if (sep == ',') fputc('"',f);
 fputc(sep,f);
 if (sep == ',') fputc('"',f);
 fprintf(f, "%s", el->var);
 if (sep == ',') fputc('"',f);
 fputc(sep,f);
 if (sep == ',') fputc('"',f);
 fprintf(f, "%s", el->varType);
 if (sep == ',') fputc('"',f);
 fputc(sep,f);
 if (sep == ',') fputc('"',f);
 fprintf(f, "%s", el->val);
 if (sep == ',') fputc('"',f);
 fputc(lastSep,f);
 }
 
 void mdbJsonOutput(struct mdb *el, FILE *f)
 /* Print out mdb in JSON format. */
 {
 fputc('{',f);
 fputc('"',f);
 fprintf(f,"obj");
 fputc('"',f);
 fputc(':',f);
 fputc('"',f);
 fprintf(f, "%s", el->obj);
 fputc('"',f);
 fputc(',',f);
 fputc('"',f);
 fprintf(f,"var");
 fputc('"',f);
 fputc(':',f);
 fputc('"',f);
 fprintf(f, "%s", el->var);
 fputc('"',f);
 fputc(',',f);
 fputc('"',f);
 fprintf(f,"varType");
 fputc('"',f);
 fputc(':',f);
 fputc('"',f);
 fprintf(f, "%s", el->varType);
 fputc('"',f);
 fputc(',',f);
 fputc('"',f);
 fprintf(f,"val");
 fputc('"',f);
 fputc(':',f);
 fputc('"',f);
 fprintf(f, "%s", el->val);
 fputc('"',f);
 fputc('}',f);
 }
 
 /* -------------------------------- End autoSql Generated Code -------------------------------- */
 
 
 #include "ra.h"
 #include "hgConfig.h"
 #include "obscure.h"
 
 #define MDB_METADATA_KEY  "metadata"
 #define MDB_METAOBJ_RAKEY "metaObject"
 #define MDB_METAVAR_RAKEY "metaVariable"
 #define MDB_OBJ_TYPE "objType"
 
 // ------- (static) convert from autoSql -------
 static void mdbVarFree(struct mdbVar **mdbVarPtr)
 // Frees a single mdbVar struct
 {
     freeMem((*mdbVarPtr)->val);
     freeMem((*mdbVarPtr)->var);
     freez(mdbVarPtr);
 }
 
 static void mdbLeafObjFree(struct mdbLeafObj **leafObjPtr)
 // Frees a single mdbVar struct
 {
     freeMem((*leafObjPtr)->obj);
     freez(leafObjPtr);
 }
 
 static void mdbLimbValFree(struct mdbLimbVal **limbValPtr)
 // Frees a single mdbVar struct
 {
 struct mdbLimbVal *limbVal = *limbValPtr;
 
     // Free hash first (shared memory)
     hashFree(&(limbVal->objHash));
 
     struct mdbLeafObj *leafObj = NULL;
     while((leafObj = slPopHead(&(limbVal->objs))) != NULL)
         mdbLeafObjFree(&leafObj);
 
     freeMem(limbVal->val);
     freez(limbValPtr);
 }
 
 static struct mdbObj *mdbObjsLoadFromMemory(struct mdb **mdbPtr,boolean buildHashes)
 // Load all mdbObjs from in memory mdb struct, cannibalize strings.  Expects sorted order.
 {
 struct mdbObj *mdbObj  = NULL;
 struct mdbObj *mdbObjs = NULL;
 struct mdbVar *mdbVar;
 struct mdb *thisRow;
 while((thisRow = slPopHead(mdbPtr)) != NULL)
     {
     if (mdbObj == NULL || differentString(thisRow->obj,mdbObj->obj) )
         {
         // Finish last object before starting next!
         if(mdbObj!= NULL)
             slReverse(&(mdbObjs->vars));
         // Start new object
         AllocVar(mdbObj);
         mdbObj->obj     = thisRow->obj;
         if ( buildHashes )
             mdbObj->varHash = hashNew(0);
         slAddHead(&mdbObjs,mdbObj);
         }
     else
         {
         freeMem(thisRow->obj);  // Already got this from prev row
         }
 
     AllocVar(mdbVar);
     mdbVar->var     = thisRow->var;
     mdbVar->varType = mdbVarTypeStringToEnum(thisRow->varType);
     mdbVar->val     = thisRow->val;
     slAddHead(&(mdbObj->vars),mdbVar);
     if ( buildHashes )
         hashAddUnique(mdbObj->varHash, mdbVar->var, mdbVar); // pointer to struct to resolve type
 
     freeMem(thisRow);
     }
 
 // Finish very last object
 if(mdbObjs && mdbObjs->vars)
     slReverse(&(mdbObjs->vars));
 if(mdbObjs)
     slReverse(&mdbObjs);
 
 return mdbObjs;
 }
 
 static struct mdbByVar *mdbByVarsLoadFromMemory(struct mdb **mdbPtr,boolean buildHashes)
 // Load all mdbVars from in memorys mdb struct, cannibalize strings.  Expects sorted order.
 {
 struct mdbByVar *rootVars = NULL;
 struct mdbByVar *rootVar  = NULL;
 struct mdbLimbVal *limbVal  = NULL;
 struct mdbLeafObj *leafObj;
 struct mdb *thisRow;
 while((thisRow = slPopHead(mdbPtr)) != NULL)
     {
     // Start at root
     if (rootVar == NULL || differentString(thisRow->var,rootVar->var) )
         {
         // Finish last var before starting next!
         if(rootVars && rootVars->vals && rootVars->vals->objs)
             slReverse(&(rootVars->vals->objs));
         if(rootVars && rootVars->vals)
             slReverse(&(rootVars->vals));
         // Start new var
         AllocVar(rootVar);
         limbVal = NULL;  // Very important!
         rootVar->var     = thisRow->var;
         rootVar->varType = mdbVarTypeStringToEnum(thisRow->varType);
         freeMem(thisRow->varType);
         if ( buildHashes )
             rootVar->valHash = hashNew(0);
         slAddHead(&rootVars,rootVar);
         }
     else
         {
         freeMem(thisRow->var);  // Already got this from prev row
         freeMem(thisRow->varType);
         }
 
     // Continue with limb
     if (limbVal == NULL || differentString(thisRow->val,limbVal->val) )
         {
         // Finish last val before starting next!
         if(limbVal != NULL && limbVal->objs != NULL)
             slReverse(&(limbVal->objs));
         // Start new val
         AllocVar(limbVal);
         limbVal->val     = thisRow->val;     // FIXME: binary?
         if ( buildHashes )
             {
             hashAddUnique(rootVar->valHash, limbVal->val, limbVal); // Pointer to struct to get to objHash
             limbVal->objHash = hashNew(0);
             }
         slAddHead(&(rootVar->vals),limbVal);
         }
     else
         freeMem(thisRow->val);  // Already got this from prev row
 
     // End with leaf
     AllocVar(leafObj);
     leafObj->obj     = thisRow->obj;
     if ( buildHashes )
         hashAddUnique(limbVal->objHash, leafObj->obj, leafObj); // Pointer to struct to resolve type!
     slAddHead(&(limbVal->objs),leafObj);
 
     freeMem(thisRow);
     }
 
 // Finish very last object
 if(rootVars && rootVars->vals && rootVars->vals->objs)
     slReverse(&(rootVars->vals->objs));
 if(rootVars && rootVars->vals)
     slReverse(&(rootVars->vals));
 if(rootVars && rootVars->vals)
     slReverse(&rootVars);
 
 return rootVars;
 }
 
 
 static int mdbObjCRC(struct mdbObj *mdbObjs)
 // returns a summ of all individual CRC values of all metObj strings
 {
 int crc = 0;
 struct mdbObj *mdbObj = NULL;
 for(mdbObj=mdbObjs;mdbObj!=NULL;mdbObj=mdbObj->next)
     {
     if(mdbObj->obj != NULL)
         crc += hashCrc(mdbObj->obj);
 
     struct mdbVar *mdbVar = NULL;
     for(mdbVar=mdbObj->vars;mdbVar!=NULL;mdbVar=mdbVar->next)
         {
         if(mdbVar->var != NULL)
             crc += hashCrc(mdbVar->var);
         if(mdbVar->varType == vtTxt && mdbVar->val != NULL)
             crc += hashCrc(mdbVar->val);
         }
     }
 
 return crc;
 }
 
 // -------------- Sort primitives --------------
 int mdbObjCmp(const void *va, const void *vb)
 /* Compare to sort on label. */
 {
 const struct mdbObj *a = *((struct mdbObj **)va);
 const struct mdbObj *b = *((struct mdbObj **)vb);
 return strcasecmp(a->obj, b->obj);
 }
 
 int mdbVarCmp(const void *va, const void *vb)
 /* Compare to sort on label. */
 {
 const struct mdbVar *a = *((struct mdbVar **)va);
 const struct mdbVar *b = *((struct mdbVar **)vb);
 return strcasecmp(a->var, b->var);
 }
 
 
 // -------------- Enum to Strings --------------
 enum mdbVarType mdbVarTypeStringToEnum(char *varType)
 // Convert metadata varType string to enum
 {
 if(sameWord(varType,"txt"))
     return vtTxt;
 if(sameWord(varType,"binary"))
     return vtBinary;
 return vtUnknown;
 }
 
 char *mdbVarTypeEnumToString(enum mdbVarType varType)
 // Convert metadata varType enum string
 {
 switch (varType)
     {
     case vtTxt:    return "txt";
     case vtBinary: return "binary";
     default:       return "unknown";
     }
 }
 
 // ------ Parsing lines ------
 
 struct mdbObj *mdbObjAddVarPairs(struct mdbObj *oldObj,char *varPairs)
 // Parses line of var=val pairs adding to a mdbObj.  Creates mdbObj if NULL
 {
 struct mdbObj *mdbObj = oldObj;
 struct mdbVar *mdbVar;
 char *cloneVars = cloneString(varPairs);
 
     // initial chop and determine if this looks like metadata
     int count = chopByWhiteRespectDoubleQuotes(cloneVars,NULL,0);
     char **words = needMem(sizeof(char *) * count);
     count = chopByWhiteRespectDoubleQuotes(cloneVars,words,count);
     if(count < 1 || words[0] == NULL)
         {
         errAbort("This is not formatted var=val pairs:\n\t%s\n",varPairs);
         }
 
     verbose(3, "mdbObjAddVarPairs() word count:%d\n\t%s\n",count,varPairs);
 
     if(mdbObj == NULL)
         AllocVar(mdbObj);
     if(mdbObj->varHash == NULL)
         mdbObj->varHash = hashNew(0);
 
     int ix;
     for(ix = 0;ix<count;ix++)
         {
         if(*words[ix] == '#')
             break;
         if(strchr(words[ix], '=') == NULL)
             errAbort("This is not formatted var=val pairs: '%s'\n\t%s\n",words[ix],varPairs);
 
         AllocVar(mdbVar);
         mdbVar->var = cloneNextWordByDelimiter(&(words[ix]),'=');
         mdbVar->varType = vtTxt;                               // FIXME: binary?
         mdbVar->val = cloneString(words[ix]);
         verbose(3, "mdbObjAddVarPairs() var=val: %s=%s\n",mdbVar->var,mdbVar->val);
         struct mdbVar *oldVar = (struct mdbVar *)hashFindVal(mdbObj->varHash, mdbVar->var);
         if(oldVar)
             {
             verbose(1, "The same variable appears twice: %s=%s and %s=%s.  Ignoring second value.\n\t%s\n",
                 oldVar->var,oldVar->val,mdbVar->var,mdbVar->val,varPairs);
             mdbVarFree(&mdbVar);
             }
         else
             {
             hashAdd(mdbObj->varHash, mdbVar->var, mdbVar); // pointer to struct to resolve type
             slAddHead(&(mdbObj->vars),mdbVar);
             }
         }
     freeMem(words);
     freeMem(cloneVars);
 
     // Special for old style ENCODE metadata
 #define ENCODE_ALN  "Alignments"
 #define ENCODE_RSIG "RawSignal"
     if(mdbObj->obj == NULL)
         {
         char * tableName = NULL;
         char * fileName = NULL;
         for(mdbVar  = mdbObj->vars;
             mdbVar != NULL && (tableName == NULL || fileName == NULL);
             mdbVar  = mdbVar->next)
             {
             if(sameString(mdbVar->var,"tableName"))
                 tableName = mdbVar->val;
             else if(sameString(mdbVar->var,"fileName"))
                 fileName = mdbVar->val;
             }
 
         mdbVar = NULL; // assertably so, but this is conditioanally created below
         if(tableName != NULL)
             {
             verbose(3, "tableName:%s\n",tableName);
             if(fileName == NULL || startsWithWordByDelimiter(tableName,'.',fileName))
                 {
                 mdbObj->obj     = cloneString(tableName);
                 AllocVar(mdbVar);
                 mdbVar->var = cloneString(MDB_OBJ_TYPE);
                 mdbVar->val = cloneString("table");
                 }
             else if(stringIn(ENCODE_ALN,fileName) && stringIn(ENCODE_RSIG,tableName))// Messier case where the file has "Alignment" but the table has "RawSignal"
                 {
                 char *tmpFilName = cloneString(fileName);
                 strSwapStrs(tmpFilName, strlen(tmpFilName),ENCODE_ALN, ENCODE_RSIG);
                 if(startsWithWordByDelimiter(tableName,'.',tmpFilName))
                     {
                     mdbObj->obj     = cloneString(tableName);
                     AllocVar(mdbVar);
                     mdbVar->var = cloneString(MDB_OBJ_TYPE);
                     mdbVar->val = cloneString("table");
                     }
                 freeMem(tmpFilName);
                 }
             }
         else if(fileName != NULL)
             {
             verbose(3, "fileName:%s\n",fileName);
             // NOTE: that the file object is the root of the name, so both file.fastq.gz and file.fastq are same obj!
             mdbObj->obj     = cloneFirstWordByDelimiter(fileName,'.');
             AllocVar(mdbVar);
             mdbVar->var = cloneString(MDB_OBJ_TYPE);
             mdbVar->val = cloneString("file");
             }
 
         if(mdbVar != NULL) // Just determined an objType
             {
             verbose(3, "mdbObjAddVarPairs() var=val: %s=%s\n",mdbVar->var,mdbVar->val);
             struct mdbVar *oldVar = (struct mdbVar *)hashFindVal(mdbObj->varHash, mdbVar->var);
             if(oldVar)
                 mdbVarFree(&mdbVar);
             else
                 {
                 hashAdd(mdbObj->varHash, mdbVar->var, mdbVar); // pointer to struct to resolve type
                 slAddHead(&(mdbObj->vars),mdbVar);
                 }
             }
         }
 
 if(mdbObj->obj == NULL) // NOTE: Should this be a hard error!
     errAbort("No obj found. This is not properly formatted metadata:\n\t%s\n",varPairs);
 
     //slReverse(&(mdbObj->vars)); Could have added vars so sort instead
     slSort(&(mdbObj->vars),&mdbVarCmp); // Should be in determined order
     mdbVar = (struct mdbVar *)hashFindVal(mdbObj->varHash, MDB_OBJ_TYPE);
     if(mdbVar == NULL)
         mdbVar = mdbObj->vars;
     verbose(3, "mdbObjAddVarPairs() obj=%s %s=%s\n",
     mdbObj->obj, mdbVar->var,mdbVar->val);
 return mdbObj;
 }
 
 struct mdbObj *metadataLineParse(char *line)
 /* Parses a single formatted metadata line into mdbObj for updates or queries. */
 {
 char *fromTheTop = line;
 char*nibbledWord = cloneNextWordByDelimiter(&line,' ');
 if(nibbledWord == NULL || differentWord(nibbledWord,MDB_METADATA_KEY))
     errAbort("This is not a formatted metadata line:\n\t%s\n",fromTheTop);
 freeMem(nibbledWord);
 
 struct mdbObj *mdbObj = NULL;
 char*varPairs = line;
 nibbledWord = cloneNextWordByDelimiter(&line,' ');;
 if(nibbledWord == NULL)
     errAbort("This is not a formatted metadata line:\n\t%s\n",fromTheTop);
 if(strchr(nibbledWord, '=') == NULL) // If this is not a var=val then it should be obj
     {
     AllocVar(mdbObj);
     mdbObj->obj = nibbledWord;
     verbose(3, "metadataLineParse() obj=%s\n",mdbObj->obj);
     varPairs = line;
     while(strlen(line) > 0)
         {
         nibbledWord = cloneNextWordByDelimiter(&line,' ');;
         if(nibbledWord == NULL)
             errAbort("This is not a formatted metadata line:\n\t%s\n",fromTheTop);
         if(*nibbledWord == '#' || strchr(nibbledWord, '=') != NULL) // IS commnet OR start of var=val pairs
             break;
 
         if(sameWord(nibbledWord,"delete"))
             mdbObj->deleteThis = TRUE;
         else
             errAbort("This is not a formatted metadata line:\n\t%s\n",fromTheTop);
         varPairs = line;
         freeMem(nibbledWord);
         }
     }
 if(varPairs != NULL && strlen(varPairs) > 0 && *varPairs != '#')
         mdbObj = mdbObjAddVarPairs(mdbObj,varPairs);
 else if(mdbObj->deleteThis == FALSE)
     errAbort("This is not a formatted metadata line:\n\t%s\n",fromTheTop);
 return mdbObj;
 }
 
 struct mdbByVar *mdbByVarsLineParse(char *line)
 /* Parses a line of "var1=val1 var2=val2 into a mdbByVar object for queries. */
 {
 int thisWord = 0;
 struct mdbByVar   *mdbByVars = NULL;
 struct mdbByVar   *rootVar = NULL;
 struct mdbLimbVal *limbVal = NULL;
 char *cloneLine = cloneString(line);
 struct hash* varHash;     // There must not be multiple occurrances of the same var
 
     // initial chop and determine if this looks like metadata
     int count = chopByWhiteRespectDoubleQuotes(cloneLine,NULL,0);
     char **words = needMem(sizeof(char *) * count);
     count = chopByWhiteRespectDoubleQuotes(cloneLine,words,count);
 
     verbose(3, "mdbByVarsLineParse() word count:%d\n\t%s\n",count,line);
     // Get obj and figure out if this is a delete line
     varHash = hashNew(0);
 
     // All words are expected to be var=val pairs!
     for(thisWord=0;thisWord<count;thisWord++)
         {
         if(strchr(words[thisWord], '=') == NULL)
             {
             errAbort("Expected 'var=val' but found '%s'.  This is not properly formatted metadata:\n\t%s\n",words[thisWord],line);
             //mdbObjsFree(&mdbObj);
             //return NULL;
             }
         AllocVar(rootVar);
         rootVar->var = cloneNextWordByDelimiter(&(words[thisWord]),'=');
         rootVar->notEqual = (rootVar->var[strlen(rootVar->var)-1] == '!'); // requested not equal
         if(rootVar->notEqual)
             rootVar->var[strlen(rootVar->var)-1] = 0;
         char *val = cloneString(words[thisWord]);
         if(sameWord(val,"?"))  // "var=?" or "var=" will query by var name only
             freez(&val);
 
         struct mdbByVar *oldVar = (struct mdbByVar *)hashFindVal(varHash, rootVar->var);
         if(oldVar)
             {  // FIXME: Could build this for 'or' queries!
             verbose(1, "The same variable appears twice: %s=%s and %s=%s.  Ignoring second value.\n",
                 oldVar->var,oldVar->vals->val,rootVar->var,val);
             freeMem(rootVar->var);
             freeMem(rootVar);
             freeMem(val);
             }
         else
             {
             AllocVar(limbVal);
             limbVal->val = val;
             rootVar->vals = limbVal;
             hashAdd(varHash, rootVar->var, rootVar);
             slAddHead(&mdbByVars,rootVar);
             }
         }
     freeMem(words);
     slReverse(&mdbByVars);
     verbose(3, "mdbByVarsLineParse() parsed:%d first: %s=%s.\n",
         slCount(mdbByVars->vals),mdbByVars->var,mdbByVars->vals->val);
 return mdbByVars;
 }
 
 // ------ Loading from args, hashes and tdb ------
 struct mdbByVar*mdbByVarCreate(char *var, char *varType,char *val)
 /* Creates a singular var=val pair struct for metadata queries. */
 {
 struct mdbByVar *mdbByVar = NULL;
 
     if(var == NULL)
         errAbort("Need variable to create mdbByVar query object.\n");
 
     AllocVar(mdbByVar);
     mdbByVar->var = cloneString(var);
     mdbByVar->varType = (varType==NULL?vtUnknown:mdbVarTypeStringToEnum(varType));
 
     if(val != NULL)
         {
         struct mdbLimbVal * limbVal;
         AllocVar(limbVal);
 
         limbVal->val    = cloneString(val);
         mdbByVar->vals = limbVal; // Only one
         }
 
 return mdbByVar;
 }
 
 struct mdbObj *mdbObjCreate(char *obj,char *var, char *varType,char *val)
 /* Creates a singular mdbObj query object based on obj and all other optional params. */
 {
 struct mdbObj *mdbObj = NULL;
 
     if(obj == NULL)
         errAbort("Need obj to create mdbObj query object.\n");
 
     AllocVar(mdbObj);
     mdbObj->obj     = cloneString(obj);
 
     if(var != NULL)
         {
         struct mdbVar * mdbVar;
         AllocVar(mdbVar);
 
         mdbVar->var     = cloneString(var);
         mdbVar->varType = (varType==NULL?vtUnknown:mdbVarTypeStringToEnum(varType));
         if(val != NULL)
             mdbVar->val     = cloneString(val);
         mdbObj->vars = mdbVar; // Only one
         }
 return mdbObj;
 }
 
 struct mdbObj *mdbObjsLoadFromHashes(struct hash *objsHash)
 // Load all mdbObjs from a file containing metadata formatted lines
 {
 struct mdbObj *mdbObjs = NULL;
 struct hashEl* objEl = NULL;
 
 struct hashCookie objCookie = hashFirst(objsHash);
 while((objEl = hashNext(&objCookie)) != NULL)
     {
     struct mdbObj *mdbObj;
     AllocVar(mdbObj);
     mdbObj->obj     = cloneString(objEl->name);
     mdbObj->varHash = hashNew(0);
     struct hash *hashedVars = objEl->val;
     struct hashCookie varCookie = hashFirst(hashedVars);
     struct hashEl* varEl = NULL;
     while((varEl = hashNext(&varCookie)) != NULL)
         {
         if(sameString(varEl->name,MDB_METAOBJ_RAKEY))
             continue;
 
         struct mdbVar * mdbVar;
         AllocVar(mdbVar);
         mdbVar->var     = cloneString(varEl->name);
         mdbVar->varType = vtTxt;                    // FIXME: binary?
         mdbVar->val     = cloneString(varEl->val);
         hashAdd(mdbObj->varHash, mdbVar->var, mdbVar); // pointer to struct to resolve type
         slAddHead(&(mdbObj->vars),mdbVar);
         }
         slSort(&(mdbObj->vars),&mdbVarCmp); // Should be in determined order
         slAddHead(&mdbObjs,mdbObj);
     }
     slSort(&mdbObjs,&mdbObjCmp); // Should be in determined order
     return mdbObjs;
 }
 
 // ------ Loading from files ------
 struct mdbObj *mdbObjsLoadFromFormattedFile(char *fileName,boolean *validated)
 // Load all mdbObjs from a file containing metadata formatted lines
 {
 struct mdbObj *mdbObjs = NULL;
 struct lineFile *lf = lineFileOpen(fileName, TRUE);
 char *line;
 
 while (lineFileNext(lf, &line,NULL))
     {
     char *start = skipLeadingSpaces(line);
     if(start == NULL || *start == '#')
         continue;
     if(startsWithWord(MDB_METAOBJ_RAKEY,line))
         {
         // This is the RA style file!!
         lineFileClose(&lf);
         return mdbObjsLoadFromRAFile(fileName,validated);
         }
     struct mdbObj *mdbObj = metadataLineParse(line);
     if(mdbObj == NULL)
         {
         mdbObjsFree(&mdbObjs);
         return NULL;
         }
     slAddHead(&mdbObjs,mdbObj);
     }
     lineFileClose(&lf);
     slReverse(&mdbObjs);  // Go ahead and keep this in file order
     if(validated)
         *validated = FALSE;
     return mdbObjs;
 }
 
 #define MDB_MAGIC_PREFIX "# MAGIC: "
 struct mdbObj *mdbObjsLoadFromRAFile(char *fileName,boolean *validated)
 // Load all mdbObjs from a file containing RA formatted 'metaObjects'
 {
 struct hash *mdHash = raReadAll(fileName, MDB_METAOBJ_RAKEY);
 if(mdHash == NULL)
     {
     verbose(1,"Missing, empty or badly formated RA file:%s\n",fileName);
     return NULL;
     }
 struct mdbObj *mdbObjs = mdbObjsLoadFromHashes(mdHash);
 hashFree(&mdHash);
 
 // Try to validate file
 if(validated)
     {
     *validated = FALSE;
     struct lineFile *lf = lineFileOpen(fileName, TRUE);
     char *line = lineFileSkipToLineStartingWith(lf,MDB_MAGIC_PREFIX,1000000);
     if(line != NULL)
         {
         int fileMagic = atoi(line+strlen(MDB_MAGIC_PREFIX));
         int objsMagic = mdbObjCRC(mdbObjs);
         verbose(3,"Objects magic: %d  Files magic: %d (%s)\n",objsMagic,fileMagic,line+strlen(MDB_MAGIC_PREFIX));
         *validated = (fileMagic == objsMagic);
         }
     else
         verbose(3,"Can't find magic number on this file.\n");
     }
 return mdbObjs;
 }
 
 // ------ Table name and creation ------
 #define MDB_SPEC_LOCATION "/cluster/bin/sqlCreate/metaDb.sql"
 
 void mdbReCreate(struct sqlConnection *conn,char *tblName,boolean testOnly)
 // Creates ore Recreates the named mdb.
 {
 if(sqlTableExists(conn,tblName))
     verbose(2, "Table '%s' already exists.  It will be recreated.\n",tblName);
 
 char *sql = NULL;
 readInGulp(MDB_SPEC_LOCATION, &sql, NULL);
 char *pos = strchr(sql, ';');
 if ( pos != NULL)
     *pos = 0;
 char *oldSql = cloneString(sql);
 pos = stringIn("CREATE TABLE ", oldSql);
 if (pos == NULL)
     errAbort("Can't find CREATE TABLE in %s\n", MDB_SPEC_LOCATION);
 nextWord(&pos);
 nextWord(&pos);
 char *oldName = nextWord(&pos);
 if(differentWord(oldName, tblName))
     {
     char *saveSql = cloneString(sql);
     freeMem(sql);
     sql = replaceChars(saveSql, oldName, tblName);
     freeMem(saveSql);
     }
 freeMem(oldSql);
 verbose(2, "Requesting table creation:\n\t%s;\n", sql);
 if(!testOnly)
     sqlRemakeTable(conn,tblName, sql);
 freeMem(sql);
 }
 
 char*mdbTableName(struct sqlConnection *conn,boolean mySandBox)
 // returns the mdb table name or NULL if conn supplied but the table doesn't exist
 {
 char *tblName = NULL;
 char *root = NULL;
 char *sand = NULL;
 char *name = cfgOption("db.metaDb");
 if(name == NULL)
     {
     name = cfgOption("db.trackDb");
     if(name == NULL)
         root = cloneString(MDB_DEFAULT_NAME);
     }
 
 // Divide name into root and sand
 if(root == NULL)
     {
     char delimit = '_';
     if((sand = strchr(name,delimit)) == NULL)
         {
         delimit = '-';
         if((sand = strchr(name,delimit)) == NULL)
             root = cloneString(name);  // No sandBox portion
         }
 
     if(root == NULL)  // There should be a sandbox portion
         {
         root = cloneNextWordByDelimiter(&name,delimit);
         if(mySandBox && *name != 0)
             sand = name;
         }
     }
 
 // Since db.trackDb was used, make sure to swap it
 if(sameWord("trackDb",root))
     {
     freeMem(root);
     root = cloneString(MDB_DEFAULT_NAME);
     }
 
 if(!mySandBox || sand == NULL)
     tblName = root;
 else
     {
     int size = strlen(root) + strlen(sand) + 2;
     tblName = needMem(size);
     safef(tblName,size,"%s_%s",root,sand);
     }
 
 // Test for table
 if(conn != NULL && !sqlTableExists(conn,tblName))
     {
     if(sand == NULL || sameWord(tblName,root)) // Then try the root
         return NULL;
     freeMem(tblName);
     tblName = root;
     if(!sqlTableExists(conn,tblName))
         return NULL;
     }
 
 return tblName;
 }
 
 // -------------- Updating the DB --------------
 int mdbObjsSetToDb(struct sqlConnection *conn,char *tableName,struct mdbObj *mdbObjs,boolean replace,boolean testOnly)
 // Adds or updates metadata obj/var pairs into the named table.  Returns total rows affected
 {
 char query[8192];
 struct mdbObj *mdbObj;
 struct mdbVar *mdbVar;
 int count = 0;
 
 if(tableName == NULL)
     tableName = MDB_DEFAULT_NAME;
 
 if(!sqlTableExists(conn,tableName))
     errAbort("mdbObjsSetToDb attempting to update non-existent table named '%s'.\n",tableName);
 
 for(mdbObj = mdbObjs;mdbObj != NULL; mdbObj = mdbObj->next)
     {
     // Handle delete requests first
     if(mdbObj->deleteThis)
         {
         if(mdbObj->vars == NULL) // deletes all
             {
             safef(query, sizeof(query),"%s where obj = '%s'",tableName,mdbObj->obj);
             int delCnt = sqlRowCount(conn,query);
 
             if(delCnt>0)
                 {
                 safef(query, sizeof(query),
                     "delete from %s where obj = '%s'",tableName,mdbObj->obj);
                 verbose(2, "Requesting delete of %d rows:\n\t%s;\n",delCnt, query);
                 if(!testOnly)
                     sqlUpdate(conn, query);
                 count += delCnt;
                 }
             }
         else  // deletes selected vars
             {
             for(mdbVar = mdbObj->vars;mdbVar != NULL; mdbVar = mdbVar->next)
                 {
                 safef(query, sizeof(query),
                     "select obj from %s where obj = '%s' and var = '%s'",
                     tableName,mdbObj->obj,mdbVar->var);
                 if(sqlExists(conn,query))
                     {
                     safef(query, sizeof(query),
                         "delete from %s where obj = '%s' and var = '%s'",
                         tableName,mdbObj->obj,mdbVar->var);
                     verbose(2, "Requesting delete of 1 row:\n\t%s;\n",query);
                     if(!testOnly)
                         sqlUpdate(conn, query);
                     count++;
                     }
                 }
             }
         continue;  // Done with this mdbObj
         }
     else if (replace)  // If replace then clear out deadwood before inserting new vars
         {
         safef(query, sizeof(query),"%s where obj = '%s'",tableName,mdbObj->obj);
         int delCnt = sqlRowCount(conn,query);
 
         if(delCnt>0)
             {
             safef(query, sizeof(query),
                 "delete from %s where obj = '%s'",tableName,mdbObj->obj);
             verbose(2, "Requesting replacement of %d rows:\n\t%s;\n",delCnt, query);
             if(!testOnly)
                 sqlUpdate(conn, query);
             count += delCnt;
             }
         }
 
     // Now it is time for update or add!
     for(mdbVar = mdbObj->vars;mdbVar != NULL; mdbVar = mdbVar->next)
         {
         stripEnclosingDoubleQuotes(mdbVar->val); // Ensures values are stripped of enclosing quotes
 
         // Be sure to check for var existence first, then update
         if (!replace)
             {
             struct mdbObj *objExists = mdbObjQueryByObj(conn,tableName,mdbObj->obj,mdbVar->var);
             if(objExists)
                 {
                 if(differentString(mdbVar->val,objExists->vars->val)
                 || mdbVar->varType != objExists->vars->varType)
                     {
                     safef(query, sizeof(query),
                         "update %s set varType = '%s', val = '%s' where obj = '%s' and var = '%s'",
                             tableName,
                             mdbVarTypeEnumToString(mdbVar->varType),sqlEscapeString(mdbVar->val), // FIXME: binary val?
                             mdbObj->obj,mdbVar->var);
                     verbose(2, "Requesting update of 1 row:\n\t%s;\n",query);
                     if(!testOnly)
                         sqlUpdate(conn, query);
                     count++;
                     }
                 mdbObjsFree(&objExists);
                 continue;  // The object was found/updated so done with it
                 }
             }
         // Finally ready to insert new vars
         safef(query, sizeof(query),
             "insert into %s values ( '%s','%s','%s','%s')",
                 tableName,mdbObj->obj,mdbVar->var,mdbVarTypeEnumToString(mdbVar->varType),
                 sqlEscapeString(mdbVar->val)); // FIXME: binary val?  // FIXME Strip quotes
         verbose(2, "Requesting insert of one row:\n\t%s;\n",query);
         if(!testOnly)
             sqlUpdate(conn, query);
         count++;
         }
     }
 return count;
 }
 
 // ------------------ Querys -------------------
 struct mdbObj *mdbObjQuery(struct sqlConnection *conn,char *table,struct mdbObj *mdbObj)
 // Query the metadata table by obj and optional vars and vals in metaObj struct.  If mdbObj is NULL query all.
 // Returns new mdbObj struct fully populated and sorted in obj,var order.
 {
 //  select obj,var,val where (var= [and val=]) or ([var= and] val=) order by obj,var
     boolean buildHash = TRUE;
 
     if(table == NULL)
         table = MDB_DEFAULT_NAME;
 
     if(!sqlTableExists(conn,table))
         return NULL;
 
     struct dyString *dy = newDyString(4096);
     dyStringPrintf(dy, "select obj,var,varType,val from %s", table);
     if(mdbObj != NULL && mdbObj->obj != NULL)
         {
         dyStringPrintf(dy, " where obj %s '%s'",
             (strchr(mdbObj->obj,'%')?"like":"="),mdbObj->obj);
 
         struct mdbVar *mdbVar;
         for(mdbVar=mdbObj->vars;mdbVar!=NULL;mdbVar=mdbVar->next)
             {
             if(mdbVar==mdbObj->vars)
                 dyStringPrintf(dy, " and (");
             else
                 dyStringPrintf(dy, " or ");
             if(mdbVar->var != NULL)
                 {
                 if(mdbVar->val != NULL)
                     dyStringPrintf(dy, "(");
                 dyStringPrintf(dy, "var %s '%s'",
                     (strchr(mdbVar->var,'%')?"like":"="),mdbVar->var);
                 }
             if(mdbVar->val != NULL)
                 {
                 if(mdbVar->var != NULL)
                     dyStringPrintf(dy, " and ");
                 dyStringPrintf(dy, "val %s '%s'",
                     (strchr(mdbVar->val,'%')?"like":"="), sqlEscapeString(mdbVar->val));
                 if(mdbVar->var != NULL)
                     dyStringPrintf(dy, ")");
                 }
             if(mdbVar->var == NULL && mdbVar->val)
                 errAbort("mdbObjQuery has empty mdbVar struct.\n");
             buildHash = FALSE;  // too few variables
             }
         if(mdbObj->vars != NULL)
             dyStringPrintf(dy, ")");
         }
     dyStringPrintf(dy, " order by obj, var");
     verbose(2, "Requesting query:\n\t%s;\n",dyStringContents(dy));
 
     struct mdb *mdb = mdbLoadByQuery(conn, dyStringCannibalize(&dy));
     struct mdbObj *mdbObjs = mdbObjsLoadFromMemory(&mdb,buildHash);
     verbose(3, "Returned %d object(s) with %d var(s).\n",
         mdbObjCount(mdbObjs,TRUE),mdbObjCount(mdbObjs,FALSE));
     return mdbObjs;
 }
 
 struct mdbObj *mdbObjQueryByObj(struct sqlConnection *conn,char *table,char *obj,char *var)
 // Query a single metadata object and optional var from a table (default mdb).
 {
 if(obj == NULL)
     return mdbObjQuery(conn,table,NULL);
 
 struct mdbObj *queryObj  = mdbObjCreate(obj,var,NULL,NULL);
 struct mdbObj *resultObj = mdbObjQuery(conn,table,queryObj);
 mdbObjsFree(&queryObj);
 return resultObj;
 }
 
 struct mdbByVar *mdbByVarsQuery(struct sqlConnection *conn,char *table,struct mdbByVar *mdbByVars)
 // Query the metadata table by one or more var=val pairs to find the distinct set of objs that satisfy ANY conditions.
 // Returns new mdbByVar struct fully populated and sorted in var,val,obj order.
 {
 //  select obj,var,val where (var= [and val in (val1,val2)]) or (var= [and val in (val1,val2)]) order by var,val,obj
 
     if(table == NULL)
         table = MDB_DEFAULT_NAME;
     if(!sqlTableExists(conn,table))
         return NULL;
 
     struct dyString *dy = newDyString(4096);
-    dyStringPrintf(dy, "select distinct obj,var,varType,val from %s", table);
+    dyStringPrintf(dy, "select obj,var,varType,val from %s", table);
 
     struct mdbByVar *rootVar;
     for(rootVar=mdbByVars;rootVar!=NULL;rootVar=rootVar->next)
         {
         if(rootVar==mdbByVars)
             dyStringPrintf(dy, " where (var ");
         else
             dyStringPrintf(dy, " OR (var ");
         if(rootVar->notEqual && rootVar->vals == NULL)
             dyStringPrintf(dy, "%s",strchr(rootVar->var,'%')?"NOT ":"!");
 
         dyStringPrintf(dy, "%s '%s'",
             (strchr(rootVar->var,'%')?"like":"="), rootVar->var);
 
         struct mdbLimbVal *limbVal;
         boolean multiVals = FALSE;
         for(limbVal=rootVar->vals;limbVal!=NULL;limbVal=limbVal->next)
             {
             if(limbVal->val == NULL || strlen(limbVal->val) < 1)
                 continue;
 
             if(!multiVals)
                 {
                 dyStringPrintf(dy, " and val ");
                 if(rootVar->notEqual)
                     dyStringPrintf(dy, "%s",strchr(limbVal->val,'%')?"NOT ":"!");
                 if(limbVal->next == NULL) // only one val
                     {
                     dyStringPrintf(dy, "%s '%s'",
                         (strchr(limbVal->val,'%')?"like":"="), sqlEscapeString(limbVal->val));
                     break;
                     }
                 else
                     dyStringPrintf(dy, "in (");
                 multiVals=TRUE;
                 }
             else
                 dyStringPrintf(dy, ",");
             dyStringPrintf(dy, "'%s'", sqlEscapeString(limbVal->val));
             }
         if(multiVals)
             dyStringPrintf(dy, ")");
         dyStringPrintf(dy, ")");
         }
     dyStringPrintf(dy, " order by var, val, obj");
     verbose(2, "Requesting query:\n\t%s;\n",dyStringContents(dy));
 
     struct mdb *mdb = mdbLoadByQuery(conn, dyStringCannibalize(&dy));
     verbose(3, "rows (vars) returned: %d\n",slCount(mdb));
     struct mdbByVar *mdbByVarsFromMem = mdbByVarsLoadFromMemory(&mdb,TRUE);
     verbose(3, "Returned %d vars(s) with %d val(s) with %d object(s).\n",
         mdbByVarCount(mdbByVarsFromMem,TRUE ,FALSE),
         mdbByVarCount(mdbByVarsFromMem,FALSE,TRUE ),
         mdbByVarCount(mdbByVarsFromMem,FALSE,FALSE));
     return mdbByVarsFromMem;
 }
 
 struct mdbByVar *mdbByVarQueryByVar(struct sqlConnection *conn,char *table,char *varName,char *val)
 // Query a single metadata variable and optional val from a table (default mdb) for searching val->obj.
 {
 if(varName == NULL)
     return mdbByVarsQuery(conn,table,NULL);
 
 struct mdbByVar *queryVar  = mdbByVarCreate(varName,NULL,val);
 struct mdbByVar *resultVar = mdbByVarsQuery(conn,table,queryVar);
 mdbByVarsFree(&queryVar);
 return resultVar;
 }
 
 struct mdbObj *mdbObjsQueryByVars(struct sqlConnection *conn,char *table,struct mdbByVar *mdbByVars)
 // Query the metadata table by one or more var=val pairs to find the distinct set of objs that satisfy ALL conditions.
 // Returns new mdbObj struct fully populated and sorted in obj,var order.
 {
 //  select obj,var,val where (var= [and val in (val1,val2)]) or (var= [and val in (val1,val2)]) order by obj,var
 
     if(table == NULL)
         table = MDB_DEFAULT_NAME;
     if(!sqlTableExists(conn,table))
         return NULL;
 
     struct dyString *dy = newDyString(4096);
-    dyStringPrintf(dy, "select distinct obj,var,varType,val from %s", table);
+    dyStringPrintf(dy, "select t1.obj,t1.var,t1.varType,t1.val from %s t1", table);
 
     struct mdbByVar *rootVar;
     boolean gotVar = FALSE;
-    for(rootVar=mdbByVars;rootVar!=NULL;rootVar=rootVar->next)
+    int tix;
+    for(rootVar=mdbByVars,tix=2;rootVar!=NULL;rootVar=rootVar->next,tix++)
         {
         if(!gotVar)
             {
-            dyStringPrintf(dy, " where obj in ");
+            dyStringPrintf(dy, " where t1.obj in ");
             gotVar=TRUE;
             }
         else
-            dyStringPrintf(dy, " AND obj in ");
-        dyStringPrintf(dy, "(select obj from %s where var ",table);
+            dyStringPrintf(dy, " AND t1.obj in ");
+        dyStringPrintf(dy, "(select t%d.obj from %s t%d where t%d.obj = t1.obj and t%d.var ",tix,table,tix,tix,tix);
 
         if(rootVar->notEqual && rootVar->vals == NULL)
             dyStringPrintf(dy, "%s",strchr(rootVar->var,'%')?"NOT ":"!");
 
         dyStringPrintf(dy, "%s '%s'",
             (strchr(rootVar->var,'%')?"like":"="), rootVar->var);
 
         struct mdbLimbVal *limbVal;
         boolean multiVals = FALSE;
         for(limbVal=rootVar->vals;limbVal!=NULL;limbVal=limbVal->next)
             {
             if(limbVal->val == NULL || strlen(limbVal->val) < 1)
                 continue;
 
             if(!multiVals)
                 {
-                dyStringPrintf(dy, " and val ");
+                dyStringPrintf(dy, " and t%d.val ",tix);
                 if(rootVar->notEqual)
                     dyStringPrintf(dy, "%s",strchr(limbVal->val,'%')?"NOT ":"!");
                 if(limbVal->next == NULL) // only one val
                     {
                     dyStringPrintf(dy, "%s '%s'",
                         (strchr(limbVal->val,'%')?"like":"="), sqlEscapeString(limbVal->val));
                     break;
                     }
                 else
                     dyStringPrintf(dy, "in (");
                 multiVals=TRUE;
                 }
             else
                 dyStringPrintf(dy, ",");
             dyStringPrintf(dy, "'%s'", sqlEscapeString(limbVal->val));
             }
         if(multiVals)
             dyStringPrintf(dy, ")");
         dyStringPrintf(dy, ")");
         }
     dyStringPrintf(dy, " order by obj, var");
     verbose(2, "Requesting query:\n\t%s;\n",dyStringContents(dy));
 
     struct mdb *mdb = mdbLoadByQuery(conn, dyStringCannibalize(&dy));
     verbose(3, "rows (vars) returned: %d\n",slCount(mdb));
     struct mdbObj *mdbObjs = mdbObjsLoadFromMemory(&mdb,TRUE);
     verbose(3, "Returned %d object(s) with %d var(s).\n",
         mdbObjCount(mdbObjs,TRUE),mdbObjCount(mdbObjs,FALSE));
     return mdbObjs;
 }
 
 
 // ----------- Printing and Counting -----------
 static void mdbVarValPrint(struct mdbVar *mdbVar,boolean raStyle)
 {
 if(mdbVar != NULL && mdbVar->var != NULL)
     {
     if(raStyle)
         printf("\n%s ",mdbVar->var);
     else
         printf(" %s=",mdbVar->var);
     if(mdbVar->val != NULL)
         {
         if(mdbVar->varType == vtBinary)
             printf("binary");
         else if(!raStyle && strchr(mdbVar->val, ' ') != NULL) // Has blanks
             printf("\"%s\"",mdbVar->val);
         else
             printf("%s",mdbVar->val);
         }
     }
 }
 
 
 void mdbObjPrint(struct mdbObj *mdbObjs,boolean raStyle)
 // prints objs and var=val pairs as formatted metadata lines or ra style
 {
 // Single line:
 //   metadata iLoveLucy table lucy=ricky ethyl=fred
 // ra style
 //   metadata iLoveLucy table
 //       lucy ricky
 //       ethy fred
 // TODO: Expand for mutilple var types; strip quotes from vals on ra style
 struct mdbObj *mdbObj = NULL;
 for(mdbObj=mdbObjs;mdbObj!=NULL;mdbObj=mdbObj->next)
     {
     if(mdbObj->obj == NULL)
         continue;
 
     printf("%s %s",(raStyle?MDB_METAOBJ_RAKEY:MDB_METADATA_KEY),mdbObj->obj);
     if(mdbObj->deleteThis)
         printf(" delete");
 
     struct mdbVar *mdbVar = NULL;
 
     // If hash available, force objType to front
     if(mdbObj->varHash != NULL)
         {
         mdbVar = hashFindVal(mdbObj->varHash,MDB_OBJ_TYPE);
         mdbVarValPrint(mdbVar,raStyle);
         }
     for(mdbVar=mdbObj->vars;mdbVar!=NULL;mdbVar=mdbVar->next)
         {
         if(mdbObj->varHash == NULL || !sameOk(MDB_OBJ_TYPE,mdbVar->var))
             mdbVarValPrint(mdbVar,raStyle);
         }
     printf("%s",(raStyle?"\n\n":"\n"));
     }
 if(raStyle) // NOTE: currently only supporting validation of RA files
     printf("%s%d\n",MDB_MAGIC_PREFIX,mdbObjCRC(mdbObjs));
 }
 
 void mdbByVarPrint(struct mdbByVar *mdbByVars,boolean raStyle)
 // prints var=val pairs and objs that go with them single lines or ra style
 {
 // Single line:
 //   mdbVariable lucy=ethyl bestFriends lifePartners
 //   mdbVariable lucy=ricky iLoveLucy divorces
 // NOT QUITE ra style
 //   metadata Fred wife=Ethyl
 //   metadata Lucy wife=Ethyl
 // Results in:
 //   mdbVariable wife Ethyl
 //   metaObject Fred
 //   metaObject Lucy
 struct mdbByVar *rootVar = NULL;
 for(rootVar=mdbByVars;rootVar!=NULL;rootVar=rootVar->next)
     {
     if(rootVar->var == NULL)
         continue;
 
     struct mdbLimbVal *limbVal = NULL;
     for(limbVal=rootVar->vals;limbVal!=NULL;limbVal=limbVal->next)
         {
         if(limbVal->val == NULL)
             continue;
 
         if(raStyle)
             printf("%s %s ",MDB_METAVAR_RAKEY,rootVar->var);
         else
             printf("%s %s=",MDB_METAVAR_RAKEY,rootVar->var);
 
         if(rootVar->varType == vtBinary)
             printf("binary");
         else if(!raStyle && strchr(limbVal->val, ' ') != NULL) // Has blanks
             printf("\"%s\"",limbVal->val);
         else
             printf("%s",limbVal->val);
 
         struct mdbLeafObj *leafObj = NULL;
         for(leafObj=limbVal->objs;leafObj!=NULL;leafObj=leafObj->next)
             {
             if(leafObj->obj == NULL)
                 continue;
 
             if(raStyle)
                 printf("\n%s %s",MDB_METAOBJ_RAKEY,leafObj->obj);
             else
                 printf(" %s",leafObj->obj);
             }
         printf("\n");
         if(raStyle)
             printf("\n");
         }
     }
 }
 
 
 int mdbObjCount(struct mdbObj *mdbObjs,boolean objs)
 // returns the count of vars belonging to this obj or objs;
 {
 int count = 0;
 struct mdbObj *mdbObj = NULL;
 for(mdbObj=mdbObjs;mdbObj!=NULL;mdbObj=mdbObj->next)
     {
     if(mdbObj->obj == NULL)
         continue;
     if(objs)
         count++;
     else
         {
         struct mdbVar *mdbVar = NULL;
         for(mdbVar=mdbObj->vars;mdbVar!=NULL;mdbVar=mdbVar->next)
             {
             if(mdbVar->var != NULL && mdbVar->val != NULL)
                 count++;
             }
         }
     }
 
 return count;
 }
 
 int mdbByVarCount(struct mdbByVar *mdbByVars,boolean vars, boolean vals)
 // returns the count of objs belonging to this set of vars;
 {
 int count = 0;
 struct mdbByVar *rootVar = NULL;
 for(rootVar=mdbByVars;rootVar!=NULL;rootVar=rootVar->next)
     {
     if(rootVar->var == NULL)
         continue;
     if(vars)
         count++;
     else
         {
         struct mdbLimbVal *limbVal = NULL;
         for(limbVal=rootVar->vals;limbVal!=NULL;limbVal=limbVal->next)
             {
             if(limbVal->val == NULL)
                 continue;
             if(vals)
                 count++;
             else
                 {
                 struct mdbLeafObj *leafObj = NULL;
                 for(leafObj=limbVal->objs;leafObj!=NULL;leafObj=leafObj->next)
                     {
                     if(leafObj->obj != NULL)
                         count++;
                     }
                 }
             }
         }
     }
 return count;
 }
 
 // ----------------- Utilities -----------------
 
 char *mdbObjFindValue(struct mdbObj *mdbObj, char *var)
 // Finds the val associated with the var or retruns NULL
 {
 if (mdbObj == NULL)
     return NULL;
 
 struct mdbVar *mdbVar = NULL;
 if(mdbObj->varHash != NULL)
     mdbVar = hashFindVal(mdbObj->varHash,var);
 else
     {
     for(mdbVar=mdbObj->vars;mdbVar!=NULL;mdbVar=mdbVar->next)
         {
         if(sameOk(var,mdbVar->var))
             break;
         }
     }
 if(mdbVar == NULL)
     return NULL;
 
 return mdbVar->val;
 }
 
 boolean mdbObjContains(struct mdbObj *mdbObj, char *var, char *val)
 // Returns TRUE if object contains var, val or both
 {
 if (mdbObj == NULL)
     return FALSE;
 
 if(var != NULL)
     {
     char *foundVal = mdbObjFindValue(mdbObj,var);
     if(foundVal == NULL)
         return FALSE;
     if(val == NULL)
         return TRUE;
     return sameOk(foundVal,val);
     }
 struct mdbVar *mdbVar = NULL;
 for(mdbVar=mdbObj->vars;mdbVar!=NULL;mdbVar=mdbVar->next)
     {
     if(differentStringNullOk(var,mdbVar->var) != 0)
         continue;
     if(differentStringNullOk(val,mdbVar->val) != 0)
         continue;
     return TRUE;
     }
 
 return FALSE;
 }
 
 boolean mdbByVarContains(struct mdbByVar *mdbByVar, char *val, char *obj)
 // Returns TRUE if var contains val, obj or both
 {
 if (mdbByVar != NULL)
     {
     struct mdbLimbVal *limbVal = NULL;
     struct mdbLeafObj *leafObj = NULL;
     if(mdbByVar->valHash != NULL && val != NULL)
         {
         limbVal = hashFindVal(mdbByVar->valHash,val);
         if(limbVal == NULL || limbVal->val == NULL)
             return FALSE;
         if(limbVal->objHash != NULL && obj != NULL)
             {
             leafObj = hashFindVal(limbVal->objHash,obj);
             if(leafObj == NULL)
                 return FALSE;
             return sameOk(leafObj->obj,obj);
             }
         }
     for(limbVal=mdbByVar->vals;limbVal!=NULL;limbVal=limbVal->next)
         {
         if(differentStringNullOk(val,limbVal->val) != 0)
             continue;
 
         for(leafObj=limbVal->objs;leafObj!=NULL;leafObj=leafObj->next)
             {
             if(differentStringNullOk(obj,leafObj->obj) != 0)
                 continue;
             return TRUE;
             }
         }
     }
 return FALSE;
 }
 
 void mdbObjReorderVars(struct mdbObj *mdbObjs, char *vars,boolean back)
 // Reorders vars list based upon list of vars "cell antibody treatment".  Send to front or back.
 {
 //char *words[48];
 char *cloneLine = cloneString(vars);
 int count = chopByWhite(cloneLine,NULL,0);
 char **words = needMem(sizeof(char *) * count);
 count = chopByWhite(cloneLine,words,count);
 //int count = chopLine(cloneLine,words);
 
 struct mdbObj *mdbObj = NULL;
 for( mdbObj=mdbObjs; mdbObj!=NULL; mdbObj=mdbObj->next )
     {
     int ix;
     struct mdbVar *orderedVars = NULL;
     struct mdbVar **varsToReorder = needMem(sizeof(struct mdbVar *) * count);
 
     struct mdbVar *mdbVar = NULL;
     while((mdbVar = slPopHead(&(mdbObj->vars))) != NULL)
         {
         ix = stringArrayIx(mdbVar->var,words,count);
         if(ix < 0)
             slAddHead(&orderedVars,mdbVar);
         else
             varsToReorder[ix] = mdbVar;
         }
 
     if(back) // add to front of backward list
         {
         for( ix=0; ix<count; ix++ )
             {
             if(varsToReorder[ix] != NULL)
                 slAddHead(&orderedVars,varsToReorder[ix]);
             }
         }
     slReverse(&orderedVars);
 
     if(!back)  // Add to front of forward list
         {
         for( ix=count-1; ix>=0; ix-- )
             {
             if(varsToReorder[ix] != NULL)
                 slAddHead(&orderedVars,varsToReorder[ix]);
             }
         }
 
     mdbObj->vars = orderedVars;
     freeMem(varsToReorder);
     }
     freeMem(words);
 }
 
 void mdbObjRemoveVars(struct mdbObj *mdbObjs, char *vars)
 // Prunes list of vars for an object, freeing the memory.  Doesn't touch DB.
 {
 char *cloneLine = NULL;
 int count = 0;
 char **words = NULL;
 if(vars != NULL)
     {
     cloneLine = cloneString(vars);
     count = chopByWhite(cloneLine,NULL,0);
     words = needMem(sizeof(char *) * count);
     count = chopByWhite(cloneLine,words,count);
     }
 struct mdbObj *mdbObj = NULL;
 for( mdbObj=mdbObjs; mdbObj!=NULL; mdbObj=mdbObj->next )
     {
     int ix;
     struct mdbVar *keepTheseVars = NULL;
 
     if(count == 0 && mdbObj->varHash != NULL)
         hashFree(&mdbObj->varHash);
 
     struct mdbVar *mdbVar = NULL;
     while((mdbVar = slPopHead(&(mdbObj->vars))) != NULL)
         {
         if(count == 0)
             ix = 1;
         else
             ix = stringArrayIx(mdbVar->var,words,count);
         if(ix < 0)
             slAddHead(&keepTheseVars,mdbVar);
         else
             {
             if(count != 0 && mdbObj->varHash != NULL)
                 hashRemove(mdbObj->varHash, mdbVar->var);
 
             mdbVarFree(&mdbVar);
             }
         }
 
     if(keepTheseVars != NULL)
         slReverse(&keepTheseVars);
     mdbObj->vars = keepTheseVars;
     }
     if(words != NULL)
         freeMem(words);
 }
 
 void mdbObjSwapVars(struct mdbObj *mdbObjs, char *vars,boolean deleteThis)
 // Replaces objs' vars with var=vap pairs provided, preparing for DB update.
 {
 struct mdbObj *mdbObj = NULL;
 for( mdbObj=mdbObjs; mdbObj!=NULL; mdbObj=mdbObj->next )
     {
     mdbObj->deleteThis = deleteThis;
 
     if(mdbObj->varHash != NULL)
         hashFree(&mdbObj->varHash);
 
     struct mdbVar *mdbVar = NULL;
     while((mdbVar = slPopHead(&(mdbObj->vars))) != NULL)
         mdbVarFree(&mdbVar);
 
     mdbObjAddVarPairs(mdbObj,vars);
     }
 }
 
 void mdbObjTransformToUpdate(struct mdbObj *mdbObjs, char *var, char *varType,char *val,boolean deleteThis)
 // Turns one or more mdbObjs into the stucture needed to add/update or delete.
 {
 struct mdbObj *mdbObj = NULL;
 for( mdbObj=mdbObjs; mdbObj!=NULL; mdbObj=mdbObj->next )
     {
     mdbObj->deleteThis = deleteThis;
 
     if(mdbObj->varHash != NULL)
         hashFree(&mdbObj->varHash);
 
     struct mdbVar *mdbVar = NULL;
     while((mdbVar = slPopHead(&(mdbObj->vars))) != NULL)
         mdbVarFree(&mdbVar);
 
     if(var != NULL)
         {
         AllocVar(mdbVar);
 
         mdbVar->var     = cloneString(var);
         mdbVar->varType = (varType==NULL?vtUnknown:mdbVarTypeStringToEnum(varType));
         if(val != NULL)
             mdbVar->val     = cloneString(val);
         mdbObj->vars = mdbVar; // Only one
         }
     }
 }
 
 struct mdbObj *mdbObjClone(const struct mdbObj *mdbObj)
 // Clones a single mdbObj, including hash and maintining order
 {
 if(mdbObj == NULL)
     return NULL;
 
 struct mdbObj *newObj;
 AllocVar(newObj);
 
 if(mdbObj->obj != NULL)
     newObj->obj    = cloneString(mdbObj->obj);
 newObj->deleteThis = mdbObj->deleteThis;
 if(mdbObj->vars != NULL)
     {
     if(mdbObj->varHash != NULL)
         newObj->varHash = hashNew(0);
 
     struct mdbVar *mdbVar = NULL;
     for(mdbVar = mdbObj->vars; mdbVar != NULL; mdbVar = mdbVar->next )
         {
         struct mdbVar *newVar = NULL;
         AllocVar(newVar);
         if(mdbVar->var != NULL)
             newVar->var = cloneString(mdbVar->var);
         if(mdbVar->val != NULL)
             newVar->val = cloneString(mdbVar->val);
         newVar->varType    = mdbVar->varType;
         if(newVar->var != NULL && newVar->val != NULL)
             hashAdd(newObj->varHash, newVar->var, newVar); // pointer to struct to resolve type
         slAddHead(&(newObj->vars),newVar);
         }
     slReverse(&(newObj->vars));
     }
 return newObj;
 }
 
 // --------------- Free at last ----------------
 void mdbObjsFree(struct mdbObj **mdbObjsPtr)
 // Frees one or more metadata objects and any contained mdbVars.  Will free any hashes as well.
 {
 
 if(mdbObjsPtr != NULL && *mdbObjsPtr != NULL)
     {
     // free all roots
     struct mdbObj *mdbObj = NULL;
     while((mdbObj = slPopHead(mdbObjsPtr)) != NULL)
         {
         // Free hash first (shared memory)
         hashFree(&(mdbObj->varHash));
 
         // free all leaves
         struct mdbVar *mdbVar = NULL;
         while((mdbVar = slPopHead(&(mdbObj->vars))) != NULL)
             mdbVarFree(&mdbVar);
 
         // The rest of root
         freeMem(mdbObj->obj);
         freeMem(mdbObj);
         }
     freez(mdbObjsPtr);
     }
 }
 
 void mdbByVarsFree(struct mdbByVar **mdbByVarsPtr)
 // Frees one or more metadata vars and any contained vals and objs.  Will free any hashes as well.
 {
 if(mdbByVarsPtr != NULL && *mdbByVarsPtr != NULL)
     {
     // free all roots
     struct mdbByVar *rootVar = NULL;
     while((rootVar = slPopHead(mdbByVarsPtr)) != NULL)
         {
         // Free hash first (shared memory)
         hashFree(&(rootVar->valHash));
 
         // free all limbs
         struct mdbLimbVal *limbVal = NULL;
         while((limbVal = slPopHead(&(rootVar->vals))) != NULL)
             mdbLimbValFree(&limbVal);
 
         // The rest of root
         if(rootVar->var)
             freeMem(rootVar->var);
         freeMem(rootVar);
         }
     freez(mdbByVarsPtr);
     }
 }
 
 // ----------------- CGI specific routines for use with tdb -----------------
 #define MDB_NOT_FOUND ((struct mdbObj *)-666)
 #define METADATA_NOT_FOUND ((struct mdbObj *)-999)
 #define MDB_OBJ_KEY "mdbObj"
 static struct mdbObj *metadataForTableFromTdb(struct trackDb *tdb)
 // Returns the metadata for a table from a tdb setting.
 {
 char *setting = trackDbSetting(tdb, MDB_METADATA_KEY);
 if(setting == NULL)
     return NULL;
 struct mdbObj *mdbObj;
 AllocVar(mdbObj);
 mdbObj->obj     = cloneString(tdb->table);
 AllocVar(mdbObj->vars);
 mdbObj->vars->var = cloneString(MDB_OBJ_TYPE);
 mdbObj->vars->val = cloneString("table");
 mdbObj->varHash = hashNew(0);
 hashAdd(mdbObj->varHash, mdbObj->vars->var, mdbObj->vars);
 return mdbObjAddVarPairs(mdbObj,setting);
 }
 
 const struct mdbObj *metadataForTable(char *db,struct trackDb *tdb,char *table)
 // Returns the metadata for a table.  NEVER FREE THIS STRUCT!
 {
 struct mdbObj *mdbObj = NULL;
 
 // See of the mdbObj was already built
 if(tdb != NULL)
     {
     mdbObj = tdbExtrasGetOrDefault(tdb, MDB_OBJ_KEY,NULL);
     if(mdbObj == METADATA_NOT_FOUND) // NOT in mtatbl, not in tdb metadata setting!
         return NULL;
     else if(mdbObj == MDB_NOT_FOUND) // looked mdb already and not found!
         return metadataForTableFromTdb(tdb);
     else if(mdbObj != NULL)
         {
         return mdbObj;  // No reason to query the table again!
         }
     }
 
 struct sqlConnection *conn = sqlConnect(db);
 char *mdb = mdbTableName(conn,TRUE);  // Look for sandbox name first
 if(tdb != NULL && tdb->table != NULL)
     table = tdb->table;
 if(mdb != NULL)
     mdbObj = mdbObjQueryByObj(conn,mdb,table,NULL);
 sqlDisconnect(&conn);
 
 // save the mdbObj for next time
 if(tdb)
     {
     if(mdbObj != NULL)
         tdbExtrasAddOrUpdate(tdb,MDB_OBJ_KEY,mdbObj);
     else
         {
         tdbExtrasAddOrUpdate(tdb,MDB_OBJ_KEY,MDB_NOT_FOUND);
         return metadataForTableFromTdb(tdb);  // FIXME: metadata setting in TDB is soon to be obsolete
         }
     }
 
 // FIXME: Temporary to distinguish mdb metadata from trackDb metadata:
 mdbObjRemoveVars(mdbObj,"tableName");
 
 
 return mdbObj;
 }
 
 const char *metadataFindValue(struct trackDb *tdb, char *var)
 // Finds the val associated with the var or retruns NULL
 {
 struct mdbObj *mdbObj = tdbExtrasGetOrDefault(tdb, MDB_OBJ_KEY,NULL);
 if(mdbObj == MDB_NOT_FOUND) // Note, only we if already looked for mdb (which requires db)
     mdbObj = metadataForTableFromTdb(tdb);
 if (mdbObj == NULL || mdbObj == METADATA_NOT_FOUND)
     return NULL;
 
 return mdbObjFindValue(mdbObj,var);
 }
 

