981ef07d25e4560559dae28d3e316913569944d7 tdreszer Fri Jul 6 17:51:06 2012 -0700 Next batch of many checkins as dictated by Jim. Formatting space after if and limiting lines to 100 chars. Changes limited to lines last touched by tdreszer (git blame) so as not to ruin history. None of these changes should affect executables in any way. diff --git src/hg/lib/mdb.c src/hg/lib/mdb.c index 18fe651..37d132a 100644 --- src/hg/lib/mdb.c +++ src/hg/lib/mdb.c @@ -349,32 +349,32 @@ else { freeMem(thisRow->var); // Already got this from prev row } // Continue with limb if (limbVal == NULL || differentString(thisRow->val,limbVal->val) ) { // Finish last val before starting next! if(limbVal != NULL && limbVal->objs != NULL) slReverse(&(limbVal->objs)); // Start new val AllocVar(limbVal); limbVal->val = thisRow->val; if ( buildHashes ) - { - hashAddUnique(rootVar->valHash, limbVal->val, limbVal); // Pointer to struct to get to objHash + { // Pointer to struct to get to objHash + hashAddUnique(rootVar->valHash, limbVal->val, limbVal); limbVal->objHash = hashNew(10); } slAddHead(&(rootVar->vals),limbVal); } else freeMem(thisRow->val); // Already got this from prev row // End with leaf AllocVar(leafObj); leafObj->obj = thisRow->obj; if ( buildHashes ) hashAddUnique(limbVal->objHash, leafObj->obj, leafObj); // Pointer to struct to resolve type! slAddHead(&(limbVal->objs),leafObj); freeMem(thisRow); @@ -467,31 +467,32 @@ AllocVar(mdbVar); if(strchr(words[ix], '=') == NULL) // treat this the same as "var=" { mdbVar->var = cloneString(words[ix]); mdbVar->val = NULL; } else { mdbVar->var = cloneNextWordByDelimiter(&(words[ix]),'='); mdbVar->val = cloneString(words[ix]); } verbose(3, "mdbObjAddVarPairs() var=val: %s=%s\n",mdbVar->var,mdbVar->val); struct mdbVar *oldVar = (struct mdbVar *)hashFindVal(mdbObj->varHash, mdbVar->var); if(oldVar) { - verbose(1, "The same variable appears twice: %s=%s and %s=%s. Ignoring second value.\n\t%s\n", + verbose(1,"The same variable appears twice: %s=%s and %s=%s. " + "Ignoring second value.\n\t%s\n", oldVar->var,oldVar->val,mdbVar->var,mdbVar->val,varPairs); mdbVarFree(&mdbVar); } else { hashAdd(mdbObj->varHash, mdbVar->var, mdbVar); // pointer to struct to resolve type slAddHead(&(mdbObj->vars),mdbVar); } } freeMem(words); freeMem(cloneVars); // Special for old style ENCODE metadata #define ENCODE_ALN "Alignments" #define ENCODE_RSIG "RawSignal" @@ -508,48 +509,50 @@ else if(sameString(mdbVar->var,MDB_VAR_FILENAME)) fileName = mdbVar->val; } mdbVar = NULL; // assertably so, but this is conditioanally created below if(tableName != NULL) { verbose(3, "%s:%s\n",MDB_VAR_TABLENAME,tableName); if(fileName == NULL || startsWithWordByDelimiter(tableName,'.',fileName)) { mdbObj->obj = cloneString(tableName); AllocVar(mdbVar); mdbVar->var = cloneString(MDB_OBJ_TYPE); mdbVar->val = cloneString(MDB_OBJ_TYPE_TABLE); } - else if(stringIn(ENCODE_ALN,fileName) && stringIn(ENCODE_RSIG,tableName))// Messier case where the file has "Alignment" but the table has "RawSignal" + else if (stringIn(ENCODE_ALN,fileName) // Messier case where the file has "Alignment" + && stringIn(ENCODE_RSIG,tableName)) // but the table has "RawSignal" { char *tmpFilName = cloneString(fileName); strSwapStrs(tmpFilName, strlen(tmpFilName),ENCODE_ALN, ENCODE_RSIG); if(startsWithWordByDelimiter(tableName,'.',tmpFilName)) { mdbObj->obj = cloneString(tableName); AllocVar(mdbVar); mdbVar->var = cloneString(MDB_OBJ_TYPE); mdbVar->val = cloneString(MDB_OBJ_TYPE_TABLE); } freeMem(tmpFilName); } } else if(fileName != NULL) { verbose(3, "%s:%s\n",MDB_VAR_FILENAME,fileName); - // NOTE: that the file object is the root of the name, so both file.fastq.gz and file.fastq are same obj! + // NOTE: that the file object is the root of the name, + // so both file.fastq.gz and file.fastq are same obj! mdbObj->obj = cloneFirstWordByDelimiter(fileName,'.'); AllocVar(mdbVar); mdbVar->var = cloneString(MDB_OBJ_TYPE); mdbVar->val = cloneString(MDB_OBJ_TYPE_FILE); } if(mdbVar != NULL) // Just determined an objType { verbose(3, "mdbObjAddVarPairs() var=val: %s=%s\n",mdbVar->var,mdbVar->val); struct mdbVar *oldVar = (struct mdbVar *)hashFindVal(mdbObj->varHash, mdbVar->var); if(oldVar) mdbVarFree(&mdbVar); else { hashAdd(mdbObj->varHash, mdbVar->var, mdbVar); // pointer to struct to resolve type @@ -584,31 +587,32 @@ char*varPairs = line; nibbledWord = cloneNextWordByDelimiter(&line,' ');; if(nibbledWord == NULL) errAbort("This is not a formatted metadata line:\n\t%s\n",fromTheTop); if(strchr(nibbledWord, '=') == NULL) // If this is not a var=val then it should be obj { AllocVar(mdbObj); mdbObj->obj = nibbledWord; verbose(3, "metadataLineParse() %s=%s\n",MDB_OBJ,mdbObj->obj); varPairs = line; while(strlen(line) > 0) { nibbledWord = cloneNextWordByDelimiter(&line,' ');; if(nibbledWord == NULL) errAbort("This is not a formatted metadata line:\n\t%s\n",fromTheTop); - if(*nibbledWord == '#' || strchr(nibbledWord, '=') != NULL) // IS commnet OR start of var=val pairs + if (*nibbledWord == '#' // IS comment + || strchr(nibbledWord, '=') != NULL) // OR start of var=val pairs break; if(sameWord(nibbledWord,"delete")) mdbObj->deleteThis = TRUE; else errAbort("This is not a formatted metadata line:\n\t%s\n",fromTheTop); varPairs = line; freeMem(nibbledWord); } } if(varPairs != NULL && strlen(varPairs) > 0 && *varPairs != '#') mdbObj = mdbObjAddVarPairs(mdbObj,varPairs); else if(mdbObj->deleteThis == FALSE) errAbort("This is not a formatted metadata line:\n\t%s\n",fromTheTop); return mdbObj; @@ -622,44 +626,47 @@ struct mdbByVar *rootVar = NULL; struct mdbLimbVal *limbVal = NULL; char *cloneLine = cloneString(line); // initial chop and determine if this looks like metadata int count = chopByWhiteRespectDoubleQuotes(cloneLine,NULL,0); char **words = needMem(sizeof(char *) * count); count = chopByWhiteRespectDoubleQuotes(cloneLine,words,count); verbose(3, "mdbByVarsLineParse() word count:%d\n\t%s\n",count,line); // All words are expected to be var=val pairs! for (thisWord=0; thisWord<count; thisWord++) { if (strchr(words[thisWord], '=') == NULL) - errAbort("Expected '%s=%s' but found '%s'. This is not properly formatted metadata:\n\t%s\n",MDB_VAR,MDB_VAL,words[thisWord],line); + errAbort("Expected '%s=%s' but found '%s'. This is not properly formatted metadata:\n" + "\t%s\n",MDB_VAR,MDB_VAL,words[thisWord],line); // Set up var struct from 1st half of pair - // NOTE: Do not try to combine repeated vars because "fob=a fob=b" is 'AND' while "fob=a,b" is 'OR'. + // NOTE: Do not try to combine repeated vars because "fob=a fob=b" is 'AND' + // while "fob=a,b" is 'OR'. // Does this make sense? Yes: select * ... where fob like 'Fr%' and fob != 'Frunk' AllocVar(rootVar); rootVar->var = cloneNextWordByDelimiter(&(words[thisWord]),'='); rootVar->notEqual = (rootVar->var[strlen(rootVar->var)-1] == '!'); // requested not equal if (rootVar->notEqual) rootVar->var[strlen(rootVar->var)-1] = 0; // Fill in the val(s) from second half of pair char *val = NULL; - if (words[thisWord][0] != '\0' && words[thisWord][0] != '?') // "var=?" or "var=" will query by var name only + if (words[thisWord][0] != '\0' + && words[thisWord][0] != '?') // "var=?" or "var=" will query by var name only val = cloneString(words[thisWord]); if (val != NULL) { // Strip any single or double quotes first. char *end = val + strlen(val) - 1; if ((*val == '"' && *end == '"') || (*val == '\'' && *end == '\'')) { *end = '\0'; val++; } // handle comma separated list of vals (if not framed with widcards) if (strchr(val,',') != NULL && (*val != '%' || *(val + strlen(val) - 1) != '%')) { char * aVal = NULL; @@ -670,31 +677,32 @@ slAddTail(&rootVar->vals,limbVal); } } else { AllocVar(limbVal); limbVal->val = val; rootVar->vals = limbVal; } } slAddHead(&mdbByVars,rootVar); } freeMem(words); slReverse(&mdbByVars); verbose(3, "mdbByVarsLineParse() parsed:%d first: %s%s='%s'.\n", - slCount(mdbByVars),mdbByVars->var,(mdbByVars->notEqual?"!":""),(mdbByVars->vals?mdbByVars->vals->val:"")); + slCount(mdbByVars),mdbByVars->var,(mdbByVars->notEqual?"!":""), + (mdbByVars->vals?mdbByVars->vals->val:"")); return mdbByVars; } // ------ Loading from args, hashes and tdb ------ struct mdbByVar*mdbByVarCreate(char *var, char *val) /* Creates a singular var=val pair struct for metadata queries. */ { struct mdbByVar *mdbByVar = NULL; if(var == NULL) errAbort("Need variable to create mdbByVar query object.\n"); AllocVar(mdbByVar); mdbByVar->var = cloneString(var); @@ -775,31 +783,31 @@ errAbort("Need obj to create mdbObj object.\n"); if (mdbVars == NULL) { AllocVar(mdbObj); mdbObj->obj = cloneString(obj); return mdbObj; } else { mdbObj = mdbObjCreate(obj,mdbVars->var,mdbVars->val); mdbObj->varHash = hashNew(8); hashAddUnique(mdbObj->varHash, mdbVars->var, mdbObj->vars); // pointer to struct to resolve type struct mdbVar *var = mdbVars->next; - for(;var != NULL;var = var->next); + for(;var != NULL;var = var->next) mdbObjSetVar(mdbObj, var->var,var->val); } return mdbObj; } struct mdbObj *mdbObjsLoadFromHashes(struct hash *objsHash) // Load all mdbObjs from a file containing metadata formatted lines { struct mdbObj *mdbObjs = NULL; struct hashEl* objEl = NULL; struct hashCookie objCookie = hashFirst(objsHash); while((objEl = hashNext(&objCookie)) != NULL) { struct mdbObj *mdbObj; @@ -873,31 +881,32 @@ return NULL; } struct mdbObj *mdbObjs = mdbObjsLoadFromHashes(mdHash); hashFree(&mdHash); // Try to validate file if(validated) { *validated = FALSE; struct lineFile *lf = lineFileOpen(fileName, TRUE); char *line = lineFileSkipToLineStartingWith(lf,MDB_MAGIC_PREFIX,1000000); if(line != NULL) { int fileMagic = atoi(line+strlen(MDB_MAGIC_PREFIX)); int objsMagic = mdbObjCRC(mdbObjs); - verbose(3,"Objects magic: %d Files magic: %d (%s)\n",objsMagic,fileMagic,line+strlen(MDB_MAGIC_PREFIX)); + verbose(3,"Objects magic: %d Files magic: %d (%s)\n", + objsMagic,fileMagic,line+strlen(MDB_MAGIC_PREFIX)); *validated = (fileMagic == objsMagic); } else verbose(3,"Can't find magic number on this file.\n"); } return mdbObjs; } // ------ Table name and creation ------ void mdbReCreate(struct sqlConnection *conn,char *tblName,boolean testOnly) // Creates ore Recreates the named mdb. { char *sqlCreate = "# Contains metadata for a table, file or other objects.\n" @@ -994,40 +1003,41 @@ return NULL; } freeMem(table); table = cloneString(MDB_DEFAULT_NAME); if (!sqlTableExists(conn,table)) { freeMem(table); return NULL; } } return table; } // -------------- Updating the DB -------------- -int mdbObjsSetToDb(struct sqlConnection *conn,char *tableName,struct mdbObj *mdbObjs,boolean replace,boolean testOnly) +int mdbObjsSetToDb(struct sqlConnection *conn,char *tableName,struct mdbObj *mdbObjs, + boolean replace,boolean testOnly) // Adds or updates metadata obj/var pairs into the named table. Returns total rows affected { char query[8192]; struct mdbObj *mdbObj; struct mdbVar *mdbVar; int count = 0; if(tableName == NULL) - tableName = mdbTableName(conn,TRUE); // defaults to sandbox, if it exists, else MDB_DEFAULT_NAME; + tableName = mdbTableName(conn,TRUE); // defaults to sandbox, if exists, else MDB_DEFAULT_NAME else if(!sqlTableExists(conn,tableName)) errAbort("mdbObjsSetToDb attempting to update non-existent table named '%s'.\n",tableName); // Table specific lock (over-cautious, since most work is done on sandbox tables) char lock[64]; safef(lock,sizeof lock,"lock_%s",tableName); sqlGetLock(conn, lock); for(mdbObj = mdbObjs;mdbObj != NULL; mdbObj = mdbObj->next) { // Handle delete requests first if(mdbObj->deleteThis) { if(mdbObj->vars == NULL) // deletes all { @@ -1111,80 +1121,83 @@ // Finally ready to insert new vars safef(query, sizeof(query), "insert into %s set obj='%s', var='%s', val='%s'", tableName,mdbObj->obj,mdbVar->var, sqlEscapeString(mdbVar->val)); // FIXME Strip quotes verbose(2, "Requesting insert of one row:\n\t%s;\n",query); if(!testOnly) sqlUpdate(conn, query); count++; } } sqlReleaseLock(conn, lock); return count; } -int mdbObjsLoadToDb(struct sqlConnection *conn,char *tableName,struct mdbObj *mdbObjs,boolean testOnly) +int mdbObjsLoadToDb(struct sqlConnection *conn,char *tableName,struct mdbObj *mdbObjs, + boolean testOnly) // Adds mdb Objs with minimal error checking { int count = 0; verboseTime(2, "Start of mdbObjsLoadToDb %s", tableName); if (tableName == NULL) - tableName = mdbTableName(conn,TRUE); // defaults to sandbox, if it exists, else MDB_DEFAULT_NAME; + tableName = mdbTableName(conn,TRUE); // defaults to sandbox, if exists, else MDB_DEFAULT_NAME else if (!sqlTableExists(conn,tableName)) errAbort("mdbObjsLoadToDb attempting to load non-existent table named '%s'.\n",tableName); assert(mdbObjs != NULL); // If this is the case, then be vocal #define MDB_TEMPORARY_TAB_FILE "temporaryMdb.tab" long lastTime = 0; count = mdbObjPrintToTabFile(mdbObjs,MDB_TEMPORARY_TAB_FILE); verboseTime(2, "past mdbObjPrintToTabFile()"); // Disable keys in hopes of speeding things up. No danger since it only disables non-unique keys char query[8192]; safef(query, sizeof(query),"alter table %s disable keys",tableName); sqlUpdate(conn, query); // Quick? load -sqlLoadTabFile(conn, MDB_TEMPORARY_TAB_FILE, tableName, SQL_TAB_FILE_WARN_ON_ERROR|SQL_TAB_FILE_WARN_ON_WARN); +sqlLoadTabFile(conn, MDB_TEMPORARY_TAB_FILE, tableName, + SQL_TAB_FILE_WARN_ON_ERROR|SQL_TAB_FILE_WARN_ON_WARN); verboseTime(2, "past sqlLoadTabFile()"); // Enabling the keys again safef(query, sizeof(query),"alter table %s enable keys",tableName); sqlUpdate(conn, query); verboseTime(2, "Past alter table"); //unlink(MDB_TEMPORARY_TAB_FILE); -verbose(0,"%04ldms - Done loading mdb with 'LOAD DATA INFILE' mysql command.\n",(clock1000() - lastTime)); +verbose(0,"%04ldms - Done loading mdb with 'LOAD DATA INFILE' mysql command.\n", + (clock1000() - lastTime)); return count; } // ------------------ Querys ------------------- struct mdbObj *mdbObjQuery(struct sqlConnection *conn,char *table,struct mdbObj *mdbObj) -// Query the metadata table by obj and optional vars and vals in metaObj struct. If mdbObj is NULL query all. -// Returns new mdbObj struct fully populated and sorted in obj,var order. +// Query the metadata table by obj and optional vars and vals in metaObj struct. +// If mdbObj is NULL query all. Returns new mdbObj struct fully populated & sorted in obj,var order { // select obj,var,val where (var= [and val=]) or ([var= and] val=) order by obj,var boolean buildHash = TRUE; if(table == NULL) - table = mdbTableName(conn,TRUE); // defaults to sandbox, if it exists, else MDB_DEFAULT_NAME; + table = mdbTableName(conn,TRUE); // defaults to sandbox, if exists, else MDB_DEFAULT_NAME else if(!sqlTableExists(conn,table)) return NULL; struct dyString *dy = newDyString(4096); dyStringPrintf(dy, "select obj,var,val from %s", table); if(mdbObj != NULL && mdbObj->obj != NULL) { dyStringPrintf(dy, " where obj %s '%s'", (strchr(mdbObj->obj,'%')?"like":"="),mdbObj->obj); struct mdbVar *mdbVar; for(mdbVar=mdbObj->vars;mdbVar!=NULL;mdbVar=mdbVar->next) { if(mdbVar==mdbObj->vars) dyStringPrintf(dy, " and ("); @@ -1224,53 +1237,56 @@ } struct mdbObj *mdbObjQueryByObj(struct sqlConnection *conn,char *table,char *obj,char *var) // Query a single metadata object and optional var from a table (default mdb). { if(obj == NULL) return mdbObjQuery(conn,table,NULL); struct mdbObj *queryObj = mdbObjCreate(obj,var,NULL); struct mdbObj *resultObj = mdbObjQuery(conn,table,queryObj); mdbObjsFree(&queryObj); return resultObj; } struct mdbByVar *mdbByVarsQuery(struct sqlConnection *conn,char *table,struct mdbByVar *mdbByVars) -// Query the metadata table by one or more var=val pairs to find the distinct set of objs that satisfy ANY conditions. +// Query the metadata table by one or more var=val pairs to find the distinct set of objs +// that satisfy ANY conditions. // Returns new mdbByVar struct fully populated and sorted in var,val,obj order. { -// select obj,var,val where (var= [and val in (val1,val2)]) or (var= [and val in (val1,val2)]) order by var,val,obj +// select obj,var,val where (var= [and val in (val1,val2)]) +// or (var= [and val in (val1,val2)]) order by var,val,obj if(table == NULL) - table = mdbTableName(conn,TRUE); // defaults to sandbox, if it exists, else MDB_DEFAULT_NAME; + table = mdbTableName(conn,TRUE); // defaults to sandbox, if exists, else MDB_DEFAULT_NAME else if(!sqlTableExists(conn,table)) return NULL; struct dyString *dy = newDyString(4096); dyStringPrintf(dy, "select obj,var,val from %s", table); struct mdbByVar *rootVar; for(rootVar=mdbByVars;rootVar!=NULL;rootVar=rootVar->next) { if(rootVar==mdbByVars) dyStringPrintf(dy, " where (var "); else dyStringPrintf(dy, " OR (var "); if(rootVar->notEqual && rootVar->vals == NULL) - dyStringPrintf(dy, "%s",strchr(rootVar->var,'%')?"NOT ":"!"); // one of: "NOT LIKE". "!=" or "NOT EXISTS" + dyStringPrintf(dy, "%s",strchr(rootVar->var,'%')?"NOT ":"!"); + // one of: "NOT LIKE". "!=" or "NOT EXISTS" if(rootVar->vals != NULL && rootVar->vals->val != NULL && strlen(rootVar->vals->val) > 0) { dyStringPrintf(dy, "%s '%s'", (strchr(rootVar->var,'%')?"like":"="), rootVar->var); } else dyStringPrintf(dy, "EXISTS"); struct mdbLimbVal *limbVal; boolean multiVals = FALSE; for(limbVal=rootVar->vals;limbVal!=NULL;limbVal=limbVal->next) { if(limbVal->val == NULL || strlen(limbVal->val) < 1) continue; @@ -1300,91 +1316,124 @@ } dyStringPrintf(dy, " order by var, val, obj"); verbose(2, "Requesting query:\n\t%s;\n",dyStringContents(dy)); struct mdb *mdb = mdbLoadByQuery(conn, dyStringCannibalize(&dy)); verbose(3, "rows (vars) returned: %d\n",slCount(mdb)); struct mdbByVar *mdbByVarsFromMem = mdbByVarsLoadFromMemory(&mdb,TRUE); verbose(3, "Returned %d vars(s) with %d val(s) with %d object(s).\n", mdbByVarCount(mdbByVarsFromMem,TRUE ,FALSE), mdbByVarCount(mdbByVarsFromMem,FALSE,TRUE ), mdbByVarCount(mdbByVarsFromMem,FALSE,FALSE)); return mdbByVarsFromMem; } struct mdbByVar *mdbByVarQueryByVar(struct sqlConnection *conn,char *table,char *varName,char *val) -// Query a single metadata variable and optional val from a table (default mdb) for searching val->obj. +// Query a single metadata variable & optional val from a table (default mdb) for searching val->obj { if(varName == NULL) return mdbByVarsQuery(conn,table,NULL); struct mdbByVar *queryVar = mdbByVarCreate(varName,val); struct mdbByVar *resultVar = mdbByVarsQuery(conn,table,queryVar); mdbByVarsFree(&queryVar); return resultVar; } struct mdbObj *mdbObjsQueryByVars(struct sqlConnection *conn,char *table,struct mdbByVar *mdbByVars) -// Query the metadata table by one or more var=val pairs to find the distinct set of objs that satisfy ALL conditions. +// Query the metadata table by one or more var=val pairs +// to find the distinct set of objs that satisfy ALL conditions. // Returns new mdbObj struct fully populated and sorted in obj,var order. { // MOST POPULAR WAY TO QUERY MDB. Building example queries like: // "cell=GM12878" or "cell!=GM12878" -// SELECT T1.obj,T1.var,T1.val FROM metaDb T1 WHERE EXISTS (SELECT T2.obj FROM metaDb T2 WHERE T2.obj = T1.obj AND T2.var = 'cell' AND T2.val = 'GM12878') ORDER BY T1.obj, T1.var; -// SELECT T1.obj,T1.var,T1.val FROM metaDb T1 WHERE EXISTS (SELECT T2.obj FROM metaDb T2 WHERE T2.obj = T1.obj AND T2.var = 'cell' AND T2.val != 'GM12878') ORDER BY T1.obj, T1.var; +// SELECT T1.obj,T1.var,T1.val FROM metaDb T1 +// WHERE EXISTS (SELECT T2.obj FROM metaDb T2 +// WHERE T2.obj = T1.obj AND T2.var = 'cell' AND T2.val = 'GM12878') +// ORDER BY T1.obj, T1.var; +// SELECT T1.obj,T1.var,T1.val FROM metaDb T1 +// WHERE EXISTS (SELECT T2.obj FROM metaDb T2 +// WHERE T2.obj = T1.obj AND T2.var = 'cell' AND T2.val != 'GM12878') +// ORDER BY T1.obj, T1.var; // "cell=GM%" or "cell!=GM%" -// SELECT T1.obj,T1.var,T1.val FROM metaDb T1 WHERE EXISTS (SELECT T2.obj FROM metaDb T2 WHERE T2.obj = T1.obj AND T2.var = 'cell' AND T2.val LIKE 'GM%') ORDER BY T1.obj, T1.var; -// SELECT T1.obj,T1.var,T1.val FROM metaDb T1 WHERE EXISTS (SELECT T2.obj FROM metaDb T2 WHERE T2.obj = T1.obj AND T2.var = 'cell' AND T2.val NOT LIKE 'GM%') ORDER BY T1.obj, T1.var; +// SELECT T1.obj,T1.var,T1.val FROM metaDb T1 +// WHERE EXISTS (SELECT T2.obj FROM metaDb T2 +// WHERE T2.obj = T1.obj AND T2.var = 'cell' AND T2.val LIKE 'GM%') +// ORDER BY T1.obj, T1.var; +// SELECT T1.obj,T1.var,T1.val FROM metaDb T1 +// WHERE EXISTS (SELECT T2.obj FROM metaDb T2 +// WHERE T2.obj = T1.obj AND T2.var = 'cell' AND T2.val NOT LIKE 'GM%') +// ORDER BY T1.obj, T1.var; // "cell=" or "cell!=" -// SELECT T1.obj,T1.var,T1.val FROM metaDb T1 WHERE EXISTS (SELECT T2.obj FROM metaDb T2 WHERE T2.obj = T1.obj AND T2.var = 'cell') ORDER BY T1.obj, T1.var; -// SELECT T1.obj,T1.var,T1.val FROM metaDb T1 WHERE NOT EXISTS (SELECT T2.obj FROM metaDb T2 WHERE T2.obj = T1.obj AND T2.var = 'cell') ORDER BY T1.obj, T1.var; +// SELECT T1.obj,T1.var,T1.val FROM metaDb T1 +// WHERE EXISTS (SELECT T2.obj FROM metaDb T2 +// WHERE T2.obj = T1.obj AND T2.var = 'cell') +// ORDER BY T1.obj, T1.var; +// SELECT T1.obj,T1.var,T1.val FROM metaDb T1 +// WHERE NOT EXISTS (SELECT T2.obj FROM metaDb T2 +// WHERE T2.obj = T1.obj AND T2.var = 'cell') +// ORDER BY T1.obj, T1.var; // "cell=GM12878,K562" or "cell!=GM12878,K562" -// SELECT T1.obj,T1.var,T1.val FROM metaDb T1 WHERE EXISTS (SELECT T2.obj FROM metaDb T2 WHERE T2.obj = T1.obj AND T2.var = 'cell' AND T2.val IN ('GM12878','K562')) ORDER BY T1.obj, T1.var; -// SELECT T1.obj,T1.var,T1.val FROM metaDb T1 WHERE EXISTS (SELECT T2.obj FROM metaDb T2 WHERE T2.obj = T1.obj AND T2.var = 'cell' AND T2.val NOT IN ('K562','GM12878')) ORDER BY T1.obj, T1.var; +// SELECT T1.obj,T1.var,T1.val FROM metaDb T1 +// WHERE EXISTS (SELECT T2.obj FROM metaDb T2 +// WHERE T2.obj = T1.obj AND T2.var = 'cell' AND T2.val IN ('GM12878','K562')) +// ORDER BY T1.obj, T1.var; +// SELECT T1.obj,T1.var,T1.val FROM metaDb T1 +// WHERE EXISTS (SELECT T2.obj FROM metaDb T2 +// WHERE T2.obj = T1.obj AND T2.var = 'cell' +// AND T2.val NOT IN ('K562','GM12878')) +// ORDER BY T1.obj, T1.var; // "cell=GM% cell!=GM12878" (very powerful) -// SELECT T1.obj,T1.var,T1.val FROM metaDb T1 WHERE EXISTS (SELECT T2.obj FROM metaDb T2 WHERE T2.obj = T1.obj AND T2.var = 'cell' AND T2.val LIKE 'GM%') -// AND EXISTS (SELECT T3.obj FROM metaDb T3 WHERE T3.obj = T1.obj AND T3.var = 'cell' AND T3.val != 'GM12878') ORDER BY T1.obj, T1.var; +// SELECT T1.obj,T1.var,T1.val FROM metaDb T1 +// WHERE EXISTS (SELECT T2.obj FROM metaDb T2 +// WHERE T2.obj = T1.obj AND T2.var = 'cell' AND T2.val LIKE 'GM%') +// AND EXISTS (SELECT T3.obj FROM metaDb T3 +// WHERE T3.obj = T1.obj AND T3.var = 'cell' AND T3.val != 'GM12878') +// ORDER BY T1.obj, T1.var; if(table == NULL) - table = mdbTableName(conn,TRUE); // defaults to sandbox, if it exists, else MDB_DEFAULT_NAME; + table = mdbTableName(conn,TRUE); // defaults to sandbox, if exists, else MDB_DEFAULT_NAME else if(!sqlTableExists(conn,table)) return NULL; struct dyString *dy = newDyString(4096); dyStringPrintf(dy, "SELECT T1.obj,T1.var,T1.val FROM %s T1", table); struct mdbByVar *rootVar; boolean gotVar = FALSE; int tix; for(rootVar=mdbByVars,tix=2;rootVar!=NULL;rootVar=rootVar->next,tix++) { boolean hasVal = (rootVar->vals != NULL); - //boolean hasVal = (rootVar->vals != NULL && rootVar->vals->val != NULL && strlen(rootVar->vals->val) > 0); + //boolean hasVal = ( rootVar->vals != NULL + // && rootVar->vals->val != NULL + // && strlen(rootVar->vals->val) > 0); if(!gotVar) { dyStringPrintf(dy, " WHERE "); gotVar=TRUE; } else dyStringPrintf(dy, " AND "); if(!hasVal && rootVar->notEqual) dyStringPrintf(dy, "NOT EXISTS "); else dyStringPrintf(dy, "EXISTS "); - dyStringPrintf(dy, "(SELECT T%d.obj FROM %s T%d WHERE T%d.obj = T1.obj AND T%d.var ",tix,table,tix,tix,tix); + dyStringPrintf(dy, "(SELECT T%d.obj FROM %s T%d WHERE T%d.obj = T1.obj AND T%d.var ", + tix,table,tix,tix,tix); if(hasVal && rootVar->notEqual && rootVar->vals == NULL) dyStringPrintf(dy, "%s",strchr(rootVar->var,'%')?"NOT ":"!"); dyStringPrintf(dy, "%s '%s'", (strchr(rootVar->var,'%')?"LIKE":"="), rootVar->var); struct mdbLimbVal *limbVal; boolean multiVals = (rootVar->vals != NULL && rootVar->vals->next != NULL); boolean wilds = FALSE; for(limbVal=rootVar->vals;limbVal!=NULL;limbVal=limbVal->next) { if (strchr(limbVal->val,'%') != NULL) wilds = TRUE; } @@ -1426,56 +1475,59 @@ dyStringPrintf(dy, ")"); dyStringPrintf(dy, ")"); } dyStringPrintf(dy, " ORDER BY binary T1.obj, T1.var"); // binary forces case sensitive sort verbose(2, "Requesting query:\n\t%s;\n",dyStringContents(dy)); struct mdb *mdb = mdbLoadByQuery(conn, dyStringCannibalize(&dy)); verbose(3, "rows (vars) returned: %d\n",slCount(mdb)); struct mdbObj *mdbObjs = mdbObjsLoadFromMemory(&mdb,TRUE); verbose(3, "Returned %d object(s) with %d var(s).\n", mdbObjCount(mdbObjs,TRUE),mdbObjCount(mdbObjs,FALSE)); return mdbObjs; } struct mdbObj *mdbObjsQueryByVarValString(struct sqlConnection *conn,char *tableName,char *varVals) -// returns mdbObjs matching varVals in form of: [var1=val1 var2=val2a,val2b var3=v%3 var4="val 4" var5!=val5 var6=] +// returns mdbObjs matching varVals in form of: +// [var1=val1 var2=val2a,val2b var3=v%3 var4="val 4" var5!=val5 var6=] // var2=val2a,val2b: matches asny of comma separated list // var3=v%3 : matches '%' and '?' wild cards. // var4="val 4" : matches simple double quoted strings. // var5!=val5 : matches not equal. // var6= : matches that var exists (same as var6=%). var6!= also works. { struct mdbByVar *mdbByVars = mdbByVarsLineParse(varVals); if (mdbByVars == NULL) return NULL; return mdbObjsQueryByVars(conn,tableName,mdbByVars); } -struct mdbObj *mdbObjsQueryByVarPairs(struct sqlConnection *conn,char *tableName,struct slPair *varValPairs) +struct mdbObj *mdbObjsQueryByVarPairs(struct sqlConnection *conn,char *tableName, + struct slPair *varValPairs) // returns mdbObjs matching varValPairs provided. // The != logic of mdbObjsQueryByVarValString() is not possible, but other cases are supported: // as val may be NULL, a comma delimited list, double quoted string, containing wilds: % and ? { -// Note: there is a bit of inefficiency creating a string then tearing it down, but it streamlines code +// Note: there is inefficiency in creating a string then tearing it down, but it streamlines code char *varValString = slPairListToString(varValPairs,TRUE); // quotes added when spaces found struct mdbObj *mdbObjs = mdbObjsQueryByVarValString(conn,tableName,varValString); freeMem(varValString); return mdbObjs; } -struct mdbObj *mdbObjQueryCompositeObj(struct sqlConnection *conn,char *tableName,struct mdbObj *mdbObj) +struct mdbObj *mdbObjQueryCompositeObj(struct sqlConnection *conn,char *tableName, + struct mdbObj *mdbObj) // returns NULL or the composite mdbObj associated with the object passed in. { char *objType = mdbObjFindValue(mdbObj,MDB_OBJ_TYPE); assert(objType != NULL); if (sameWord(objType,MDB_VAR_COMPOSITE)) return mdbObjClone(mdbObj); char *compName = mdbObjFindValue(mdbObj,MDB_VAR_COMPOSITE); if (compName == NULL) return NULL; return mdbObjQuery(conn,tableName,mdbObjCreate(compName,NULL,NULL)); } // ----------- Printing and Counting ----------- @@ -1502,34 +1554,34 @@ // prints objs and var=val pairs as formatted metadata lines or ra style { // Single line: // metadata iLoveLucy table lucy=ricky ethyl=fred // ra style // metadata iLoveLucy table // lucy ricky // ethy fred // TODO: Expand for mutilple var types; strip quotes from vals on ra style struct mdbObj *mdbObj = NULL; for(mdbObj=mdbObjs;mdbObj!=NULL;mdbObj=mdbObj->next) { if(mdbObj->obj == NULL) continue; + fprintf(outF, "%s %s",(raStyle?MDB_METAOBJ_RAKEY:MDB_METADATA_KEY),mdbObj->obj); if(mdbObj->deleteThis) fprintf(outF, " delete"); - struct mdbVar *mdbVar = NULL; // If hash available, force objType to front if(mdbObj->varHash != NULL) { mdbVar = hashFindVal(mdbObj->varHash,MDB_OBJ_TYPE); mdbVarValPrint(mdbVar,raStyle, outF); } for(mdbVar=mdbObj->vars;mdbVar!=NULL;mdbVar=mdbVar->next) { if(mdbObj->varHash == NULL || !sameOk(MDB_OBJ_TYPE,mdbVar->var)) mdbVarValPrint(mdbVar,raStyle, outF); } fprintf(outF, "%s",(raStyle?"\n\n":"\n")); } @@ -1586,31 +1638,32 @@ // prints objs and var=val pairs as formatted metadata lines or ra style { mdbObjPrintToStream(mdbObjs, raStyle, stdout); } void mdbObjPrintToFile(struct mdbObj *mdbObjs,boolean raStyle, char *file) // prints (to file) objs and var=val pairs as formatted metadata lines or ra style { FILE *f = mustOpen(file, "w"); mdbObjPrintToStream(mdbObjs, raStyle, f); fclose(f); } -void mdbObjPrintOrderedToStream(FILE *outF,struct mdbObj **mdbObjs,char *order, char *separator, boolean header) +void mdbObjPrintOrderedToStream(FILE *outF,struct mdbObj **mdbObjs,char *order, char *separator, + boolean header) // prints mdbObjs as a table, but only the vars listed in comma delimited order. // Examples of separator: " " "\t\t" or "<TD>", in which case this is an HTML table. // mdbObjs list will be reordered. Sort fails when vars are missing in objs. { if (separator == NULL) separator = " "; boolean html = FALSE; if (startsWith("<T",separator) || startsWith("<t",separator)) { if(!endsWith(separator,">")) errAbort("mdbObjPrintOrdered() separator is invalid HTML '%s'.\n",separator); html = TRUE; } if (!startsWithWordByDelimiter("obj" ,',',order) @@ -1639,33 +1692,36 @@ if (html) fprintf(outF, "</tr>"); fprintf(outF, "\n"); } struct mdbObj *mdbObj = *mdbObjs; for (;mdbObj != NULL; mdbObj = mdbObj->next) { if (html) fprintf(outF, "<tr>"); for (var = vars;var != NULL; var = var->next) { char *val = mdbObjFindValue(mdbObj, var->name); if (val == NULL) { - /*if (sameWord(var->name,"obj") || sameWord(var->name,"objName") || sameWord(var->name,"metaObject")) + /*if (sameWord(var->name,"obj") + || sameWord(var->name,"objName") + || sameWord(var->name,"metaObject")) val = mdbObj->obj; - else*/ if (html) + else*/ + if (html) val = " "; else val = " "; } if (html) fprintf(outF, "%s%s",separator,val); // <td> is first else fprintf(outF, "%s%s",val,separator); if (html) fprintf(outF, "</td>"); } if (html) fprintf(outF, "</tr>"); fprintf(outF, "\n"); } @@ -1919,31 +1975,32 @@ #define MDB_COMMON_VARS_OBJ_SEARCH_LIMIT 10 struct mdbObj *mdbObjsCommonVars(struct mdbObj *mdbObjs) // Returns a new mdbObj with all vars that are contained in every obj passed in. // Note that the returnd mdbObj has a meaningles obj name and vals. { if (mdbObjs == NULL || mdbObjs->vars == NULL) return NULL; struct mdbObj *mdbObj = mdbObjs; struct mdbObj *commonVars = mdbObjClone(mdbObj); // Clone the first obj then prune it commonVars->next = NULL; mdbObj=mdbObj->next; // No need to include first obj in search if (mdbObj != NULL) { int count = 1; - // NOTE: This should not loop through all, as the list could be huge. Just compare the first 10 for now + // NOTE: This should not loop through all, as the list could be huge. + // Just compare the first 10 for now struct dyString *dyPruneVars = dyStringNew(512); for(;mdbObj != NULL && count < MDB_COMMON_VARS_OBJ_SEARCH_LIMIT;mdbObj=mdbObj->next, count++) { struct mdbVar *mdbVar = commonVars->vars; // Will walk through the first obj's vars for(; mdbVar != NULL; mdbVar = mdbVar->next ) { if (mdbObjsContainAtleastOne(mdbObj, mdbVar->var) == FALSE) dyStringPrintf(dyPruneVars,"%s ",mdbVar->var); // var not found so add to prune list } if (dyStringLen(dyPruneVars) > 0) { mdbObjRemoveVars(commonVars,dyStringContents(dyPruneVars)); dyStringClear(dyPruneVars); } } @@ -2019,32 +2076,32 @@ struct mdbVar **varsToReorder = needMem(sizeof(struct mdbVar *) * count); struct mdbVar *mdbVar = NULL; while((mdbVar = slPopHead(&(mdbObj->vars))) != NULL) { ix = stringArrayIx(mdbVar->var,words,count); // Is case insensitive if(ix < 0) slAddHead(&orderedVars,mdbVar); else varsToReorder[ix] = mdbVar; } if(back) // add to front of backward list { for( ix=0; ix<count; ix++ ) - { - if(varsToReorder[ix] != NULL) // NOTE: For NULL, could add "None" but that would be too much "inside ball" + { // NOTE: For NULL, could add "None" + if (varsToReorder[ix] != NULL) // but that would be too much "inside ball" slAddHead(&orderedVars,varsToReorder[ix]); } } slReverse(&orderedVars); if(!back) // Add to front of forward list { for( ix=count-1; ix>=0; ix-- ) { if(varsToReorder[ix] != NULL) slAddHead(&orderedVars,varsToReorder[ix]); } } mdbObj->vars = orderedVars; @@ -2054,125 +2111,130 @@ } void mdbObjReorderByCv(struct mdbObj *mdbObjs, boolean includeHidden) // Reorders vars list based upon cv.ra typeOfTerms priority { struct hash *cvTermTypes = (struct hash *)cvTermTypeHash(); struct hashEl *el, *elList = hashElListHash(cvTermTypes); struct slPair *cvVars = NULL; for (el = elList; el != NULL; el = el->next) { struct hash *varHash = el->val; if (includeHidden || !cvTermIsHidden(el->name)) // Skip the hidden ones { char *priority = hashFindVal(varHash, CV_TOT_PRIORITY); - if (priority != NULL) // If there is no priority it will randomly fall to the back of the list + if (priority != NULL) // If no priority it will randomly fall to the back of the list slPairAdd(&cvVars,el->name,(char *)sqlUnsignedLong(priority)); } } hashElFreeList(&elList); if (cvVars) { slPairIntSort(&cvVars); // sorts on the integer val // Now convert this to a string of names char *orderedVars = slPairNameToString(cvVars,' ',FALSE); slPairFreeList(&cvVars); if (orderedVars != NULL) { - mdbObjReorderVars(mdbObjs, orderedVars,FALSE); // Finally we can reorder the vars in the mdbObjs + mdbObjReorderVars(mdbObjs, orderedVars,FALSE); // Finally we can reorder the vars freeMem(orderedVars); } } } int mdbObjVarCmp(const void *va, const void *vb) /* Compare to sort on full list of vars and vals. */ { const struct mdbObj *a = *((struct mdbObj **)va); const struct mdbObj *b = *((struct mdbObj **)vb); struct mdbVar* aVar = a->vars; struct mdbVar* bVar = b->vars; for(;aVar != NULL && bVar != NULL;aVar=aVar->next,bVar=bVar->next) { int ret = differentWord(aVar->var, bVar->var); // case insensitive if(ret != 0) { // Look for it by walking vars struct mdbVar* tryVar = bVar->next; for(;tryVar;tryVar=tryVar->next) { if (sameWord(aVar->var, tryVar->var)) - return -1; // Current aVar found in B so B has extra var and A has NULL: A sorts first + return -1; // Current aVar found in B so B has extra var & A has NULL: A sorts first } tryVar = aVar->next; for(;tryVar;tryVar=tryVar->next) { if (sameWord(tryVar->var, bVar->var)) - return 1; // Current bVar found in A so A has extra var and B has NULL: B sorts first - } - return ret; // Current aVar and bVar are not shared so prioritize them alphabetically (What else can I do?) + return 1; // Current bVar found in A so A has extra var & B has NULL: B sorts first } + return ret; // Current aVar and bVar are not shared so prioritize them alphabetically + } // (What else can I do?) ret = differentString(aVar->val, bVar->val); // case sensitive on val if(ret != 0) return ret; } if(bVar != NULL) return -1; // B has extra var and A has NULL: A sorts first if(aVar != NULL) return 1; // A has extra var and B has NULL: B sorts first return 0; } void mdbObjsSortOnVars(struct mdbObj **mdbObjs, char *vars) -// Sorts on var,val pairs vars lists: fwd case-sensitive. Assumes all objs' vars are in identical order. -// Optionally give list of vars "cell antibody treatment" to sort on (bringing to front of vars lists). -{ // NOTE: assumes all var pairs match (e.g. every obj has cell,treatment,antibody,... and missing treatment messes up sort) +// Sorts on var,val pairs vars lists: fwd case-sensitive. Assumes objs' vars are in identical order +// Optionally give list of vars "cell antibody treatment" to sort on (bringing to front of lists). +// NOTE: assumes all var pairs match (e.g. every obj has cell,treatment,antibody,... +// and missing treatment messes up sort) +{ if(vars != NULL) mdbObjReorderVars(*mdbObjs,vars,FALSE); slSort(mdbObjs, mdbObjVarCmp); } void mdbObjsSortOnVarPairs(struct mdbObj **mdbObjs,struct slPair *varValPairs) -// Sorts on var,val pairs vars lists: fwd case-sensitive. Assumes all objs' vars are in identical order. +// Sorts on var,val pairs vars lists: fwd case-sensitive. +// Assumes all objs' vars are in identical order. // This method will use mdbObjsSortOnVars() { if (varValPairs == NULL) return; struct slPair *onePair = varValPairs; struct dyString *dyTerms = dyStringNew(256); dyStringAppend(dyTerms,onePair->name); onePair = onePair->next; for(; onePair != NULL; onePair = onePair->next) dyStringPrintf(dyTerms,",%s",onePair->name); mdbObjsSortOnVars(mdbObjs,dyStringContents(dyTerms)); dyStringFree(&dyTerms); } void mdbObjsSortOnCv(struct mdbObj **mdbObjs, boolean includeHidden) // Puts obj->vars in order based upon cv.ra typeOfTerms priority, // then case-sensitively sorts all objs in list based upon that var order. -{ // NOTE: assumes all var pairs match (e.g. every obj has cell,treatment,antibody,... and missing treatment messes up sort) +// NOTE: assumes all var pairs match (e.g. every obj has cell,treatment,antibody,... +// and missing treatment messes up sort) +{ mdbObjReorderByCv(*mdbObjs, includeHidden); -slSort(mdbObjs, mdbObjVarCmp); // While sort will not be perfect (given missing values) it does a good job none the less. -} +slSort(mdbObjs, mdbObjVarCmp); // While sort will not be perfect (given missing values) +} // it does a good job none the less. boolean mdbObjRemoveOneVar(struct mdbObj *mdbObj, char *var, char *val) // returns TRUE if var (and optional val) are found and surgically removed from one mdbObj { struct mdbVar *lastVar = NULL; struct mdbVar *mdbVar = mdbObj->vars; for(;mdbVar != NULL;lastVar=mdbVar,mdbVar=mdbVar->next) { if (sameWord(mdbVar->var,var)) { if (val && differentString(mdbVar->val,val)) break; // No need to continue if (lastVar != NULL) lastVar->next = mdbVar->next; else @@ -2350,125 +2412,135 @@ for( mdbObj=mdbObjs; mdbObj!=NULL; mdbObj=mdbObj->next ) { mdbObj->deleteThis = deleteThis; if (mdbObj->varHash != NULL) hashFree(&mdbObj->varHash); mdbVarsFree(&(mdbObj->vars)); if (vars != NULL) mdbObjAddVarPairs(mdbObj,vars); } } struct mdbObj *mdbObjsFilter(struct mdbObj **pMdbObjs, char *var, char *val,boolean returnMatches) -// Filters mdb objects to only those that include/exclude vars. Optionally checks (case insensitive) val too. +// Filters mdb objects to only those that include/exclude vars. +// Optionally checks (case insensitive) val too. // Returns matched or unmatched items objects as requested, maintaining sort order { struct mdbObj *mdbObjsReturned = NULL; struct mdbObj *mdbObjs = *pMdbObjs; *pMdbObjs = NULL; boolean wildValMatch = (val != NULL && strchr(val,'*') != NULL); -struct mdbObj **pMatchTail = returnMatches ? &mdbObjsReturned : pMdbObjs; // Slightly faster than slAddHead/slReverse -struct mdbObj **pNoMatchTail = returnMatches ? pMdbObjs : &mdbObjsReturned; // Also known as too clever by half + // pMatchTail: Slightly faster than slAddHead/slReverse +struct mdbObj **pMatchTail = returnMatches ? &mdbObjsReturned : pMdbObjs; + // pNoMatchTail: Also known as too clever by half +struct mdbObj **pNoMatchTail = returnMatches ? pMdbObjs : &mdbObjsReturned; while (mdbObjs!=NULL) { boolean match = FALSE; struct mdbObj *obj = slPopHead(&mdbObjs); char *foundVal = mdbObjFindValue(obj,var); // Case sensitive (unfortunately) if (val == NULL) match = (foundVal != NULL); // any val will match else if (foundVal) { if (wildValMatch) match = (wildMatch(val,foundVal)); else match = (sameWord(foundVal,val)); // must be same val (case insensitive) } if (match) { *pMatchTail = obj; pMatchTail = &((*pMatchTail)->next); } else { *pNoMatchTail = obj; pNoMatchTail = &((*pNoMatchTail)->next); } } return mdbObjsReturned; } -struct mdbObj *mdbObjsFilterByVars(struct mdbObj **pMdbObjs,char *vars,boolean noneEqualsNotFound,boolean returnMatches) -// Filters mdb objects to only those that include/exclude var=val pairs (e.g. "var1=val1 var2 var3!=val3 var4=None"). +struct mdbObj *mdbObjsFilterByVars(struct mdbObj **pMdbObjs,char *vars, + boolean noneEqualsNotFound,boolean returnMatches) +// Filters mdb objects to only those that include/exclude var=val pairs +// (e.g. "var1=val1 var2 var3!=val3 var4=None"). // Supports != ("var!=" means var not found). Optionally supports var=None equal to var is not found -// Returns matched or unmatched items objects as requested. Multiple passes means sort order is destroyed. +// Returns matched or unmatched items objects as requested. +// Multiple passes means sort order is destroyed. { struct mdbObj *mdbObjsMatch = *pMdbObjs; struct mdbObj *mdbObjsNoMatch = NULL; char *varsLine = cloneString(vars); int ix=0,count = chopByWhite(varsLine,NULL,0); char **var = needMem(count * sizeof(char *)); chopByWhite(varsLine,var,count); for(ix=0;ix<count;ix++) { boolean notEqual = FALSE; char *val = strchr(var[ix],'='); // list may be vars alone! (var1=val1 var2 var3!=val3 var4=None) if (val != NULL) { notEqual = (*(val - 1) == '!'); if (notEqual) *(val - 1) = '\0'; *val = '\0'; val += 1; if (*val == '\0') val = NULL; } - struct mdbObj *objNotMatching = mdbObjsFilter(&mdbObjsMatch,var[ix],val,notEqual); // exclude non-matching + struct mdbObj *objNotMatching = mdbObjsFilter(&mdbObjsMatch,var[ix],val,notEqual); + // 1st match on var=None, now match on var!= (var not defined) if (noneEqualsNotFound && val != NULL && sameWord(val,MDB_VAL_ENCODE_EDV_NONE)) - mdbObjsMatch = slCat(mdbObjsMatch,mdbObjsFilter(&objNotMatching,var[ix],NULL,notEqual)); // 1st match on var=None, now match on var!= (var not defined) - mdbObjsNoMatch = slCat(mdbObjsNoMatch,objNotMatching); // Multiple passes "cat" non-matching and destroys sort order - } + mdbObjsMatch = slCat(mdbObjsMatch,mdbObjsFilter(&objNotMatching,var[ix],NULL,notEqual)); + mdbObjsNoMatch = slCat(mdbObjsNoMatch,objNotMatching); // Multiple passes "cat" non-matching + } // and destroys sort order freeMem(var); freeMem(varsLine); if (returnMatches) { *pMdbObjs = mdbObjsNoMatch; return mdbObjsMatch; } *pMdbObjs = mdbObjsMatch; return mdbObjsNoMatch; } struct mdbObj *mdbObjsFilterTablesOrFiles(struct mdbObj **pMdbObjs,boolean tables, boolean files) -// Filters mdb objects to only those that have associated tables or files. Returns removed non-table/file objects +// Filters mdb objects to only those that have associated tables or files. +// Returns removed non-table/file objects // Note: Since table/file objects overlap, there are 3 possibilites: tables, files, table && files { assert(tables || files); // Cant exclude both struct mdbObj *mdbObjs = *pMdbObjs; struct mdbObj *mdbObjsDropped = NULL; if (tables) mdbObjsDropped = mdbObjsFilter(&mdbObjs,MDB_OBJ_TYPE,MDB_OBJ_TYPE_TABLE,FALSE); if (files) { - struct mdbObj *mdbObjsNoFileName = mdbObjsDropped = mdbObjsFilter(&mdbObjs,MDB_VAR_FILENAME,NULL,FALSE); + struct mdbObj *mdbObjsNoFileName = mdbObjsDropped = mdbObjsFilter(&mdbObjs,MDB_VAR_FILENAME, + NULL,FALSE); if (mdbObjsNoFileName) { - struct mdbObj *mdbObjsNoFileIndex = mdbObjsFilter(&mdbObjsNoFileName,MDB_VAR_FILEINDEX,NULL,FALSE); + struct mdbObj *mdbObjsNoFileIndex = mdbObjsFilter(&mdbObjsNoFileName,MDB_VAR_FILEINDEX, + NULL,FALSE); if (mdbObjsNoFileIndex) { mdbObjs = slCat(mdbObjs,mdbObjsNoFileName); mdbObjsDropped = slCat(mdbObjsDropped,mdbObjsNoFileIndex); } } } slSort(&mdbObjs, &mdbObjCmp); // Need to be returned to obj order slSort(&mdbObjsDropped,&mdbObjCmp); *pMdbObjs = mdbObjs; return mdbObjsDropped; } struct mdbObj *mdbObjIntersection(struct mdbObj **pA, struct mdbObj *b) @@ -2646,385 +2718,422 @@ char *edvs = mdbObjFindValue(compObj,MDB_VAR_ENCODE_EDVS); if (edvs == NULL) return NULL; edvs = cloneString(edvs); if (strchr( edvs,',') != NULL) // Tolerate delimit by commas strSwapChar(edvs,',',' '); else if (strchr(edvs,';') != NULL) // Tolerate delimit by semicolons strSwapChar(edvs,';',' '); struct slName *compositeEdvs = slNameListFromString(edvs,' '); freeMem(edvs); return compositeEdvs; } -static struct mdbVar *mdbObjEncodeEdvsAsMdbVars(struct mdbObj *mdbObj,struct slName *compositeEdvs,boolean includeNone) +static struct mdbVar *mdbObjEncodeEdvsAsMdbVars(struct mdbObj *mdbObj,struct slName *compositeEdvs, + boolean includeNone) // returns the EDVs and values for the composite member object // If includeNone, then defined variables not found in obj will be included as {var}="None". { struct mdbVar *edvVars = NULL; struct slName *var = compositeEdvs; for(;var!=NULL;var=var->next) { char *val = mdbObjFindValue(mdbObj,var->name); if (val) mdbVarAdd(&edvVars, var->name,val); else if (includeNone) { - if (differentWord(var->name,ENCODE_EXP_FIELD_ORGANISM)) // Does not go into EDV's sent to encodeExp table - mdbVarAdd(&edvVars, var->name,MDB_VAL_ENCODE_EDV_NONE); + if (differentWord(var->name,ENCODE_EXP_FIELD_ORGANISM)) // Does not go into EDV's + mdbVarAdd(&edvVars, var->name,MDB_VAL_ENCODE_EDV_NONE); // sent to encodeExp table } } slReverse(&edvVars); return edvVars; } -struct slName *mdbObjFindCompositeNamedEncodeEdvs(struct sqlConnection *conn,char *tableName,struct mdbObj *mdbObj) +struct slName *mdbObjFindCompositeNamedEncodeEdvs(struct sqlConnection *conn,char *tableName, + struct mdbObj *mdbObj) // returns NULL or the Experiment Defining Variable names for this composite { if (!mdbObjIsCompositeMember(mdbObj)) return NULL; // This should be a valid composite memeber struct mdbObj *compObj = mdbObjQueryCompositeObj(conn,tableName,mdbObj); if (compObj == NULL) return NULL; struct slName *edvs = mdbObjGetNamedEncodeEdvs(compObj); mdbObjFree(&compObj); return edvs; } -struct mdbVar *mdbObjFindEncodeEdvPairs(struct sqlConnection *conn,char *tableName,struct mdbObj *mdbObj,boolean includeNone) +struct mdbVar *mdbObjFindEncodeEdvPairs(struct sqlConnection *conn,char *tableName, + struct mdbObj *mdbObj,boolean includeNone) // returns NULL or the Experiment Defining Variables and values for this composite member object // If includeNone, then defined variables not found in obj will be included as {var}="None". { // In rare cases, the EDVs reside with the object and NOT in a objType=composite. struct slName *compositeEdvs = mdbObjGetNamedEncodeEdvs(mdbObj); // looking locally first. if (compositeEdvs == NULL) { compositeEdvs = mdbObjFindCompositeNamedEncodeEdvs(conn,tableName,mdbObj); if (compositeEdvs == NULL) return NULL; } return mdbObjEncodeEdvsAsMdbVars(mdbObj,compositeEdvs,includeNone); } -struct mdbObj *mdbObjsEncodeExperimentify(struct sqlConnection *conn,char *db,char *tableName,char *expTable, - struct mdbObj **pMdbObjs,int warn,boolean createExpIfNecessary,boolean updateAccession) -// Organizes objects into experiments and validates experiment IDs. Will add/update the ids in the structures. +struct mdbObj *mdbObjsEncodeExperimentify(struct sqlConnection *conn,char *db,char *tableName, + char *expTable, struct mdbObj **pMdbObjs,int warn, + boolean createExpIfNecessary,boolean updateAccession) +// Organizes objects into experiments and validates experiment IDs. +// Will add/update the ids in the structures. // If warn=1, then prints to stdout all the experiments/obs with missing or wrong expIds; // warn=2, then print line for each obj with expId or warning. // createExpIfNecessary means add expId to encodeExp table. updateAccession too if necessary. -// Returns a new set of mdbObjs that is what can (and should) be used to update the mdb via mdbObjsSetToDb(). +// Returns a new set of mdbObjs that is what can (and should) +// be used to update the mdb via mdbObjsSetToDb(). { // Here is what "experimentify" does from "mdbPrint -encodeExp" and "mdbUpdate -encodeExp": -// - Uses normal selection methods to get a set of objects (e.g. one composite worth) or all objs. (in mdbPrint and mdbUpdate) +// - Uses normal selection methods to get a set of objects (e.g. one composite worth) +// or all objs. (in mdbPrint and mdbUpdate) // - This API: // - Breaks up and walks through set of objects composite by composite // - Looks up EDVs (Experiment Defining Variables) for composite. // These are defined in the mdb under objType=composite expVars= // - Breaks up and walks through composite objects exp by exp as defined by EDVs // - Uses encodeExp API to determine what expId should be. // - Creates new mdbObjs list of updates needed to put expId and dccAccession into the mdb. // - From "mdbPrint", this API warns of mismatches or missing expIds -// - From "mdbUpdate" (not -test) then that utility will update the mdb from this API's return structs. If -test, will reveal what would be updated. -// - FIXME: Need to extend this to update dccAccession separately from expId. +// - From "mdbUpdate" (not -test) then that utility will update the mdb from this API's +// return structs. If -test, will reveal what would be updated. if (pMdbObjs == NULL || *pMdbObjs == NULL) return 0; struct mdbObj *mdbObjs = *pMdbObjs; struct mdbObj *mdbProcessedObs = NULL; struct mdbObj *mdbUpdateObjs = NULL; if (expTable == NULL) expTable = ENCODE_EXP_TABLE; verbose(2, "mdbObjsEncodeExperimentify() beginning for %d objects.\n",slCount(*pMdbObjs)); // Sort all objects by composite, so that we handle composite by composite mdbObjsSortOnVars(&mdbObjs, MDB_VAR_COMPOSITE); struct dyString *dyVars = dyStringNew(256); while(mdbObjs != NULL) { // Work on a composite at a time boolean compositelessObj = FALSE; char *compName = NULL; while(mdbObjs != NULL && compName == NULL) { compName = mdbObjFindValue(mdbObjs,MDB_VAR_COMPOSITE); if (compName == NULL) { if (mdbObjFindValue(mdbObjs,MDB_VAR_ENCODE_EDVS) == NULL) { - verbose(1, "Object '%s' has no %s or %s defined.\n",mdbObjs->obj,MDB_VAR_COMPOSITE,MDB_VAR_ENCODE_EDVS); + verbose(1, "Object '%s' has no %s or %s defined.\n", + mdbObjs->obj,MDB_VAR_COMPOSITE,MDB_VAR_ENCODE_EDVS); mdbProcessedObs = slCat(mdbProcessedObs,slPopHead(&mdbObjs)); continue; } verbose(2, "mdbObjsEncodeExperimentify() starting on compositeless set.\n"); break; } } struct mdbObj *mdbCompositeObjs = NULL; if (compName != NULL) mdbCompositeObjs = mdbObjsFilter(&mdbObjs, MDB_VAR_COMPOSITE, compName,TRUE); else mdbCompositeObjs = slPopHead(&mdbObjs); // Rare cases there is no composite set. assert(mdbCompositeObjs != NULL); - // --- At this point we have nibbled off a composite worth of objects from the full set of objects + // --- At this point we have nibbled off a composite worth of objs from the full set of objects // Find the composite obj if it exists struct mdbObj *compObj = NULL; if (compName != NULL) { compObj =mdbObjsFilter(&mdbCompositeObjs, MDB_OBJ_TYPE, MDB_OBJ_TYPE_COMPOSITE,TRUE); - if (compObj == NULL) // May be NULL if mdbObjs passed in was produced by too narrow of selection criteria - { - compObj = mdbObjQueryCompositeObj(conn,tableName,mdbCompositeObjs); // First obj on list will do + if (compObj == NULL) // May be NULL if mdbObjs passed in was produced by + { // too narrow of selection criteria + compObj = mdbObjQueryCompositeObj(conn,tableName,mdbCompositeObjs); // 1st obj will do if(compObj == NULL) // This should be assertable { verbose(1, "Composite '%s' has not been defined.\n",compName); mdbProcessedObs = slCat(mdbProcessedObs,mdbCompositeObjs); mdbCompositeObjs = NULL; continue; } } else - slAddHead(&mdbProcessedObs,compObj); // We can still use the pointer, but will not "process" it. NOTE: leak the queried one + slAddHead(&mdbProcessedObs,compObj); + // We can still use the pointer, but will not "process" it. NOTE: leak the queried one } else { compObj = mdbCompositeObjs; // Should be only one compName = mdbCompositeObjs->obj; compositelessObj = TRUE; } - verbose(2, "mdbObjsEncodeExperimentify() working on %s %s%s.\n",compName,MDB_VAR_COMPOSITE,(compositelessObj?"less set":"")); + verbose(2, "mdbObjsEncodeExperimentify() working on %s %s%s.\n", + compName,MDB_VAR_COMPOSITE,(compositelessObj?"less set":"")); // Obtain experiment defining variables for the composite (or compositeless obj) struct slName *compositeEdvs = mdbObjGetNamedEncodeEdvs(compObj); if (compositeEdvs == NULL) { - verbose(1, "There are no experiment defining variables established for this %s%s. Add them to obj %s => var:%s.\n", - MDB_VAR_COMPOSITE,(compositelessObj?"less set":""), compName,MDB_VAR_ENCODE_EDVS); + verbose(1, "There are no experiment defining variables established for this %s%s. " + "Add them to obj %s => var:%s.\n", MDB_VAR_COMPOSITE, + (compositelessObj?"less set":""), compName,MDB_VAR_ENCODE_EDVS); mdbProcessedObs = slCat(mdbProcessedObs,mdbCompositeObjs); mdbCompositeObjs = NULL; continue; } dyStringClear(dyVars); dyStringAppend(dyVars,slNameListToString(compositeEdvs, ' ')); if (warn > 0) - printf("Composite%s '%s' with %d objects has %d EDVs(%s): [%s].\n",(compositelessObj?"less set":""),compName, - slCount(mdbCompositeObjs),slCount(compositeEdvs),MDB_VAR_ENCODE_EDVS,dyStringContents(dyVars)); // Set the stage + printf("Composite%s '%s' with %d objects has %d EDVs(%s): [%s].\n", + (compositelessObj?"less set":""),compName,slCount(mdbCompositeObjs), + slCount(compositeEdvs),MDB_VAR_ENCODE_EDVS,dyStringContents(dyVars));// Set the stage // Organize composite objs by EDVs dyStringPrintf(dyVars, " %s %s ",MDB_VAR_VIEW,MDB_VAR_REPLICATE); // Allows for nicer sorted list char *edvSortOrder = cloneString(dyStringContents(dyVars)); // Walk through objs for an exp as defined by EDVs int expCount=0; // Count of experiments in composite int expMissing=0; // Count of objects with missing expId int accMissing=0; // Count of objects with missing accessions int expObjsCount=0; // Total of all experimental object accoss the composite int expMax=0; // Largest experiment (in number of objects) int expMin=999; // Smallest experiment (in number of objects) while(mdbCompositeObjs != NULL) { // Must sort each cycle, because sort order is lost during mdbObjs FilterByVars(); mdbObjsSortOnVars(&mdbCompositeObjs, edvSortOrder); // Get the EDVs for the first obj - struct mdbVar *edvVarVals = mdbObjEncodeEdvsAsMdbVars(mdbCompositeObjs,compositeEdvs,TRUE); // use first obj on list and include Nones + struct mdbVar *edvVarVals = mdbObjEncodeEdvsAsMdbVars(mdbCompositeObjs,compositeEdvs,TRUE); + // use first obj on list and include Nones if (edvVarVals == NULL) { - verbose(1, "There are no experiment defining variables for this object '%s'.\n",mdbCompositeObjs->obj); + verbose(1, "There are no experiment defining variables for this object '%s'.\n", + mdbCompositeObjs->obj); slAddHead(&mdbProcessedObs,slPopHead(&mdbCompositeObjs)); // We're done with this one continue; } - // Construct the var=val string for filtering a single exp (set of objs) from composite worth of objs + // Construct the var=val string for filtering a single exp (set of objs) + // from composite worth of objs dyStringClear(dyVars); struct mdbVar *edvVar = edvVarVals; int valsFound = 0; for(;edvVar!=NULL;edvVar=edvVar->next) { dyStringPrintf(dyVars,"%s=%s ",edvVar->var,edvVar->val); if (differentString(edvVar->val,MDB_VAL_ENCODE_EDV_NONE)) valsFound++; } dyStringContents(dyVars)[dyStringLen(dyVars) -1] = '\0'; // Nicer printing is all if (valsFound == 0) { - verbose(1, "There are no experiment defining variables for this object '%s'.\n",mdbCompositeObjs->obj); + verbose(1, "There are no experiment defining variables for this object '%s'.\n", + mdbCompositeObjs->obj); slAddHead(&mdbProcessedObs,slPopHead(&mdbCompositeObjs)); // We're done with this one mdbVarsFree(&edvVarVals); continue; } // Work on one experiment at a time verbose(2, "mdbObjsEncodeExperimentify() working on EDVs: %s.\n",dyStringContents(dyVars)); - struct mdbObj *mdbExpObjs = mdbObjsFilterByVars(&mdbCompositeObjs,dyStringContents(dyVars),TRUE,TRUE); // None={notFound} + struct mdbObj *mdbExpObjs = mdbObjsFilterByVars(&mdbCompositeObjs,dyStringContents(dyVars), + TRUE,TRUE); // None={notFound} - // --- At this point we have nibbled off an experiment worth of objects from the composite set of objects + // --- At this point we have nibbled off an experiment worth of objects from the composite int objsInExp = slCount(mdbExpObjs); assert(objsInExp > 0); expCount++; expObjsCount += objsInExp; // Total of all experimental objects across the composite // Look up each exp in EXPERIMENTS_TABLE char experimentId[128]; int expId = ENCODE_EXP_IX_UNDEFINED; struct encodeExp *exp = encodeExpGetByMdbVarsFromTable(db, edvVarVals, expTable); // --------- BLOCK creation of expIds, at least during rollout of encodeExp // BLOCKED if (exp == NULL && createExpIfNecessary) // BLOCKED exp = encodeExpGetOrCreateByMdbVarsFromTable(db, edvVarVals, expTable); // --------- BLOCK creation of expIds, at least during rollout of encodeExp mdbVarsFree(&edvVarVals); // No longer needed // Make sure the accession is set if requested. if (createExpIfNecessary && updateAccession && exp != NULL && exp->ix != ENCODE_EXP_IX_UNDEFINED && exp->accession == NULL) encodeExpSetAccession(exp, expTable); if (exp != NULL) expId = exp->ix; if (expId == ENCODE_EXP_IX_UNDEFINED) { safef(experimentId,sizeof(experimentId),"{missing}"); if (warn > 0) - printf("Experiment %s EDV: [%s] is not defined in %s.%s table.\n",experimentId,dyStringContents(dyVars), ENCODE_EXP_DATABASE, expTable); - //printf("Experiment %s EDV: [%s] is not defined in %s table. Remaining:%d and %d\n",experimentId,dyStringContents(dyVars),EXPERIMENTS_TABLE,slCount(mdbCompositeObjs),slCount(mdbObjs)); - if (warn < 2) // From mdbUpdate (warn=1), just interested in testing waters. From mdbPrint (warn=2) list all objs in exp. - { + printf("Experiment %s EDV: [%s] is not defined in %s.%s table.\n", + experimentId,dyStringContents(dyVars), ENCODE_EXP_DATABASE, expTable); + //printf("Experiment %s EDV: [%s] is not defined in %s table. Remaining:%d " + // "and %d\n",experimentId,dyStringContents(dyVars),EXPERIMENTS_TABLE, + // slCount(mdbCompositeObjs),slCount(mdbObjs)); + if (warn < 2) // From mdbUpdate (warn=1), just interested in testing waters. + { // From mdbPrint (warn=2) list all objs in exp. expMissing += slCount(mdbExpObjs); mdbProcessedObs = slCat(mdbProcessedObs,mdbExpObjs); mdbExpObjs = NULL; encodeExpFree(&exp); continue; } } else { safef(experimentId,sizeof(experimentId),"%d",expId); if (warn > 0) - printf("Experiment %s has %d objects based upon %d EDVs: [%s].\n",experimentId,slCount(mdbExpObjs),valsFound,dyStringContents(dyVars)); // Set the stage + printf("Experiment %s has %d objects based upon %d EDVs: [%s].\n", + experimentId,slCount(mdbExpObjs),valsFound,dyStringContents(dyVars)); } // Now we can walk through each obj in experiment and determine if it has the correct expId int foundId = FALSE; int errors = objsInExp; if (expMax < objsInExp) expMax = objsInExp; if (expMin > objsInExp) expMin = objsInExp; while(mdbExpObjs != NULL) { struct mdbObj *obj = slPopHead(&mdbExpObjs); - { // NOTE: This list could expand but we expect only tables and files to be objs in an experiment + { // NOTE: This list could expand but we expect only + // tables and files to be objs in an experiment char *objType = mdbObjFindValue(obj,MDB_OBJ_TYPE); - assert(objType != NULL && (sameString(objType,MDB_OBJ_TYPE_TABLE) || sameString(objType,MDB_OBJ_TYPE_FILE))); + assert( objType != NULL + && ( sameString(objType,MDB_OBJ_TYPE_TABLE) + || sameString(objType,MDB_OBJ_TYPE_FILE))); } boolean updateObj = FALSE; char *val = mdbObjFindValue(obj,MDB_VAR_ENCODE_EXP_ID); if (val != NULL) { - foundId = TRUE; // warn==1 will give only 1 exp wide error if no individual errors. NOTE: would be nice if those with expId sorted to beginning, but can't have everything. + foundId = TRUE; // warn==1 will give only 1 exp wide error if no individual errors. + // NOTE: would be nice if those with expId sorted to beginning, + // but can't have everything. int thisId = atoi(val); if (expId == ENCODE_EXP_IX_UNDEFINED || thisId != expId) { updateObj = TRUE; // Always an error! expMissing++; - printf(" ERROR %s %-60s has bad %s=%s.\n",experimentId,obj->obj,MDB_VAR_ENCODE_EXP_ID,val); + printf(" ERROR %s %-60s has bad %s=%s.\n", + experimentId,obj->obj,MDB_VAR_ENCODE_EXP_ID,val); } else { char *acc = mdbObjFindValue(obj,MDB_VAR_DCC_ACCESSION); - if (updateAccession && !createExpIfNecessary && exp->accession == NULL) // -test so one wasn't created - { + if (updateAccession && !createExpIfNecessary && exp->accession == NULL) + { // -test so one wasn't created exp->accession = needMem(16); - safef(exp->accession, 16, "TEMP%06d", exp->ix); // Temporary since this is not an update but we want -test to work. + safef(exp->accession, 16, "TEMP%06d", exp->ix); + // TEMP since this is not an update but we want -test to work. } - if (exp->accession != NULL && (acc == NULL || differentString(acc,exp->accession))) + if (exp->accession != NULL + && (acc == NULL || differentString(acc,exp->accession))) { if (updateAccession) updateObj = TRUE; accMissing++; if (acc != NULL) // Always an error - printf(" ERROR %s %-60s %s set, has wrong %s: %s.\n",experimentId,obj->obj, - MDB_VAR_ENCODE_EXP_ID,MDB_VAR_DCC_ACCESSION,acc); - else if (warn > 1) // NOTE: Could give more info for each obj as per wrangler's desires - printf(" %s %-60s %s set, needs %s.\n",experimentId,obj->obj,MDB_VAR_ENCODE_EXP_ID,MDB_VAR_DCC_ACCESSION); + printf(" ERROR %s %-60s %s set, has wrong %s: %s.\n", + experimentId,obj->obj,MDB_VAR_ENCODE_EXP_ID, + MDB_VAR_DCC_ACCESSION,acc); + else if (warn > 1) // NOTE: Could give more info as per wrangler's desires + printf(" %s %-60s %s set, needs %s.\n", + experimentId,obj->obj,MDB_VAR_ENCODE_EXP_ID, + MDB_VAR_DCC_ACCESSION); } else { errors--; // One less error - if (warn > 1) // NOTE: Could give more info for each obj as per wrangler's desires - printf(" %s %-60s %s\n",experimentId,obj->obj,(exp->accession != NULL ? exp->accession : "")); + if (warn > 1) // NOTE: Could give more info as per wrangler's desires + printf(" %s %-60s %s\n", experimentId,obj->obj, + (exp->accession != NULL ? exp->accession : "")); } } } else { if (expId != ENCODE_EXP_IX_UNDEFINED) { updateObj = TRUE; expMissing++; } if ((foundId && warn > 0) || warn > 1) { if (updateObj) - printf(" %s %-60s needs updating to mdb.\n",experimentId,obj->obj); + printf(" %s %-60s needs updating to mdb.\n", + experimentId,obj->obj); else printf(" %s %s\n",experimentId,obj->obj); // missing } } // This object needs to be updated. if (updateObj) { mdbObjSetVarInt(obj,MDB_VAR_ENCODE_EXP_ID,expId); struct mdbObj *newObj = mdbObjCreate(obj->obj,MDB_VAR_ENCODE_EXP_ID, experimentId); if (updateAccession && exp != NULL && exp->accession != NULL) mdbObjSetVar(newObj,MDB_VAR_DCC_ACCESSION,exp->accession); slAddHead(&mdbUpdateObjs,newObj); } slAddHead(&mdbProcessedObs,obj); } // Done with one experiment encodeExpFree(&exp); if (!foundId && errors > 0 && warn > 0) - printf(" %s all %d objects are missing an %s.\n",experimentId,objsInExp,MDB_VAR_ENCODE_EXP_ID); + printf(" %s all %d objects are missing an %s.\n", + experimentId,objsInExp,MDB_VAR_ENCODE_EXP_ID); } // Done with one composite if (expCount > 0) { printf("Composite%s '%s' has %d recognizable experiment%s with %d objects needing %s", - (compositelessObj?"less set":""),compName,expCount,(expCount != 1?"s":""),expMissing,MDB_VAR_ENCODE_EXP_ID); + (compositelessObj?"less set":""),compName,expCount,(expCount != 1?"s":""), + expMissing,MDB_VAR_ENCODE_EXP_ID); if (accMissing > 0) printf(" and %d objects needing %s",accMissing,MDB_VAR_DCC_ACCESSION); - printf(" updated.\n objects/experiment: min:%d max:%d mean:%lf.\n",expMin,expMax,((double)expObjsCount/expCount)); + printf(" updated.\n objects/experiment: min:%d max:%d mean:%lf.\n", + expMin,expMax,((double)expObjsCount/expCount)); } if (edvSortOrder != NULL) freeMem(edvSortOrder); slNameFreeList(compositeEdvs); } // Done with all composites dyStringFree(&dyVars); *pMdbObjs = mdbProcessedObs; return mdbUpdateObjs; } @@ -3120,32 +3229,32 @@ static struct mdbObj *metadataForTableFromTdb(struct trackDb *tdb) // Returns the metadata for a table from a tdb setting. { char *setting = trackDbSetting(tdb, MDB_METADATA_KEY); if(setting == NULL) return NULL; struct mdbObj *mdbObj; AllocVar(mdbObj); mdbObj->obj = cloneString(tdb->table?tdb->table:tdb->track); AllocVar(mdbObj->vars); mdbObj->vars->var = cloneString(MDB_OBJ_TYPE); mdbObj->vars->val = cloneString(MDB_OBJ_TYPE_TABLE); mdbObj->varHash = hashNew(8); hashAdd(mdbObj->varHash, mdbObj->vars->var, mdbObj->vars); mdbObj = mdbObjAddVarPairs(mdbObj,setting); -mdbObjRemoveVars(mdbObj,MDB_VAR_TABLENAME); // NOTE: Special hint that the tdb metadata is used since no mdb metadata is found -return mdbObj; +mdbObjRemoveVars(mdbObj,MDB_VAR_TABLENAME); // NOTE: Special hint that the tdb metadata +return mdbObj; // is used since no mdb metadata is found } const struct mdbObj *metadataForTable(char *db,struct trackDb *tdb,char *table) // Returns the metadata for a table. NEVER FREE THIS STRUCT! { struct mdbObj *mdbObj = NULL; // See of the mdbObj was already built if(tdb != NULL) { mdbObj = tdbExtrasMdb(tdb); if(mdbObj == METADATA_NOT_FOUND) // NOT in mtatbl, not in tdb metadata setting! return NULL; else if(mdbObj == MDB_NOT_FOUND) // looked mdb already and not found! return metadataForTableFromTdb(tdb); @@ -3159,212 +3268,240 @@ char *mdb = mdbTableName(conn,TRUE); // Look for sandbox name first if(tdb != NULL && tdb->table != NULL) table = tdb->table; if(mdb != NULL) mdbObj = mdbObjQueryByObj(conn,mdb,table,NULL); hFreeConn(&conn); // save the mdbObj for next time if(tdb) { if(mdbObj != NULL) tdbExtrasMdbSet(tdb,mdbObj); else { tdbExtrasMdbSet(tdb,MDB_NOT_FOUND); - return metadataForTableFromTdb(tdb); // FIXME: metadata setting in TDB is soon to be obsolete + return metadataForTableFromTdb(tdb); // FIXME: metadata setting in TDB soon to be obsolete } } return mdbObj; } const char *metadataFindValue(struct trackDb *tdb, char *var) // Finds the val associated with the var or retruns NULL { struct mdbObj *mdbObj = tdbExtrasMdb(tdb); if(mdbObj == MDB_NOT_FOUND) // Note, only we if already looked for mdb (which requires db) mdbObj = metadataForTableFromTdb(tdb); if (mdbObj == NULL || mdbObj == METADATA_NOT_FOUND) return NULL; return mdbObjFindValue(mdbObj,var); } struct mdbObj *mdbObjSearch(struct sqlConnection *conn, char *var, char *val, char *op, int limit) -// Search the metaDb table for objs by var and val. Can restrict by op "is", "like", "in" and accept (non-zero) limited string size +// Search the metaDb table for objs by var and val. +// Can restrict by op "is", "like", "in" and accept (non-zero) limited string size // Search is via mysql, so it's case-insensitive. Return is sorted on obj. { if (var == NULL && val == NULL) errAbort("mdbObjSearch requests objects but provides no criteria.\n"); char *tableName = mdbTableName(conn,TRUE); // Look for sandBox name first // Build a query string struct dyString *dyQuery = dyStringNew(512); dyStringPrintf(dyQuery,"select l1.obj, l1.var, l1.val from %s l1",tableName); if (var != NULL || val != NULL) - dyStringPrintf(dyQuery," where exists (select l2.obj from %s l2 where l2.obj = l1.obj and ",tableName); + dyStringPrintf(dyQuery," where exists (select l2.obj from %s l2 where l2.obj = l1.obj and ", + tableName); if(var != NULL) dyStringPrintf(dyQuery,"l2.var = '%s'", var); if(var != NULL && val != NULL) dyStringAppend(dyQuery," and "); if(val != NULL) { dyStringAppend(dyQuery,"l2.val "); if(sameString(op, "in")) - dyStringPrintf(dyQuery,"in (%s)", val); // Note, must be a formatted string already: 'a','b','c' or 1,2,3 + dyStringPrintf(dyQuery,"in (%s)", val); + // Note, must be a formatted string already: 'a','b','c' or 1,2,3 else if(sameString(op, "contains") || sameString(op, "like")) dyStringPrintf(dyQuery,"like '%%%s%%'", val); else if (limit > 0 && strlen(val) != limit) dyStringPrintf(dyQuery,"like '%.*s%%'", limit, val); else dyStringPrintf(dyQuery,"= '%s'", val); } dyStringAppendC(dyQuery,')'); dyStringAppend(dyQuery," order by obj"); struct mdb *mdb = mdbLoadByQuery(conn, dyStringCannibalize(&dyQuery)); verbose(3, "rows (vars) returned: %d\n",slCount(mdb)); struct mdbObj *mdbObjs = mdbObjsLoadFromMemory(&mdb,TRUE); return mdbObjs; } -struct mdbObj *mdbObjRepeatedSearch(struct sqlConnection *conn,struct slPair *varValPairs,boolean tables,boolean files) +struct mdbObj *mdbObjRepeatedSearch(struct sqlConnection *conn,struct slPair *varValPairs, + boolean tables,boolean files) // Search the metaDb table for objs by var,val pairs. Uses mdbCvSearchMethod() if available. // This method will use mdbObjsQueryByVars() { struct slPair *onePair; struct dyString *dyTerms = dyStringNew(256); // Build list of terms as "var1=val1 var2=val2a,val2b,val2c var3=%val3%" for(onePair = varValPairs; onePair != NULL; onePair = onePair->next) { - if (isEmpty(((char *)(onePair->val)))) // NOTE: All the parens are needed to get the macro to do the right thing - continue; + if (isEmpty(((char *)(onePair->val)))) // NOTE: All the parens are needed to get the macro + continue; // to do the right thing enum cvSearchable searchBy = cvSearchMethod(onePair->name); - if (searchBy == cvSearchByMultiSelect // multiSelect val will be filled with a comma delimited list - || searchBy == cvSearchBySingleSelect + if (searchBy == cvSearchByMultiSelect // multiSelect val will be filled with + || searchBy == cvSearchBySingleSelect // comma delimited list || searchBy == cvSearchByWildList) { if (strchr((char *)onePair->val,' ')) dyStringPrintf(dyTerms,"%s=\"%s\" ",onePair->name,(char *)onePair->val); else dyStringPrintf(dyTerms,"%s=%s ",onePair->name,(char *)onePair->val); } else if (searchBy == cvSearchByFreeText) // If select is by free text then like dyStringPrintf(dyTerms,"%s=\"%%%s%%\" ",onePair->name,(char *)onePair->val); - else if (sameWord(onePair->name,MDB_VAR_COMPOSITE)) // special case. Not directly searchable by UI but indirectly and will show up here. + else if (sameWord(onePair->name,MDB_VAR_COMPOSITE)) + { + // special case. Not directly searchable by UI but indirectly and will show up here. dyStringPrintf(dyTerms,"%s=%s ",onePair->name,(char *)onePair->val); + } else if (searchBy == cvSearchByDateRange || searchBy == cvSearchByIntegerRange) { // TO BE IMPLEMENTED } } // Be sure to include table or file in selections if (tables) dyStringPrintf(dyTerms,"%s=%s ",MDB_OBJ_TYPE,MDB_OBJ_TYPE_TABLE); if (files) dyStringPrintf(dyTerms,"%s=? ",MDB_VAR_FILENAME); // Build the mdbByVals struct and then select all mdbObjs in one query struct mdbObj *mdbObjs = mdbObjsQueryByVarVals(conn,dyStringContents(dyTerms)); dyStringFree(&dyTerms); return mdbObjs; } -struct slName *mdbObjNameSearch(struct sqlConnection *conn, char *var, char *val, char *op, int limit, boolean tables, boolean files) -// Search the metaDb table for objs by var and val. Can restrict by op "is", "like", "in" and accept (non-zero) limited string size +struct slName *mdbObjNameSearch(struct sqlConnection *conn, char *var, char *val, char *op, + int limit, boolean tables, boolean files) +// Search the metaDb table for objs by var and val. +// Can restrict by op "is", "like", "in" and accept (non-zero) limited string size // Search is via mysql, so it's case-insensitive. Return is sorted on obj. { // Note: This proves faster than getting mdbObjs then converting to slNames struct mdbObj *mdbObjs = mdbObjSearch(conn,var,val,op,limit); // May only be interested in tables or files: if (tables || files) { struct mdbObj *mdbObjsDropped = mdbObjsFilterTablesOrFiles(&mdbObjs,tables,files); mdbObjsFree(&mdbObjsDropped); } struct slName *mdbNames = mdbObjToSlName(mdbObjs); mdbObjsFree(&mdbObjs); return mdbNames; } -static void mdbSearchableQueryRestictForTablesOrFiles(struct dyString *dyQuery,char *tableName, char letter,boolean hasTableName, boolean hasFileName) +static void mdbSearchableQueryRestictForTablesOrFiles(struct dyString *dyQuery,char *tableName, + char letter,boolean hasTableName, boolean hasFileName) // Append table and file restrictions onto an mdb query. -// letter (e.g. 'A') should be used in original query to alias table name: "select A.val from metaDb A where A.val = 'fred'". +// letter (e.g. 'A') should be used in original query to alias table name: +// "select A.val from metaDb A where A.val = 'fred'". { -// A note about tables and files: objType=table may have fileNames associated, but objType=file will not have tableNames +// A note about tables and files: objType=table may have fileNames associated, +// but objType=file will not have tableNames // While objType=table should have a var=tableName, this is redundant because the obj=tableName // So the lopsided 'exists' queries below are meant to be the most flexible/efficent assert(isalpha(letter) && isalpha(letter + 2)); // will need one or two sub-queries. char nextLtr = letter + 1; -// We are only searching for objects that are of objType table or file. objType=composite are not search targets! -if (hasTableName && !hasFileName) // objType=table may have fileNames associated, but objType=file will not have tableNames - dyStringPrintf(dyQuery," and exists (select %c.obj from %s %c where %c.obj = %c.obj and %c.var='objType' and %c.val = '%s')", - nextLtr,tableName,nextLtr,nextLtr,letter,nextLtr,nextLtr,MDB_OBJ_TYPE_TABLE); +// We are only searching for objects that are of objType table or file. +// objType=composite are not search targets! +if (hasTableName && !hasFileName) + { + // objType=table may have fileNames associated, but objType=file will not have tableNames + dyStringPrintf(dyQuery," and exists (select %c.obj from %s %c where %c.obj = %c.obj and " + "%c.var='objType' and %c.val = '%s')",nextLtr,tableName,nextLtr,nextLtr, + letter,nextLtr,nextLtr,MDB_OBJ_TYPE_TABLE); + } else // tables OR files (but not objType=composite) - dyStringPrintf(dyQuery," and exists (select %c.obj from %s %c where %c.obj = %c.obj and %c.var='objType' and %c.val in ('%s','%s'))", - nextLtr,tableName,nextLtr,nextLtr,letter,nextLtr,nextLtr,MDB_OBJ_TYPE_TABLE,MDB_OBJ_TYPE_FILE); - + { + dyStringPrintf(dyQuery," and exists (select %c.obj from %s %c where %c.obj = %c.obj and " + "%c.var='objType' and %c.val in ('%s','%s'))",nextLtr,tableName,nextLtr, + nextLtr,letter,nextLtr,nextLtr,MDB_OBJ_TYPE_TABLE,MDB_OBJ_TYPE_FILE); + } nextLtr++; -if (!hasTableName && hasFileName) // last of 3 possibilites objType either table or file but must have fileName var - dyStringPrintf(dyQuery," and exists (select %c.obj from %s %c where %c.obj = %c.obj and %c.var in ('%s','%s'))", - nextLtr,tableName,nextLtr,nextLtr,letter,nextLtr,MDB_VAR_FILENAME,MDB_VAR_FILEINDEX); +// last of 3 possibilites objType either table or file but must have fileName var +if (!hasTableName && hasFileName) + dyStringPrintf(dyQuery," and exists (select %c.obj from %s %c where %c.obj = %c.obj and " + "%c.var in ('%s','%s'))",nextLtr,tableName,nextLtr,nextLtr,letter, + nextLtr,MDB_VAR_FILENAME,MDB_VAR_FILEINDEX); } -struct slName *mdbValSearch(struct sqlConnection *conn, char *var, int limit, boolean hasTableName, boolean hasFileName) -// Search the metaDb table for vals by var. Can impose (non-zero) limit on returned string size of val +struct slName *mdbValSearch(struct sqlConnection *conn, char *var, int limit, + boolean hasTableName, boolean hasFileName) +// Search the metaDb table for vals by var. +// Can impose (non-zero) limit on returned string size of val // Search is via mysql, so it's case-insensitive. Return is sorted on val. -// Searchable vars are only for table or file objects. Further restrict to vars associated with tableName, fileName or both. +// Searchable vars are only for table or file objects. +// Further restrict to vars associated with tableName, fileName or both. { // TODO: Change this to use normal mdb struct routines? struct slName *retVal; if (!hasTableName && !hasFileName) errAbort("mdbValSearch requests vals associated with neither table nor files.\n"); char *tableName = mdbTableName(conn,TRUE); // Look for sandBox name first char letter = 'A'; struct dyString *dyQuery = dyStringNew(512); if (limit > 0) dyStringPrintf(dyQuery,"select distinct LEFT(%c.val,%d)",letter,limit); else dyStringPrintf(dyQuery,"select distinct %c.val",letter); dyStringPrintf(dyQuery," from %s %c where %c.var='%s'",tableName,letter,letter,var); mdbSearchableQueryRestictForTablesOrFiles(dyQuery,tableName, letter, hasTableName, hasFileName); dyStringPrintf(dyQuery," order by %c.val",letter); retVal = sqlQuickList(conn, dyStringCannibalize(&dyQuery)); slNameSortCase(&retVal); return retVal; } -struct slPair *mdbValLabelSearch(struct sqlConnection *conn, char *var, int limit, boolean tags, boolean hasTableName, boolean hasFileName) -// Search the metaDb table for vals by var and returns val (as pair->name) and controlled vocabulary (cv) label -// (if it exists) (as pair->val). Can impose (non-zero) limit on returned string size of name. -// Searchable vars are only for table or file objects. Further restrict to vars associated with tableName, fileName or both. -// Return is case insensitive sorted on label (cv label or else val). If requested, return cv tag instead of mdb val. +struct slPair *mdbValLabelSearch(struct sqlConnection *conn, char *var, int limit, boolean tags, + boolean hasTableName, boolean hasFileName) +// Search the metaDb table for vals by var and returns val (as pair->name) +// and controlled vocabulary (cv) label (if it exists) (as pair->val). +// Can impose (non-zero) limit on returned string size of name. +// Searchable vars are only for table or file objects. +// Further restrict to vars associated with tableName, fileName or both. +// Return is case insensitive sorted on label (cv label or else val). +// If requested, return cv tag instead of mdb val. { // TODO: Change this to use normal mdb struct routines? if (!hasTableName && !hasFileName) errAbort("mdbValLabelSearch requests vals associated with neither table nor files.\n"); char *tableName = mdbTableName(conn,TRUE); // Look for sandBox name first char letter = 'A'; struct dyString *dyQuery = dyStringNew(512); if (limit > 0) dyStringPrintf(dyQuery,"select distinct LEFT(%c.val,%d)",letter,limit); else dyStringPrintf(dyQuery,"select distinct %c.val",letter); dyStringPrintf(dyQuery," from %s %c where %c.var='%s'",tableName,letter,letter,var); @@ -3401,47 +3538,50 @@ slPairAdd(&pairs,val,label); } sqlFreeResult(&sr); if (slCount(pairs) > 0) { // should have a list sorted on the label enum cvDataType eCvDataType = cvDataType(var); if (eCvDataType == cvInteger) slPairValAtoiSort(&pairs); else slPairValSortCase(&pairs); } return pairs; } -struct slPair *mdbVarsSearchable(struct sqlConnection *conn, boolean hasTableName, boolean hasFileName) +struct slPair *mdbVarsSearchable(struct sqlConnection *conn, boolean hasTableName, + boolean hasFileName) // returns a white list of mdb vars that actually exist in the current DB. -// Searchable vars are only for table or file objects. Further restrict to vars associated with tableName, fileName or both. +// Searchable vars are only for table or file objects. +// Further restrict to vars associated with tableName, fileName or both. { if (!hasTableName && !hasFileName) errAbort("mdbVarsSearchable requests vals associated with neither table nor files.\n"); char *tableName = mdbTableName(conn,TRUE); // Look for sandBox name first char letter = 'A'; struct slPair *cvApproved = cvWhiteList(TRUE,FALSE); struct slPair *relevant = NULL; struct dyString *dyQuery = dyStringNew(256); while(cvApproved != NULL) { struct slPair *oneVar = slPopHead(&cvApproved); dyStringClear(dyQuery); - dyStringPrintf(dyQuery, "select count(DISTINCT %c.val) from %s %c where %c.var = '%s'",letter,tableName,letter,letter,oneVar->name); + dyStringPrintf(dyQuery, "select count(DISTINCT %c.val) from %s %c where %c.var = '%s'", + letter,tableName,letter,letter,oneVar->name); mdbSearchableQueryRestictForTablesOrFiles(dyQuery,tableName,letter, hasTableName, hasFileName); int count = sqlQuickNum(conn,dyStringContents(dyQuery)); //warn("%d %s",count,dyStringContents(dyQuery)); if(count > 1) // If there is only one value then searching on that variable is useless! slAddHead(&relevant, oneVar); else slPairFree(&oneVar); } dyStringFree(&dyQuery); slReverse(&relevant); return relevant; }