4898794edd81be5285ea6e544acbedeaeb31bf78 max Tue Nov 23 08:10:57 2021 -0800 Fixing pointers to README file for license in all source code files. refs #27614 diff --git src/hg/makeDb/mdbUpdate/mdbUpdate.c src/hg/makeDb/mdbUpdate/mdbUpdate.c index b2b2b10..ea43bf4 100644 --- src/hg/makeDb/mdbUpdate/mdbUpdate.c +++ src/hg/makeDb/mdbUpdate/mdbUpdate.c @@ -1,413 +1,413 @@ // mdbUpdate - Adds, updates or removes metadata objects and vars from the 'mdb' metadata table. /* Copyright (C) 2012 The Regents of the University of California - * See README in this or parent directory for licensing information. */ + * See kent/LICENSE or http://genome.ucsc.edu/license/ for licensing information. */ #include "common.h" #include "linefile.h" #include "options.h" #include "mdb.h" #include "hash.h" #define OBJTYPE_DEFAULT "table" void usage() /* Explain usage and exit. */ { errAbort( "mdbUpdate - Adds, updates or removes metadata objects and variables from the\n" " '" MDB_DEFAULT_NAME "' metadata table.\n" "usage:\n" " mdbUpdate {db} [-table= [-force]] [-recreate] [-test [-verbose=2]]\n" " [{fileName}] [-replace] or\n" " [[-obj=] or [-composite=] [-vars=\"var1=val1 var2=val2...\"]]\n" " [[-setVars=\"var1=val1 ...\"]] [-delete]\n" " [-encodeExp [-accession]\n" "Options:\n" " {db} Database to update metadata in. This argument is required.\n" " -table Table to update metadata to. Default is the sandbox version of\n" " '" MDB_DEFAULT_NAME "'.\n" " -recreate Creates or empties the table. No further arguements needed.\n" " -force Overrides restrictions placed on shared '" MDB_DEFAULT_NAME "'.\n" " -test Does not actually update and does full cv validation.\n" " Use with -verbose=2 to see the SQL.\n" " SELECT Objects to work on (required if file is not specified):\n" " -obj={objName} Select this single object to update\n" " -composite={} Select all objects belonging to this composite.\n" " -vars=\"{var=val ...}\" Apply update to a group of objects matching vars.\n" " Use: 'var=val' 'var=v%%' 'var=' 'var=val1,val2' (val1 or val2).\n" " 'var!=val' 'var!=v%%' 'var!=' 'var!=val1,val2' are all supported.\n" " -var= -val= Can be used when val to select has spaces.\n" " TARGET vars to set or delete:\").\n" " -setVars={var=val...} Allows setting multiple var=val pairs.\n" " -setVar= -setVal= Can be used when val to set has spaces.\n" " ACTION (update to values defined in -setVars is the default):\n" " -delete Remove specific var (or entire object if -setVar is not used).\n" " -encodeExp={table} Update groups of objs as experiments defined in hgFixed.{table}.\n" " -accession If encodeExp, then make/update accession too.\n" " BY FILE (required if SELECT objects params are not provided):\n" " [{fileName}] File of formatted metadata from mdbPrint (RA or lines).\n" " -replace Remove all old variables for each object before adding new.\n" " The file can be mdbPrint output (RA format) or the following special formats:\n" " metadata objName var1=val1 var2=\"val2 with spaces\" var3=...\n" " Adds or updates the specific object and variables\n" " metadata objName delete\n" " delete all metadata for objName\n" " metadata objName delete var=val var2=val2\n" " deletes specifically named variables for objName (vals are ignored).\n" " Special ENCODE format as produced by the doEncodeValidate.pl\n" "NOTE: Updates to the shared '" MDB_DEFAULT_NAME "' can only be done by a file written\n" " directly from mdbPrint. Update sandbox first, then move to shared table.\n" "HINT: Use '%%' in any command line obj, var or val as a wildcard for selection.\n\n" "Examples:\n" " mdbUpdate hg19 -vars=\"grant=Snyder cell=K562 antibody=CTCF\" -setVars=\"expId=1427\"\n" " Update all objects matcing Snyder/K562/CTCF and set the expId=1472.\n" " mdbUpdate hg19 -composite=%%BroadHist%% -vars=\"dccAccession=\" -setVars=\"dccAccession=\" " "-delete\n" " Deletes all accessions from Broad Histone objects that have them.\n" " mdbUpdate mm9 -table=mdb_braney -recreate\n" " Creates or empties the named metadata table.\n" " mdbUpdate hg18 -vars=\"composite=wgEncodeDukeDNase\" -delete -test\n" " Tests the delete of all objects in the wgEncodeDukeDNase composite.\n" " mdbUpdate hg18 -table=" MDB_DEFAULT_NAME " mdbBroadHistone.ra -replace\n" " Replaces all metadata in the shared table for objects found in the\n" " provided file. File must have been printed with mdbPrint.\n" ); } static struct optionSpec optionSpecs[] = { {"table", OPTION_STRING}, // default "metaDb" {"obj", OPTION_STRING}, // objName or objId {"vars", OPTION_STRING}, // Select set of object by vars {"or", OPTION_STRING},// or var1=val1 var2=val2... {"composite",OPTION_STRING}, // Special case of a commn var // (replaces vars="composite=wgEncodeBroadHistone") {"var", OPTION_STRING}, // variable ONLY USED WHEN VAL has spaces {"val", OPTION_STRING}, // value ONLY USED WHEN VAL has spaces {"encodeExp",OPTION_STRING},// Update Experiments as defined in the hgFixed.encodeExp table {"accession",OPTION_BOOLEAN},// Adds/updates accession when experimentifying {"setVars", OPTION_STRING}, // Allows setting multiple var=val pairs {"setVar", OPTION_STRING}, // variable ONLY USED WHEN val to set has spaces {"setVal", OPTION_STRING}, // value ONLY USED WHEN val to set has spaces {"delete", OPTION_BOOLEAN},// delete one obj or obj/var {"replace", OPTION_BOOLEAN},// replace entire obj when loading from file {"recreate",OPTION_BOOLEAN},// creates or recreates the table {"force", OPTION_BOOLEAN},// override restrictions on shared table {"test", OPTION_BOOLEAN},// give it a test, will ya? {NULL, 0} }; int main(int argc, char *argv[]) // Process command line. { if (argc == 1) usage(); struct mdbObj * mdbObjs = NULL; int retCode = 0; optionInit(&argc, argv, optionSpecs); if (argc < 2) { errAbort("REQUIRED 'DB' argument not found.\n"); } char *db = argv[1]; char *table = optionVal("table",NULL); boolean deleteIt = optionExists("delete"); boolean testIt = optionExists("test"); boolean recreate = optionExists("recreate"); boolean force = optionExists("force"); boolean replace = FALSE; char *orVars = NULL; char *encodeExp = optionVal("encodeExp",NULL); if (encodeExp != NULL) { if (strlen(encodeExp) == 0 || sameWord("std",encodeExp)) encodeExp = "encodeExp"; } if (recreate && encodeExp != NULL) errAbort("Incompatible options 'recreate' and 'encodeExp':\n"); if (recreate && deleteIt) errAbort("Incompatible options 'recreate' and 'delete':\n"); if (encodeExp != NULL && deleteIt) errAbort("Incompatible options 'encodeExp' and 'delete':\n"); if (testIt && force) errAbort("Incompatible options 'test' and 'force':\n"); struct sqlConnection *conn = sqlConnect(db); // Find the table if necessary if (table == NULL) { table = mdbTableName((recreate ? NULL : conn),TRUE); // Look for sandBox name first if (table == NULL) { table = mdbTableName((recreate ? NULL : conn),FALSE); // Okay, default then if (table == NULL) // Now you are just getting me angry! { sqlDisconnect(&conn); if (!recreate) // assertable errAbort("No '%s.%s' found. Consider using -recreate flag.\n",db,MDB_DEFAULT_NAME); else errAbort("No '%s.%s' found.\n",db,MDB_DEFAULT_NAME); } } } if (encodeExp != NULL) verbose(1, "Using tables named '%s.%s' and 'hgFixed.%s'.\n",db,table,encodeExp); else verbose(1, "Using table named '%s.%s'.\n",db,table); boolean sharedTbl = sameWord(table,MDB_DEFAULT_NAME); // Special restrictions apply // Recreate the table if (recreate) { if (sharedTbl && ! force) { sqlDisconnect(&conn); errAbort("NOT SUPPORTED for shared table '%s'.\n",MDB_DEFAULT_NAME); } boolean recreated = sqlTableExists(conn,table); mdbReCreate(conn,table,testIt); if (testIt) { verbose(1, "Would %screate table named '%s'.\n", (recreated ? "re" : ""),table); if (!recreated) { sqlDisconnect(&conn); if (optionExists("obj") || optionExists("vars") || argc > 2) verbose(1, "Can't test further commands. " "Consider '-db= [-table=] -recreate' as the only arguments.\n"); return -1; // Don't test any update if we haven't actually created the table! } } else verbose(1, "%s table named '%s'.\n",(recreated ? "Recreated" : "Created"),table); } if (argc > 2) // file specified { if (argc > 3) errAbort("TOO MANY parameters.\n"); if (deleteIt || encodeExp != NULL) errAbort("INCONSISTENT REQUEST: can't combine supplied file with -delete or -encodeExp.\n"); if (optionExists("obj") || optionExists("vars") || optionExists("composite") || optionExists("var") || optionExists("val")) errAbort("INCONSISTENT REQUEST: can't combine supplied file with SELECT objects params.\n"); if (optionExists("setVars") || optionExists("setVar") || optionExists("setVal")) errAbort("INCONSISTENT REQUEST: can't combine supplied file and " "TARGET (-setVars, etc) param.\n"); replace = optionExists("replace"); boolean validated = FALSE; // Only validate magic for shared table. mdbObjs = mdbObjsLoadFromFormattedFile(argv[2],(sharedTbl ? &validated : NULL)); // FIXME: Can't test magic if RAs are catted together! Requires reading in 1 stanza at a time, // slAddTail, and testing magic when intercepted. if (sharedTbl && !force && !validated) errAbort("NOT SUPPORTED: Update to shared table '%s' requires file directly written by " "mdbPrint from sandbox file.\n", table); if (mdbObjs != NULL) verbose(1, "Read %d metadata objects from %s\n", slCount(mdbObjs),argv[1]); } else // no file specified { // SELECT objs params ok? if (!optionExists("obj") && !optionExists("vars") && !optionExists("composite") && !optionExists("var")) { if (recreate) // no problem return 0; errAbort("INCOMPLETE REQUEST: Must include file or SELECT objects params " "(-obj, -vars, etc.).\n"); } if (optionExists("obj") && (optionExists("vars") || optionExists("composite") || optionExists("var"))) errAbort("INCONSISTENT REQUEST: Can't include -obj with any other SELECT params " "(-composite, -vars, etc.).\n"); if (!optionExists("var") && optionExists("val")) errAbort("INCOMPLETE REQUEST: Must specify -var with -val.\n"); // TARGET params ok? char *setVar = optionVal("setVar", NULL); if (setVar == NULL && optionExists("setVal")) // allows: -var= -setVal= setVar = optionVal("var", NULL); if (!optionExists("setVars") && setVar == NULL && (!deleteIt && encodeExp == NULL)) errAbort("INCOMPLETE REQUEST: Must specify TARGET (-setVars) " "and/or ACTION (-delete, -encodeExp).\n"); if (setVar == NULL && optionExists("setVal")) errAbort("INCOMPLETE REQUEST: Must specify -setVar with -setVal.\n"); if (setVar != NULL && !optionExists("setVal") && !deleteIt) errAbort("INCOMPLETE REQUEST: Must specify -setVal or -delete with -setVar.\n"); if (setVar != NULL && optionExists("setVal") && deleteIt) errAbort("INCONSISTENT REQUEST: Can't specify -setVal and -delete.\n"); if ((optionExists("setVars") || setVar != NULL) && encodeExp != NULL) errAbort("INCONSISTENT REQUEST: Can't specify -setVars and -encodeExp in one command.\n"); if (deleteIt && encodeExp != NULL) errAbort("INCONSISTENT REQUEST: Can't specify -delete and -encodeExp in one command.\n"); if (sharedTbl && !force) errAbort("NOT SUPPORTED: Updating the shared table '%s' is only allowed by " "mdbPrint written file.\n",MDB_DEFAULT_NAME); // Now get the object list if (optionExists("obj")) { mdbObjs = mdbObjQueryByObj(conn,table,optionVal("obj", NULL),NULL); } else if (optionExists("vars") || optionExists("composite")) { struct mdbByVar * mdbByVars = NULL; if (optionExists("vars")) { char *vars = optionVal("vars", NULL); orVars = strstr(vars," || "); if (orVars != NULL) { *orVars = '\0'; orVars += 4; } mdbByVars = mdbByVarsLineParse(vars); if (optionExists("composite")) mdbByVarAppend(mdbByVars,"composite", optionVal("composite", NULL),FALSE); if (optionExists("var")) mdbByVarAppend(mdbByVars,optionVal("var", NULL), optionVal("val", NULL),FALSE); } else if (optionExists("composite")) { mdbByVars = mdbByVarCreate("composite", optionVal("composite", NULL)); if (optionExists("var")) mdbByVarAppend(mdbByVars,optionVal("var", NULL), optionVal("val", NULL),FALSE); } else if (optionExists("var")) mdbByVars = mdbByVarCreate(optionVal("var", NULL),optionVal("val", NULL)); mdbObjs = mdbObjsQueryByVars(conn,table,mdbByVars); } verbose(1, "Selected %d metadata objects\n", slCount(mdbObjs)); if (optionExists("or")) { if ( orVars != NULL) errAbort("Incompatible to use -or and ' || ' in -vars.\n"); if (!optionExists("vars") && !optionExists("obj")) errAbort("Incompatible to use -or without -vars or -obj'.\n"); orVars = optionVal("or", NULL); } boolean resort = FALSE; while (orVars != NULL) { char *vars = orVars; orVars = strstr(vars," || "); if (orVars != NULL) { *orVars = '\0'; orVars += 4; } struct mdbByVar * orByVars = mdbByVarsLineParse(vars); if (optionExists("composite")) mdbByVarAppend(orByVars,"composite", optionVal("composite", NULL),FALSE); if (optionExists("var")) mdbByVarAppend(orByVars,optionVal("var", NULL), optionVal("val", NULL),FALSE); struct mdbObj * orResults = mdbObjsQueryByVars(conn,table,orByVars); if (orResults != NULL) { // Merge be removing dups from orResults and cating together. orResults = mdbObjIntersection(&orResults,mdbObjs); mdbObjs = slCat(mdbObjs,orResults); resort = TRUE; } } if (resort) slSort(&mdbObjs,&mdbObjCmp); // Need to be returned to obj order if (mdbObjs != NULL) { if (encodeExp == NULL) { if (optionExists("setVars")) // replace all found vars to create update request mdbObjSwapVars(mdbObjs,optionVal("setVars", NULL),deleteIt); else if (setVar != NULL) // Only the single var val is set mdbObjTransformToUpdate(mdbObjs,setVar, optionVal("setVal", NULL),deleteIt); else mdbObjTransformToUpdate(mdbObjs,NULL,NULL,deleteIt); } verbose(2, "metadata %s %s%s%s%s\n", mdbObjs->obj,(mdbObjs->deleteThis ? "delete " : ""), (mdbObjs->vars && mdbObjs->vars->var!=NULL ? mdbObjs->vars->var : ""), (mdbObjs->vars && mdbObjs->vars->val!=NULL ? "=" : ""), (mdbObjs->vars && mdbObjs->vars->val!=NULL ? mdbObjs->vars->val : "")); } } int count = 0; if (mdbObjs != NULL) { if (testIt && verboseLevel() > 2) mdbObjPrint(mdbObjs,FALSE); if (recreate) // recreate then do the fast load count = mdbObjsLoadToDb(conn,table,mdbObjs,testIt); else { if (encodeExp != NULL) { boolean createExpIfNecessary = testIt ? FALSE : TRUE; boolean updateAccession = (optionExists("accession")); struct mdbObj *updatable = mdbObjsEncodeExperimentify(conn,db,table,encodeExp,&mdbObjs, (verboseLevel() > 1 ? 1 : 0), // 1=warnings createExpIfNecessary,updateAccession); if (updatable == NULL) { verbose(1, "No Experiment ID updates were discovered in %d object(s).\n", slCount(mdbObjs)); retCode = 2; } else { if (testIt) verbose(1, "Found %d of %d object(s) would have their experiment ID updated.\n", slCount(updatable), slCount(mdbObjs)); mdbObjsFree(&mdbObjs); mdbObjs = updatable; } } // Finally the actual update (or test update) count = mdbObjsSetToDb(conn,table,mdbObjs,replace,testIt); } if (count <= 0) retCode = 1; if (testIt && !deleteIt && encodeExp == NULL) { int invalids = mdbObjsValidate(mdbObjs,TRUE); int varsCnt=mdbObjCount(mdbObjs,FALSE); verbose(1, "%d invalid%s of %d variable%s according to the cv.ra.\n", invalids,(invalids==1?"":"s"),varsCnt,(varsCnt==1?"":"s")); if (invalids > 0) verbose(1, "Variables invalid to cv.ra ARE PERMITTED in the mdb.\n"); } } if (testIt) verbose(1, "Command would affected %d row(s) in %s.%s\n", count,db,table); else verbose(1, "Affected %d row(s) in %s.%s\n", count,db,table); sqlDisconnect(&conn); mdbObjsFree(&mdbObjs); return retCode; }