4898794edd81be5285ea6e544acbedeaeb31bf78 max Tue Nov 23 08:10:57 2021 -0800 Fixing pointers to README file for license in all source code files. refs #27614 diff --git src/hg/encode/metaCheck/metaCheck.c src/hg/encode/metaCheck/metaCheck.c index 4c6cc34..6fc512e 100644 --- src/hg/encode/metaCheck/metaCheck.c +++ src/hg/encode/metaCheck/metaCheck.c @@ -1,766 +1,766 @@ /* metaCheck - a program to validate that tables, files, and trackDb entries exist. */ /* Copyright (C) 2013 The Regents of the University of California - * See README in this or parent directory for licensing information. */ + * See kent/LICENSE or http://genome.ucsc.edu/license/ for licensing information. */ #include "common.h" #include "linefile.h" #include "hash.h" #include "options.h" #include "hdb.h" #include "mdb.h" #include "portable.h" #include "../inc/encodePatchTdb.h" void usage() /* Explain usage and exit. */ { errAbort( "metaCheck - a program to validate that tables, files, and trackDb entries exist\n" "usage:\n" " metaCheck database composite\n" "arguments:\n" " database assembly database\n" " composite name of track composite\n" "options:\n" " -outMdb=file.ra output cruft-free metaDb ra file\n" " -onlyCompTdb only check trackDb entries that start with composite\n" " -release set release state, default alpha\n" " -releaseNum= specify which release directory to look in (e.g. -releaseNum=2)\n" " -metaDb= specify a path for the metaDb, by default, this looks in <database>'s metaDb\n" " -trackDb= specify a path for the trackDb, by default, this is looks in <database>'s trackDb\n" " -downloadDir= specify a path for the downloads directory\n" " -help print out extended information about what metaCheck is doing\n" ); } char *outMdb = NULL; boolean onlyCompTdb = FALSE; char *release = "alpha"; int releaseNum = 0; static struct optionSpec options[] = { {"outMdb", OPTION_STRING}, {"help", OPTION_BOOLEAN}, {"onlyCompTdb", OPTION_BOOLEAN}, {"release", OPTION_STRING}, {"metaDb", OPTION_STRING}, {"trackDb", OPTION_STRING}, {"downloadDir", OPTION_STRING}, {"releaseNum", OPTION_INT}, {NULL, 0}, }; void checkMetaTables(struct mdbObj *mdbObj, char *database) { struct sqlConnection *conn = sqlConnect(database); verbose(1, "----------------------------------------------\n"); verbose(1, "Checking that tables specified in metaDb exist in database\n"); verbose(1, "----------------------------------------------\n"); for(; mdbObj != NULL; mdbObj=mdbObj->next) { struct mdbVar *mdbVar = hashFindVal(mdbObj->varHash, "objType"); if (mdbVar == NULL) { warn("objType not found in object %s", mdbObj->obj); continue; } if (differentString(mdbVar->val, "table")) continue; mdbObj->deleteThis = FALSE; mdbVar = hashFindVal(mdbObj->varHash, "tableName"); if (mdbVar == NULL) { mdbObj->deleteThis = TRUE; warn("tableName not found in object %s", mdbObj->obj); continue; } char *tableName = mdbVar->val; if (!startsWith(mdbObj->obj, tableName)) { warn("tableName %s does not start with object name %s", tableName, mdbObj->obj); } struct mdbVar *atticVar = hashFindVal(mdbObj->varHash, "attic"); if (!sqlTableExists(conn, tableName)) { if (atticVar) { warn("attic metaDb table %s not found in database %s", tableName, database); continue; } mdbObj->deleteThis = TRUE; warn("metaDb table %s not found in database %s", tableName, database); } } sqlDisconnect(&conn); } void checkMetaFiles(struct mdbObj *mdbObj, char *downDir, struct hash *allNames) { verbose(1, "----------------------------------------------\n"); verbose(1, "Checking that files specified in metaDb exist in download dir\n"); verbose(1, "----------------------------------------------\n"); for(; mdbObj != NULL; mdbObj=mdbObj->next) { struct mdbVar *mdbVar = hashFindVal(mdbObj->varHash, "objType"); if (mdbVar == NULL) { warn("objType not found in object %s", mdbObj->obj); continue; } if (sameString(mdbVar->val, "composite")) // skip objType composite continue; mdbObj->deleteThis = FALSE; mdbVar = hashFindVal(mdbObj->varHash, "composite"); if (mdbVar == NULL) { warn("composite not found in object %s", mdbObj->obj); continue; } // char *composite = mdbVar->val; mdbVar = hashFindVal(mdbObj->varHash, "fileName"); if (mdbVar == NULL) { mdbObj->deleteThis = TRUE; warn("fileName not found in object %s", mdbObj->obj); continue; } char *fileName = mdbVar->val; char buffer[10 * 1024]; struct hash *bamNames = hashNew(8); struct slName *list = slNameListFromString(fileName, ','), *el; for(el=list; el; el=el->next) { if (hashLookup(allNames, el->name)) { warn("duplicate fileName entry: %s", el->name); } else { hashAdd(allNames, el->name, NULL); } if (endsWith(el->name,".bam")) { hashAdd(bamNames, el->name, NULL); } if (endsWith(el->name,".bam.bai")) { el->name[strlen(el->name)-4] = 0; struct hashEl *hel = hashLookup(bamNames, el->name); el->name[strlen(el->name)] = '.'; if (hel == NULL) { warn(".bam.bai without corresponding .bam: %s", el->name); } else { hel->val = (void *)1; } } } // see if we have to add any .bam.bai to the list for(el=list; el; el=el->next) { if (endsWith(el->name,".bam")) { struct hashEl *hel = hashLookup(bamNames, el->name); if (hel->val == NULL) { // we have to add a .bam.bai to the list char *bambai = addSuffix(el->name, ".bai"); warn(".bam.bai not found for corresponding .bam in meta.fileName: %s", el->name); slNameAddTail(&list, bambai); if (hashLookup(allNames, bambai)) { warn("duplicate fileName entry: %s", bambai); } else hashAdd(allNames, bambai, NULL); } } } // make sure the files are there for(el=list; el; el=el->next) { if (!startsWith(mdbObj->obj, el->name)) { warn("fileName %s does not start with object name %s", el->name, mdbObj->obj); } safef(buffer, sizeof buffer, "%s/%s", downDir, el->name); verbose(2, "checking for fileExists %s\n", buffer); if (!fileExists(buffer)) { mdbObj->deleteThis = TRUE; warn("metaDb file %s not found in download dir %s",buffer, downDir); } } } } void checkDownloadOrphans(char *downDir, char *composite, struct hash *allDownloadNames) /* check for orphans in the download directory. */ { verbose(1, "-----------------------------------------------------------------\n"); verbose(1, "Checking that wgEncode* files in the download dir exist in metaDb\n"); verbose(1, "-----------------------------------------------------------------\n"); char buffer[10 * 1024]; safef(buffer, sizeof buffer, "%s", downDir); struct slName *list = listDir(buffer, "wgEncode*"), *el; for(el=list;el;el=el->next) { if (!hashLookup(allDownloadNames, el->name)) { warn("orphan download file %s/%s not found in metaDb\n", buffer, el->name); } } } void checkBbiOrphans(char *database, char *composite, struct hash *allBbiNames) /* check for orphans in the download directory. */ { verbose(1, "-----------------------------------------------------------------\n"); verbose(1, "Checking that <composite>* files in the bbi dir exist in metaDb\n"); verbose(1, "-----------------------------------------------------------------\n"); char buffer[10 * 1024]; char fullPath[10 * 1024]; safef(buffer, sizeof buffer, "/gbdb/%s/bbi", database); struct slName *list = listDir(buffer, addSuffix(composite,"*")), *el; for(el=list;el;el=el->next) { safef(fullPath, sizeof fullPath, "%s/%s", buffer, el->name); if (!hashLookup(allBbiNames, fullPath)) { warn("orphan bbi file %s/%s not found in metaDb\n", buffer, el->name); } } } // TODO this might get removed if not used, was copied from encodeRenameObj.c char *findNewestReleaseDir(char *downDir, char *composite) /* Find the release dir with highest number N (if any) * because the releaseLatest symlink might not really be the latest in use. * return NULL if none found. */ { char buffer[1024]; char releaseTest[1024]; char *releaseN = NULL; int i; for (i=1; i < 10; ++i) { safef(releaseTest, sizeof releaseTest, "release%d", i); safef(buffer, sizeof buffer, "%s/%s/%s", downDir, composite, releaseTest); if (fileExists(buffer)) // dir exists? { releaseN = cloneString(releaseTest); } else break; } return releaseN; } struct hash *getMetaDbHash(char *metaDb, struct mdbObj **head) { boolean validated = FALSE; struct mdbObj *mdbObjs = mdbObjsLoadFromFormattedFile(metaDb, &validated), *mdbObj; struct hash *hash = newHash(10); for(mdbObj = mdbObjs; mdbObj; mdbObj = mdbObj->next) hashAdd(hash, mdbObj->obj, mdbObj); *head = mdbObjs; return hash; } struct hash *getTrackDbHash(char *trackDb, struct trackDb **head, char *release) { struct trackDb *trackObjs = trackDbFromRa(trackDb, release), *trackObj; struct hash *hash = newHash(10); for(trackObj = trackObjs; trackObj; trackObj = trackObj->next) { char *table = trackObj->table; if (table == NULL) table = trackObj->track; hashAdd(hash, table, trackObj); } *head = trackObjs; return hash; } void checkMetaTrackDb(struct mdbObj *mdbObj, struct hash *trackHash) { verbose(1, "----------------------------------------------\n"); verbose(1, "Checking that tables specified in metaDb exist in trackDb\n"); verbose(1, "----------------------------------------------\n"); for(; mdbObj != NULL; mdbObj=mdbObj->next) { struct mdbVar *mdbVar = hashFindVal(mdbObj->varHash, "objType"); if (mdbVar == NULL) { warn("objType not found in object %s", mdbObj->obj); continue; } if (differentString(mdbVar->val, "table")) continue; if (mdbObj->deleteThis) continue; mdbVar = hashFindVal(mdbObj->varHash, "tableName"); if (mdbVar == NULL) { warn("tableName not found in object %s", mdbObj->obj); continue; } char *tableName = mdbVar->val; struct mdbVar *atticVar = hashFindVal(mdbObj->varHash, "attic"); struct mdbVar *statusVar = hashFindVal(mdbObj->varHash, "objStatus"); char *reason = NULL; if (atticVar) reason = "attic"; if (statusVar) { if (startsWith("renamed", statusVar->val)) reason = "renamed"; if (startsWith("replaced", statusVar->val)) reason = "replaced"; if (startsWith("revoked", statusVar->val)) reason = "revoked"; } if (hashLookup(trackHash, tableName)) { if (reason) { warn("%s table %s: should NOT be found in trackDb", reason, tableName); continue; } } else { if (reason) { // ok because attic, replaced, revoked, renamed should not be in trackDb continue; } warn("table %s: not found in trackDb",tableName); } } } struct mdbObj *dropDeleted(struct mdbObj *head) { struct mdbObj *next, *mdbObj, *prev; for(mdbObj = head; mdbObj != NULL; mdbObj = next) { next = mdbObj->next; if (mdbObj->deleteThis) { if (prev == NULL) head = next; else prev->next = next; } else prev = mdbObj; } return head; } void checkFilename(struct sqlConnection *conn, char *table, struct hash *allBbiNames) { char buffer[10 * 1024]; char fileName[10 * 1024]; char oldSymlink[10 * 1024]; verbose(2, "checking for fileName field in table %s \n", table); // see if this is even a bbi table boolean bbiTable = FALSE; struct slName *fnames = sqlFieldNames(conn, table); if ((slCount(fnames) == 1) && (sameString(fnames->name, "fileName"))) bbiTable = TRUE; slFreeList(&fnames); if (!bbiTable) return; sqlSafef(buffer, sizeof buffer, "select fileName from %s limit 1", table); if (sqlQuickQuery(conn, buffer, fileName, sizeof fileName) != NULL) { while(1) // loop to catch .bai as well as .bam { hashAdd(allBbiNames, fileName, NULL); verbose(2,"got table.fileName %s\n", fileName); // file exists FILE *f = fopen(fileName, "r"); if (f == NULL) { warn("fileName %s from table %s can't be opened", fileName, table); return; } else fclose(f); // check that the filename and object base match char *base = strrchr(fileName, '/'); if (base == NULL) { warn("fileName %s in table %s not absolute path", fileName, table); return; } else { base++; char *dot = strchr(base, '.'); if (dot == NULL) { warn("fileName %s in table %s does not have suffix", fileName, table); return; } else { char saveChar = *dot; *dot = 0; if (!sameString(table, base)) { warn("fileName %s doesn't match table %s", base, table); return; } *dot = saveChar; } } // this file is really a symlink, so check its link target ssize_t bufRead = readlink(fileName, oldSymlink, sizeof oldSymlink); if (bufRead == -1) { errnoWarn("error reading symlink %s", fileName); return; } else { oldSymlink[bufRead] = 0; // needs termination. if (!fileExists(oldSymlink)) { warn("symlink target %s does not exist!", oldSymlink); return; } else verbose(2,"got symlink %s\n", oldSymlink); } // check that the symlink and object base match base = strrchr(oldSymlink, '/'); if (base == NULL) { warn("symlink %s in fileName %s not absolute path",oldSymlink, fileName); return; } else { base++; char *dot = strchr(base, '.'); if (dot == NULL) { warn("symlink %s in fileName %s does not have suffix", oldSymlink, fileName); return; } else { char saveChar = *dot; *dot = 0; if (!sameString(table, base)) { warn("symlink %s doesn't match table %s", base, table); return; } *dot = saveChar; } } /* Note "fileIndex" for .bai has been made obsolete so we'll just hard-wire in the .bai support */ if (!endsWith(fileName, ".bam")) break; safecat(fileName, sizeof(fileName), ".bai"); } } } void checkTable(struct sqlConnection *conn, char *table, struct hash *mdbHash, struct hash *allBbiNames) // check to see if table is represented in the metaDb.ra // Also check gbdb referencing tables to see if files exist and have // the correct name { verbose(2, "checking table %s\n", table); if (hashLookup(mdbHash, table) == NULL) warn("table %s is not in metaDb", table); int result = sqlFieldIndex(conn, table, "fileName"); if (result >= 0) checkFilename(conn, table, allBbiNames); } void checkDbTables(char *database, char *composite, struct hash *mdbHash, struct hash *allBbiNames) // search the database for tables that begin with composite and call checkTable { struct sqlConnection *conn = sqlConnect(database); char buffer[10 * 1024]; verbose(1, "----------------------------------------------\n"); verbose(1, "Checking that tables starting with composite in db are in metaDb\n (also checks dummy table and the bbi symlink and its target)\n"); verbose(1, "----------------------------------------------\n"); sqlSafef(buffer, sizeof buffer, "show tables like '%s%%'", composite); struct sqlResult *sr; sr = sqlGetResult(conn, buffer); char **row; struct slName *list = NULL; while ((row = sqlNextRow(sr)) != NULL) { struct slName *el = slNameNew(row[0]); slAddHead(&list, el); } sqlFreeResult(&sr); for(; list; list = list->next) checkTable(conn, list->name, mdbHash, allBbiNames); sqlDisconnect(&conn); } void checkTrackDb(struct trackDb *trackObjs, char *composite, struct hash *mdbHash) // check through trackDb for tables that begin with composite // and check to see if they are represented in the metaDb.ra { struct trackDb *trackObj; verbose(1, "----------------------------------------------\n"); verbose(1, "Checking that tables in trackDb are in metaDb \n"); verbose(1, "----------------------------------------------\n"); for(trackObj = trackObjs; trackObj; trackObj = trackObj->next) { char *table = trackObj->table; if (table == NULL) table = trackObj->track; if (startsWith(composite, table)) { char *compSetting = trackDbSetting(trackObj, "compositeTrack"); if ((compSetting != NULL) && sameString("on", compSetting)) continue; char *viewSetting = trackDbSetting(trackObj, "view"); if (viewSetting != NULL) continue; if (hashLookup(mdbHash, table) == NULL) warn("trackDb entry %s isn't in metaDb", table); } else if (! onlyCompTdb) warn("trackDb entry %s doesn't start with %s", table, composite); } } void metaCheck(char *database, char *composite, char *metaDb, char *trackDb, char *downDir) /* metaCheck - a program to validate that tables, files, and trackDb entries exist. */ { struct mdbObj *mdbObjs = NULL; struct hash *mdbHash = getMetaDbHash(metaDb, &mdbObjs); struct trackDb *trackDbObjs = NULL; struct hash *trackHash = getTrackDbHash(trackDb, &trackDbObjs, release); checkMetaTables(mdbObjs, database); struct hash *allDownloadNames = hashNew(8); checkMetaFiles(mdbObjs, downDir, allDownloadNames); checkDownloadOrphans(downDir, composite, allDownloadNames); checkMetaTrackDb(mdbObjs, trackHash); trackHash = NULL; struct hash *allBbiNames = hashNew(8); checkDbTables(database, composite, mdbHash, allBbiNames); checkBbiOrphans(database, composite, allBbiNames); checkTrackDb(trackDbObjs, composite, mdbHash); if (outMdb) { mdbObjs = dropDeleted(mdbObjs); mdbObjPrintToFile(mdbObjs, TRUE, outMdb); } } char *getSrcDir() /* Get the kent/src/ dir */ { char *pwd = getCurrentDir(); char *s = strstr(pwd, "/src/"); if (!s) s = strstr(pwd, "/src"); if (!s) errAbort("unable to locate src/ directory, must run in your sandbox"); s += strlen("/src"); return cloneStringZ(pwd, s - pwd); } char *findCompositeRa(struct raFile *tdbFile, char *composite, char *release, int *pNumTagsFound) /* Find correct trackDb.ra file for the includer (trackDb.wgEncode.ra) tdbFile * and alpha. If problems, give warnings and return NULL.*/ { char *result = NULL; verbose(1, "\n"); verbose(1, "--------------------------------------------------------------------\n"); verbose(1, "Checking that track is in trackDb.wgEncode.ra with %s tag or none\n", release); verbose(1, "--------------------------------------------------------------------\n"); char *warnMsg = NULL; result = findCompositeInIncluder(tdbFile, composite, release, pNumTagsFound, FALSE, &warnMsg); if (result) { verbose(2, "Found <composite>.ra %s in trackDb.wgEncode.ra for composite %s\n", result, composite); } else { errAbort("%s", warnMsg); } return result; } void replaceTildeWithHome(char **pS) /* if the path starts with ~ (tilde) char, replace with $HOME */ { char *s = *pS; if (s[0] == '~') { *pS = replaceChars(s, "~", getenv("HOME")); } } void printHelp() { fprintf(stderr, "metaCheck tries to report on inconsistencies between all the various data stores that ENCODE uses.\n" "\n" "The checks are divided into four passes:\n" " - checking that metaDb objects of type \"table\" are tables in mySQL\n" " - checking that metaDb objects of type \"file\" are files in download dir\n" " - checking that metaDb objects of type \"table\" are found in trackDb.ra\n" " - checking that all tables in assembly that start with the composite name appear in metaDb\n" " - checking that all tables in assembly that start with the composite name and have a field called \"fileName\"" " are links to a file that can be opened\n" "\n" ); usage(); } int main(int argc, char *argv[]) /* Process command line. */ { optionInit(&argc, argv, options); if (optionExists("help")) { printHelp(); } if (argc != 3) { usage(); } outMdb = optionVal("outMdb", outMdb); onlyCompTdb = optionExists("onlyCompTdb"); release = optionVal("release", release); releaseNum = optionInt("releaseNum", releaseNum); char *database = argv[1]; char *composite = argv[2]; char defaultMetaDb[1024]; char defaultDownloadDir[1024]; char tempDownloadDir[1024]; char *src = getSrcDir(); char *org = cloneString(hOrganism(database)); org[0] = tolower(org[0]); /* If user doesn't provide a metaDB, assume the path using the database and composite */ safef(defaultMetaDb, sizeof(defaultMetaDb), "%s/hg/makeDb/trackDb/%s/%s/metaDb/%s/%s.ra", src, org, database, release, composite); /* If user doesn't provide a downloadDir, assume the path using the database and composite */ safef(defaultDownloadDir, sizeof(defaultDownloadDir), "/usr/local/apache/htdocs-hgdownload/goldenPath/%s/encodeDCC/%s", database, composite); safef(tempDownloadDir, sizeof(tempDownloadDir), "/usr/local/apache/htdocs-hgdownload/goldenPath/%s/encodeDCC/%s", database, composite); if (releaseNum) safef(defaultDownloadDir, sizeof(defaultDownloadDir), "%s/release%d", tempDownloadDir, releaseNum); /* If user doesn't provide a trackDB, assume the path using the database and composite */ char defaultTrackDb[1024]; /* Load encode composite-includer trackDb.wgEncode.ra */ char trackDbIncluder[1024]; safef(trackDbIncluder, sizeof(trackDbIncluder), "%s/hg/makeDb/trackDb/%s/%s/%s", src, org, database, "trackDb.wgEncode.ra"); struct raFile *includerFile = raFileRead(trackDbIncluder); /* Find the correct trackDb.ra for the composite */ int numTagsFound = -1; char *compositeName = findCompositeRa(includerFile, composite, release, &numTagsFound); if (!compositeName) errAbort("unable to find composite .ra for the track in trackDb.wgEncode.ra\n"); // if numTagsFound == 1 then a composite .ra with a single alpha tag exists already, // so no further work required on trackDb.wgEncode.ra safef(defaultTrackDb, sizeof(defaultTrackDb), "%s/hg/makeDb/trackDb/%s/%s/%s", src, org, database, compositeName); verbose(1,"database: %s\ncomposite: %s\nrelease %s\ndefault trackDb: %s\ndefault metaDb: %s\ndefault downloadDir: %s\n", database, composite, release, defaultTrackDb, defaultMetaDb, defaultDownloadDir); char *metaDb = optionVal("metaDb",defaultMetaDb); replaceTildeWithHome(&metaDb); if (!fileExists(metaDb)) errAbort("metaDb %s does not exist.", metaDb); char *trackDb = optionVal("trackDb",defaultTrackDb); replaceTildeWithHome(&trackDb); if (!fileExists(trackDb)) errAbort("trackDb %s does not exist.", trackDb); char *downloadDir = optionVal("downloadDir",defaultDownloadDir); replaceTildeWithHome(&downloadDir); if (!fileExists(downloadDir)) errAbort("downloadDir %s does not exist.", downloadDir); printf("metaDb = %s\n trackDb = %s\n downloadDir = %s\n",metaDb,trackDb,downloadDir); metaCheck(database, composite, metaDb, trackDb, downloadDir); return 0; }