be4311c07e14feb728abc6425ee606ffaa611a58 markd Fri Jan 22 06:46:58 2021 -0800 merge with master diff --git src/hg/makeDb/hgTrackDb/hgTrackDb.c src/hg/makeDb/hgTrackDb/hgTrackDb.c index 7dcc761..b6a1c12 100644 --- src/hg/makeDb/hgTrackDb/hgTrackDb.c +++ src/hg/makeDb/hgTrackDb/hgTrackDb.c @@ -1,36 +1,36 @@ /* hgTrackDb - Create trackDb table from text files. */ + /* Copyright (C) 2013 The Regents of the University of California * See README in this or parent directory for licensing information. */ #include "common.h" #include "linefile.h" #include "hash.h" #include "options.h" #include "sqlList.h" #include "jksql.h" #include "trackDb.h" #include "hui.h" #include "hdb.h" #include "hVarSubst.h" #include "obscure.h" #include "portable.h" #include "dystring.h" #include "regexHelper.h" - void usage() /* Explain usage and exit. */ { errAbort( "hgTrackDb - Create trackDb table from text files.\n\n" "Note that the browser supports multiple trackDb tables, usually\n" "in the form: trackDb_YourUserName. Which particular trackDb\n" "table the browser uses is specified in the hg.conf file found\n" "either in your home directory file '.hg.conf' or in the web server's\n" "cgi-bin/hg.conf configuration file with the setting: db.trackDb=trackDb\n" "see also: src/product/ex.hg.conf discussion of this setting.\n" "usage:\n" " hgTrackDb [options] org database trackDb trackDb.sql hgRoot\n" "\n" "Options:\n" @@ -41,45 +41,49 @@ " trackDb.sql - SQL definition of the table to create, typically from\n" " - the source tree file: src/hg/lib/trackDb.sql\n" " - the table name in the CREATE statement is replaced by the\n" " - table name specified on this command line.\n" " hgRoot - a directory name to prepend to org to locate the hierarchy:\n" " hgRoot/trackDb.ra - top level trackDb.ra file processed first\n" " hgRoot/org/trackDb.ra - second level file processed second\n" " hgRoot/org/database/trackDb.ra - third level file processed last\n" " - for no directory hierarchy use .\n" " -strict - only include tables that exist (and complain about missing html files).\n" " -raName=trackDb.ra - Specify a file name to use other than trackDb.ra\n" " for the ra files.\n" " -release=alpha|beta|public - Include trackDb entries with this release tag only.\n" " -settings - for trackDb scanning, output table name, type line,\n" " - and settings hash to stderr while loading everything." + " -remoteLogin - use remote login to check for the existence of bigDataUrl files\n" ); } static struct optionSpec optionSpecs[] = { {"raName", OPTION_STRING}, {"strict", OPTION_BOOLEAN}, {"release", OPTION_STRING}, {"settings", OPTION_BOOLEAN}, + {"remoteLogin", OPTION_STRING}, {NULL, 0} }; static char *raName = "trackDb.ra"; static char *release = "alpha"; +static char *remoteLogin = NULL; + // release tags #define RELEASE_ALPHA (1 << 0) #define RELEASE_BETA (1 << 1) #define RELEASE_PUBLIC (1 << 2) unsigned releaseBit = RELEASE_ALPHA; static bool showSettings = FALSE; static boolean hasNonAsciiChars(char *text) /* Check if text has any non-printing or non-ascii characters */ { char *c; for (c = text; *c != '\0'; c++) { @@ -112,31 +116,31 @@ * them alive. */ { struct trackDb *newList = NULL, *tdb, *next; for (tdb = tdbList; tdb != NULL; tdb = next) { next = tdb->next; verbose(3,"pruneStrict checking track: '%s'\n", tdb->track); if (tdb->subtracks != NULL) { tdb->subtracks = pruneStrict(tdb->subtracks, db); } if (tdb->subtracks != NULL) { slAddHead(&newList, tdb); } - else if (trackDataAccessible(db, tdb) || tdbIsDownloadsOnly(tdb)) + else if (tdbIsDownloadsOnly(tdb) || trackDataAccessibleRemote(db, tdb, remoteLogin)) { slAddHead(&newList, tdb); } // We will allow these at some point, but currently(10/10/2011) the table browser still chokes. //else if (trackDbSetting(tdb, "bigDataUrl") != NULL) // { // slAddHead(&newList, tdb); // } else verbose(3,"pruneStrict removing track: '%s' no table %s\n", tdb->track, tdb->table); } slReverse(&newList); return newList; } @@ -271,31 +275,31 @@ static void applyOverride(struct hash *trackHash, struct trackDb *overTd) /* Apply a trackDb override to a track, if it exists */ { struct trackDb *tdb = hashFindVal(trackHash, overTd->track); if (tdb != NULL) trackDbOverride(tdb, overTd); } static void addVersionRa(boolean strict, char *database, char *dirName, char *raName, struct hash *trackHash) /* Read in tracks from raName and add them to table, pruning as required. Call * top-down so that track override will work. */ { -struct trackDb *tdbList = trackDbFromRa(raName, NULL), *tdb; +struct trackDb *tdbList = trackDbFromRa(raName, NULL, NULL), *tdb; /* prune records of the incorrect release */ tdbList= pruneRelease(tdbList); /* load tracks, replacing higher-level ones with lower-level and * applying overrides*/ while ((tdb = slPopHead(&tdbList)) != NULL) { if (tdb->overrides != NULL) { verbose(3,"# override '%s'\n", tdb->track); applyOverride(trackHash, tdb); } else { verbose(3,"# track '%s'\n", tdb->track); @@ -443,35 +447,35 @@ { td->html = readHtmlRecursive(fileName, database); // Check for note ASCII characters at higher levels of verboseness. // Normally, these are acceptable ISO-8859-1 characters if ((verboseLevel() >= 2) && hasNonAsciiChars(td->html)) verbose(2, "Note: non-printing or non-ASCII characters in %s\n", fileName); } } } } static char *subsituteVariables(struct hashEl *el, char *database) /* substitute variables where supported */ { char* val = (char*)el->val; +char *name = el->name; /* Only some attribute support variable substitution, at least for now * Just leak memory when doing substitution. */ -if (sameString(el->name, "bigDataUrl") || sameString(el->name, "searchTrix") || - sameString(el->name, "xrefDataUrl")) +if (trackSettingIsFile(name)) { val = replaceChars(val, "$D", database); } return val; } static char *settingsFromHash(struct hash *hash, char* database) /* Create settings string from settings hash. */ { if (hash == NULL) return cloneString(""); else { struct dyString *dy = dyStringNew(1024); @@ -739,30 +743,31 @@ tdbList = trackDbLinkUpGenerations(tdbList); rSetTrackDbFields(tdbList); /* Fill in any additional missing info from defaults. */ rPolish(tdbList); polishSupers(tdbList); /* Optionally check for tables/tracks that actually exist and get rid of ones that don't. */ if (strict) tdbList = pruneStrict(tdbList, database); tdbList = pruneEmptyContainers(tdbList); checkSubGroups(database,tdbList,strict); trackDbPrioritizeContainerItems(tdbList); + return tdbList; } static struct trackDb *flatten(struct trackDb *tdbForest) /* Convert our peculiar forest back to a list. * This for now rescues superTracks from the heavens. */ { struct hash *superTrackHash = hashNew(0); struct slRef *ref, *refList = trackDbListGetRefsToDescendants(tdbForest); struct trackDb *tdbList = NULL; for (ref = refList; ref != NULL; ref = ref->next) { struct trackDb *tdb = ref->val; struct trackDb *parent = tdb->parent; @@ -860,30 +865,31 @@ { verbose(1, "%s: type='%s';", td->track, td->type); if (isNotEmpty(settings)) { char *oneLine = replaceChars(settings, "\n", "; "); eraseTrailingSpaces(oneLine); verbose(1, " %s", oneLine); freeMem(oneLine); } verbose(1, "\n"); } freeMem(settings); } } + sqlUpdate(conn, NOSQLINJ "flush tables"); sqlDisconnect(&conn); verbose(1, "Loaded database %s\n", database); } unlink(tab); } unsigned getReleaseBit(char *release) /* make sure that the tag is a legal release */ { if (sameString(release, "alpha")) return RELEASE_ALPHA; if (sameString(release, "beta")) return RELEASE_BETA; @@ -895,19 +901,20 @@ return 0; /* make compiler happy */ } int main(int argc, char *argv[]) /* Process command line. */ { optionInit(&argc, argv, optionSpecs); if (argc != 6) usage(); raName = optionVal("raName", raName); showSettings = optionExists("settings"); if (strchr(raName, '/') != NULL) errAbort("-raName value should be a file name without directories"); release = optionVal("release", release); releaseBit = getReleaseBit(release); +remoteLogin = optionVal("remoteLogin", remoteLogin); hgTrackDb(argv[1], argv[2], argv[3], argv[4], argv[5], optionExists("strict")); return 0; }