e9ed25747e3d8ab10963306b826f7cedc5e71897 Merge parents b18f2f9facb 992aeef92fe braney Mon Sep 7 11:35:30 2026 -0700 Merge branch 'quickLiftAlign38249' -- alignment tracks in quickLift, refs #38249 # Conflicts: # src/hg/lib/trackHub.c diff --cc src/hg/lib/trackHub.c index 42613d4d88f,7daa2c8f8a9..c70d92d40de --- src/hg/lib/trackHub.c +++ src/hg/lib/trackHub.c @@@ -1,2322 -1,2334 +1,2344 @@@ /* Copyright (C) 2014 The Regents of the University of California * See kent/LICENSE or http://genome.ucsc.edu/license/ for licensing information. */ /* trackHub - supports collections of tracks hosted on a remote site. * The basic layout of a data hub is: * hub.txt - contains information about the hub itself * genomes.txt - says which genomes are supported by hub * Contains file name of trackDb.txt for each genome * trackDb.txt - contains a stanza for each track. Stanzas * are in a subset of the usual trackDb format. * How you use the routines here most commonly is as so: * struct trackHub *hub = trackHubOpen(hubRaUrl); * struct trackHubGenome *hubGenome = trackHubFindGenome(hub, "hg19"); * struct trackDb *tdbList = trackHubTracksForGenome(hub, hubGenome); * // do something with tdbList * trackHubClose(&hub); * Note that the tdbList returned does not have the parent/subtrack pointers set. * It is just a simple list of tracks, not a tree. */ #include "common.h" #include "linefile.h" #include "hash.h" #include "options.h" #include "udc.h" #include "ra.h" #include "filePath.h" #include "htmlPage.h" #include "trackDb.h" #include "htmlSanitize.h" #include "trackHub.h" #include "errCatch.h" #include "hgBam.h" #include "bigWig.h" #include "bigBed.h" #include "barChartUi.h" #include "hdb.h" #include "chromInfo.h" #include "grp.h" #include "twoBit.h" #include "dbDb.h" #include "net.h" #include "bbiFile.h" #include "bPlusTree.h" #include "hgFind.h" #include "hubConnect.h" #include "trix.h" #include "vcf.h" #include "vcfUi.h" #include "htmshell.h" #include "bigBedFind.h" #include "customComposite.h" #include "interactUi.h" #include "bedTabix.h" #include "hic.h" #include "hui.h" #include "chromAlias.h" #include "trashDir.h" #include "hgConfig.h" #include "cartTrackDb.h" #include "quickLift.h" #include "portable.h" #ifdef USE_HAL #include "halBlockViz.h" #endif struct grp *trackHubGrps = NULL; // global with grps loaded from track hubs static struct hash *hubCladeHash; // mapping of clade name to hub pointer static struct hash *hubAssemblyHash; // mapping of assembly name to genome struct static struct hash *hubAssemblyUndecoratedHash; // mapping of undecorated assembly name to genome struct static struct hash *hubOrgHash; // mapping from organism name to hub pointer static struct trackHub *globalAssemblyHubList; // list of trackHubs in the user's cart static struct hash *trackHubHash; static boolean isValidSeqNameChar(char c) /* Return TRUE if c is a valid character for a sequence name: [A-Za-z0-9._-]. */ { return isalnum((unsigned char)c) || c == '.' || c == '_' || c == '-'; } boolean trackHubIsValidSeqName(char *name) /* Return TRUE if name is a valid sequence name: non-empty, starts with a * letter or digit, and contains only [A-Za-z0-9._-]. */ { if (!name || !name[0]) return FALSE; if (!isalnum((unsigned char)name[0])) return FALSE; char *p; for (p = name; *p; p++) if (!isValidSeqNameChar(*p)) return FALSE; return TRUE; } static void tdbListAddHubToGroup(char *hubName, struct trackDb *tdbList) /* Prepend hub name to group name for every tdb. */ { struct trackDb *tdb; for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { char buffer[4096]; char *grp = trackDbSetting(tdb, "group"); if (grp == NULL) tdb->grp = cloneString(hubName); else { safef(buffer, sizeof buffer, "%s_%s", hubName, grp); tdb->grp = cloneString(buffer); } hashReplace(tdb->settingsHash, "group", tdb->grp); } } char *trackHubRelativeUrl(char *hubUrl, char *path) /* Return full path (in URL form if it's a remote hub) given * path possibly relative to hubUrl. Do a freeMem of result * when done. */ { /* If path itself is a URL then just return a copy of it. */ if (hasProtocol(path)) return cloneString(path); /* If it's a remote hub, let html path expander handle it. */ if (hasProtocol(hubUrl) && !startsWith("/gbdb",path)) return expandUrlOnBase(hubUrl, path); /* If we got to here hub is local, and so is path. Do standard * path parsing. */ return pathRelativeToFile(hubUrl, path); } static void badGenomeStanza(struct lineFile *lf) /* Put up semi-informative error message about a genome stanza being bad. */ { errAbort("Genome stanza should have at least two lines, one with 'genome' and one with 'trackDb'\n" "Bad stanza format ending line %d of %s", lf->lineIx, lf->fileName); } char *trackHubCladeToGenome(char *clade) /* Given a track hub clade(hub name) return the default genome. */ { if (hubCladeHash == NULL) return FALSE; struct hashEl *hel = hashLookup(hubCladeHash, clade); if (hel == NULL) return FALSE; struct trackHub *trackHub = hel->val; struct trackHubGenome *hubGenome = trackHub->genomeList; for(; hubGenome; hubGenome=hubGenome->next) if (hubGenome->twoBitPath != NULL) return hubGenome->organism ; return NULL; } struct trackHubGenome *trackHubGetGenomeUndecorated(char *database) /* Get the genome structure for an undecorated genome name. */ { if (hubAssemblyUndecoratedHash == NULL) return NULL; struct hashEl *hel = hashLookup(hubAssemblyUndecoratedHash, database); if (hel == NULL) return NULL; return (struct trackHubGenome *)hel->val; } struct trackHubGenome *trackHubGetGenome(char *database) /* get genome structure for an assembly in a trackHub */ { if (hubAssemblyHash == NULL) errAbort("An error occurred while adding the custom track. This may be due to a mismatch" " between the db= parameter and the genome selected at the top of the page." " If you are having trouble resolving this error please contact us at" " genome-www@soe.ucsc.edu."); struct hashEl *hel = hashLookup(hubAssemblyHash, database); if (hel == NULL) return NULL; return (struct trackHubGenome *)hel->val; } boolean trackHubDatabase(char *database) /* Is this an assembly from an Assembly Data hub? */ { if (hubAssemblyHash == NULL) return FALSE; return trackHubGetGenome(database) != NULL; } char *trackHubDatabaseToGenome(char *db) /* get a database name that is either a genome database or a trackHub * database, return the genome assembly */ { return trackHubDatabase(db) ? trackHubAssemblyField(db, "genome") : db; } char *trackHubAssemblyField(char *database, char *field) /* Get data field from a assembly data hub. */ { struct trackHubGenome *genome = trackHubGetGenome(database); if (genome == NULL) return NULL; char *ret = hashFindVal(genome->settingsHash, field); return cloneString(ret); } static struct dbDb *makeDbDbFromAssemblyGenome(struct trackHubGenome *hubGenome) /* Make a dbdb struture from a single assembly hub database. */ { struct dbDb *db; AllocVar(db); db->genome = cloneString(hubGenome->organism); db->organism = cloneString(hubGenome->organism); db->name = cloneString(hubGenome->name); db->active = TRUE; if (hubGenome->description != NULL) db->description = cloneString(hubGenome->description); else db->description = cloneString(""); char *orderKey = hashFindVal(hubGenome->settingsHash, "orderKey"); if (orderKey != NULL) db->orderKey = sqlUnsigned(orderKey); db->defaultPos = cloneString(hubGenome->defaultPos); return db; } struct dbDb *trackHubDbDbFromAssemblyDb(char *database) /* Return a dbDb structure for just this database. */ { struct trackHubGenome *genome = trackHubGetGenome(database); if (genome == NULL) return NULL; return makeDbDbFromAssemblyGenome(genome); } struct slPair *trackHubGetCladeLabels() /* Get a list of labels describing the loaded assembly data hubs. */ { if (globalAssemblyHubList == NULL) return NULL; struct slPair *clade, *cladeList = NULL; struct trackHub *trackHub = globalAssemblyHubList; for(;trackHub; trackHub = trackHub->next) { AllocVar(clade); slAddHead(&cladeList, clade); clade->name = cloneString(trackHub->name); clade->val = cloneString(trackHub->shortLabel); } return cladeList; } struct dbDb *trackHubGetPcrServers() /* Look through attached trackHubs to see which of them have "isPcr" line in them. */ { struct dbDb *db, *dbList = NULL; if (globalAssemblyHubList != NULL) { struct trackHub *trackHub = globalAssemblyHubList; for(;trackHub; trackHub = trackHub->next) { struct trackHubGenome *hubGenome = trackHub->genomeList; for(; hubGenome; hubGenome = hubGenome->next) { if (hashFindVal(hubGenome->settingsHash,"isPcr") != NULL) { db = makeDbDbFromAssemblyGenome(hubGenome); slAddHead(&dbList, db); } } } } slReverse(&dbList); slSort(&dbList, hDbDbCmpOrderKey); return dbList; } static struct dbDb *getDbDbs(char *clade, boolean blatEnabled) /* Get a list of struct dbDbs from track hubs. Only get blat enabled ones if asked */ { struct dbDb *db, *dbList = NULL; if (globalAssemblyHubList != NULL) { struct trackHub *trackHub = globalAssemblyHubList; for(;trackHub; trackHub = trackHub->next) { if ((clade != NULL) && differentString(clade, trackHub->name)) continue; struct trackHubGenome *hubGenome = trackHub->genomeList; for(; hubGenome; hubGenome = hubGenome->next) { boolean blatCheck = !blatEnabled || ((hashFindVal(hubGenome->settingsHash,"transBlat") != NULL) || (hashFindVal(hubGenome->settingsHash,"blat") != NULL)); if ( blatCheck && (hubGenome->twoBitPath != NULL)) { db = makeDbDbFromAssemblyGenome(hubGenome); slAddHead(&dbList, db); } } } } slReverse(&dbList); slSort(&dbList, hDbDbCmpOrderKey); return dbList; } struct dbDb *trackHubGetBlatDbDbs() /* Get a list of connected track hubs that have blat servers */ { return getDbDbs(NULL, TRUE); } struct dbDb *trackHubGetDbDbs(char *clade) /* Get a list of dbDb structures for all the tracks in this clade/hub. */ { return getDbDbs(clade, FALSE); } struct slPair *trackHubDbDbToValueLabel(struct dbDb *hubDbDbList) /* Given a trackHub (list of) track hub dbDb which may be missing some info, * return an slPair of value and label suitable for making a select/menu option. */ { struct dbDb *dbDb; struct slPair *pairList = NULL; for (dbDb = hubDbDbList; dbDb != NULL; dbDb = dbDb->next) { char *db = dbDb->name; if (isEmpty(db)) db = dbDb->genome; char *label = dbDb->description; if (isEmpty(label)) label = trackHubSkipHubName(db); slAddHead(&pairList, slPairNew(db, cloneString(label))); } slReverse(&pairList); return pairList; } struct slName *trackHubAllChromNames(char *database) /* Return a list of all the chrom names in this assembly hub database. */ /* Free with slFreeList. */ { struct trackHubGenome *genome = trackHubGetGenome(database); if (genome == NULL) return NULL; struct slName *chromList = twoBitSeqNames(genome->twoBitPath); return chromList; } int trackHubChromCount(char *database) /* Return number of chromosomes in a assembly data hub. */ { struct slName *chromList = trackHubAllChromNames(database); int num = slCount(chromList); slFreeList(&chromList); return num; } char *trackHubDefaultChrom(char *database) /* Return the default chromosome for this track hub assembly. */ { struct slName *chromList = trackHubAllChromNames(database); if (chromList == NULL) return NULL; char *defaultName = cloneString( chromList->name); slFreeList(&chromList); return defaultName; } static struct twoBitFile *openTwoBit(struct trackHubGenome *genome) /* Open a twoBit file that may or may not have a bpt index. */ { if (genome->twoBitBptUrl) return twoBitOpenExternalBptIndex(genome->twoBitPath, genome->twoBitBptUrl); else return twoBitOpen(genome->twoBitPath); } struct chromInfo *trackHubMaybeChromInfo(char *database, char *chrom) /* Return a chromInfo structure for just this chrom in this database. The database * may be decorated with a hub_<id>_ prefix or undecorated. * Return NULL if chrom doesn't exist. */ { struct trackHubGenome *genome = NULL; if (hubAssemblyHash != NULL) genome = trackHubGetGenome(database); if (genome == NULL) genome = trackHubGetGenomeUndecorated(trackHubSkipHubName(database)); if (genome == NULL) return NULL; if (genome->tbf == NULL) genome->tbf = openTwoBit(genome); if (!twoBitIsSequence(genome->tbf, chrom)) return NULL; struct chromInfo *ci; AllocVar(ci); ci->chrom = cloneString(chrom); ci->fileName = genome->twoBitPath; ci->size = twoBitSeqSize(genome->tbf, chrom); return ci; } struct chromInfo *trackHubChromInfo(char *database, char *chrom) /* Return a chromInfo structure for just this chrom in this database. * errAbort if chrom doesn't exist. */ { struct chromInfo *ci = trackHubMaybeChromInfo(database, chrom); if (ci == NULL) errAbort("%s is not in %s", chrom, database); return ci; } static char *assemblyHubGenomeSetting(char *database, char *tagName) /* see if this assembly hub has specified tagName, return url if present * returns NULL when not present */ { struct trackHubGenome *genome = trackHubGetGenome(database); if (genome == NULL) return NULL; char *fileName = hashFindVal(genome->settingsHash, tagName); char *absFileName = NULL; if (fileName) absFileName = trackHubRelativeUrl((genome->trackHub)->url, fileName); if (absFileName) { hashReplace(genome->settingsHash, tagName, absFileName); fileName = absFileName; } return fileName; } char *trackHubChromSizes(char *database) /* see if this assembly hub has a chrom.sizes file, return url if present * returns NULL when not present */ { return assemblyHubGenomeSetting(database, "chromSizes"); } char *trackHubAliasBbFile(char *database) /* see if this assembly hub has an alias bigBed file, return url if present * returns NULL when not present */ { return assemblyHubGenomeSetting(database, "chromAliasBb"); } char *trackHubAliasFile(char *database) /* see if this assembly hub has an alias file, return url if present * returns NULL when not present */ { return assemblyHubGenomeSetting(database, "chromAlias"); } struct chromInfo *trackHubAllChromInfo(char *database) /* Return a chromInfo structure for all the chroms in this database. */ { struct trackHubGenome *genome = trackHubGetGenome(database); if (genome == NULL) return NULL; if (genome->tbf == NULL) genome->tbf = openTwoBit(genome); struct chromInfo *ci, *ciList = NULL; struct slName *chromList = twoBitSeqNames(genome->twoBitPath); for(; chromList; chromList = chromList->next) { AllocVar(ci); ci->chrom = cloneString(chromList->name); ci->fileName = cloneString(genome->twoBitPath); ci->size = twoBitSeqSize(genome->tbf, chromList->name); slAddHead(&ciList, ci); } slFreeList(&chromList); return ciList; } static char *getRequiredGrpSetting(struct hash *hash, char *name, struct lineFile *lf) /* Grab a group setting out of the group hash. errAbort if not found. */ { char *str; if ((str = hashFindVal(hash, name)) == NULL) errAbort("missing required setting '%s' for group on line %d in file %s\n", name, lf->lineIx, lf->fileName); return str; } static void checkHubIdName(char *type, char *name) /* Abort if name holds a character that is not valid in an identifier. A hub * identifier - a genome name, a group name - is printed into dozens of URLs, * form values and attributes all over the CGIs, so the check belongs here rather than at every * one of those places. Real names use letters, digits, underscore, dot and dash. */ { if (strchr(name, '<') || strchr(name, '>') || strchr(name, '"') || strchr(name, '\'') || strchr(name, '&')) errAbort("Bad %s name: \"%s\". The characters < > \" ' and & are not allowed in a %s name.", type, name, type); } struct grp *readGroupRa(char *groupFileName) /* Read in the ra file that describes the groups in an assembly hub. */ { if (groupFileName == NULL) return NULL; struct hash *ra; struct grp *list = NULL; struct lineFile *lf = udcWrapShortLineFile(groupFileName, NULL, MAX_HUB_GROUP_FILE_SIZE); while ((ra = raNextRecord(lf)) != NULL) { char *str; struct grp *grp; AllocVar(grp); slAddHead(&list, grp); grp->name = cloneString(getRequiredGrpSetting(ra, "name", lf)); checkHubIdName("group", grp->name); grp->label = cloneString(getRequiredGrpSetting(ra, "label", lf)); grp->priority = BIGDOUBLE; str = hashFindVal(ra, "priority"); if (str != NULL) grp->priority = atof(str); str = hashFindVal(ra, "defaultIsClosed"); if ((str != NULL) && (sameString("on",str) || sameString("1", str))) grp->defaultIsClosed = 1; hashFree(&ra); } if (list) slSort(&list, grpCmpPriorityLabel); lineFileClose(&lf); return list; } struct grp *trackHubLoadGroups(char *database) /* Load the grp structures for this track hub database. */ { struct trackHubGenome *genome = trackHubGetGenome(database); if (genome == NULL) return NULL; struct grp *list = readGroupRa(genome->groups); return list; } char *trackHubGenomeNameToDb(char *genome) /* Return assembly name given a genome name if one exists, otherwise NULL. */ { struct hashEl *hel; if ((hubOrgHash != NULL) && (hel = hashLookup(hubOrgHash, genome)) != NULL) { struct trackHub *hub = hel->val; struct trackHubGenome *genomeList = hub->genomeList; for(; genomeList; genomeList=genomeList->next) if ((genomeList->organism != NULL ) && sameString(genomeList->organism, genome)) return genomeList->name; } return NULL; } char *trackHubAssemblyClade(char *genome) /* Return the clade/hub_name that contains this genome. */ { struct hashEl *hel; if ((hubOrgHash != NULL) && (hel = hashLookup(hubOrgHash, genome)) != NULL) { struct trackHub *hub = hel->val; return cloneString(hub->name); } return NULL; } static void deleteAssembly(char *name, struct trackHubGenome *genome, struct trackHub *hub) /* delete this assembly from the assembly caches */ { hashRemove(hubCladeHash, hub->name); slRemoveEl(&globalAssemblyHubList, hub); hashRemove(hubOrgHash, genome->organism); hashRemove(hubAssemblyHash, genome->name); } static void addAssembly(char *name, struct trackHubGenome *genome, struct trackHub *hub) /* Add a new assembly hub database to our global list. */ { struct hashEl *hel; if (hubCladeHash == NULL) hubCladeHash = newHash(5); if ((hel = hashLookup(hubCladeHash, hub->name)) == NULL) { hashAdd(hubCladeHash, hub->name, hub); slAddHead(&globalAssemblyHubList, hub); } if (hubOrgHash == NULL) hubOrgHash = newHash(5); if ((hel = hashLookup(hubOrgHash, genome->organism)) == NULL) { hashAdd(hubOrgHash, genome->organism, hub); } if (hubAssemblyHash == NULL) hubAssemblyHash = newHash(5); if ((hel = hashLookup(hubAssemblyHash, genome->name)) == NULL) hashAdd(hubAssemblyHash, genome->name, genome); if (hubAssemblyUndecoratedHash == NULL) hubAssemblyUndecoratedHash = newHash(5); if ((hel = hashLookup(hubAssemblyUndecoratedHash, trackHubSkipHubName(genome->name))) == NULL) hashAdd(hubAssemblyUndecoratedHash, trackHubSkipHubName(genome->name), genome); } static char *addHubName(char *base, char *hubName) { if (base == NULL) return NULL; char buffer[4096]; if (isNotEmpty(hubName)) safef(buffer, sizeof(buffer), "%s_%s", hubName, base); else safef(buffer, sizeof(buffer), "%s", base); return cloneString(buffer); } static int genomeOrderKeyCmp(const void *va, const void *vb) /* Compare to sort based on order key */ { const struct trackHubGenome *a = *((struct trackHubGenome **)va); const struct trackHubGenome *b = *((struct trackHubGenome **)vb); if (b->orderKey > a->orderKey) return -1; else if (b->orderKey < a->orderKey) return 1; else return 0; } static struct trackHubGenome *trackHubGenomeReadRa(char *url, struct trackHub *hub, char *singleFile) /* Read in a genome.ra format url and return it as a list of trackHubGenomes. * Also add it to hash, which is keyed by genome. */ { struct lineFile *lf = udcWrapShortLineFile(url, NULL, MAX_HUB_GENOME_FILE_SIZE); struct trackHubGenome *list = NULL, *el; struct hash *hash = hub->genomeHash; struct hash *ra; while ((ra = raNextRecord(lf)) != NULL) { // allow that trackDb+hub+genome is in one single file if (hashFindVal(ra, "hub")) continue; if (hashFindVal(ra, "track")) break; char *twoBitPath = hReplaceGbdb(hashFindVal(ra, "twoBitPath")); char *twoBitBptUrl = hReplaceGbdb(hashFindVal(ra, "twoBitBptUrl")); char *genome, *trackDb; if (twoBitPath != NULL) genome = addHubName(hashFindVal(ra, "genome"), hub->name); else genome = hashFindVal(ra, "genome"); if (hub->defaultDb == NULL) hub->defaultDb = genome; if (genome == NULL) badGenomeStanza(lf); if (hasWhiteSpace(genome)) errAbort("Bad genome name: \"%s\". Only alpha-numeric characters and \"_\" are allowed ([A-Za-z0-9_]).", genome); checkHubIdName("genome", genome); if (hashLookup(hash, genome) != NULL) errAbort("Duplicate genome %s in stanza ending line %d of %s", genome, lf->lineIx, lf->fileName); if (singleFile == NULL) { trackDb = hashFindVal(ra, "trackDb"); if (trackDb == NULL) badGenomeStanza(lf); } else trackDb = singleFile; AllocVar(el); el->name = cloneString(genome); el->trackDbFile = trackHubRelativeUrl(url, trackDb); el->trackHub = hub; hashAdd(hash, el->name, el); hashAdd(hash, hubConnectSkipHubPrefix(el->name), el); slAddHead(&list, el); char *orderKey = hashFindVal(ra, "orderKey"); if (orderKey != NULL) el->orderKey = sqlUnsigned(orderKey); char *groups = hashFindVal(ra, "groups"); if (twoBitPath != NULL) { el->description = hashFindVal(ra, "description"); char *organism = hashFindVal(ra, "organism"); if (organism == NULL) errAbort("must have 'organism' set in assembly hub in stanza ending line %d of %s", lf->lineIx, lf->fileName); el->organism = addHubName(organism, hub->name); hashReplace(ra, "organism", el->organism); el->defaultPos = hashFindVal(ra, "defaultPos"); if (el->defaultPos == NULL) errAbort("must have 'defaultPos' set in assembly hub in stanza ending line %d of %s", lf->lineIx, lf->fileName); el->twoBitPath = trackHubRelativeUrl(url, twoBitPath); if (twoBitBptUrl != NULL) el->twoBitBptUrl = trackHubRelativeUrl(url, twoBitBptUrl); char *htmlPath = hashFindVal(ra, "htmlPath"); if (htmlPath != NULL) hashReplace(ra, "htmlPath",trackHubRelativeUrl(url, htmlPath)); if (groups != NULL) el->groups = trackHubRelativeUrl(url, groups); addAssembly(genome, el, hub); } else { if (groups != NULL) { el->groups = trackHubRelativeUrl(url, groups); } } el->settingsHash = ra; hashAdd(ra, "hubName", hub->shortLabel); el->chromAuthority = hashFindVal(ra, "chromAuthority"); } /* Clean up and go home. */ lineFileClose(&lf); slReverse(&list); slSort(&list, genomeOrderKeyCmp); return list; } char *trackHubSetting(struct trackHub *hub, char *name) /* Return setting if it exists, otherwise NULL. */ { return hashFindVal(hub->settings, name); } char *trackHubRequiredSetting(struct trackHub *hub, char *name) /* Return named setting. Abort with error message if not found. */ { char *val = trackHubSetting(hub, name); if (val == NULL) errAbort("Missing required setting '%s' from %s", name, hub->url); return val; } struct trackHub *grabHashedHub(char *hubName) /* see if a trackHub with this name is in the cache */ { if ( trackHubHash == NULL) trackHubHash = newHash(5); return (struct trackHub *)hashFindVal(trackHubHash, hubName); } static void cacheHub(struct trackHub *hub) { /* put this trackHub in the trackHub hash */ if ( trackHubHash == NULL) trackHubHash = newHash(5); hashAdd(trackHubHash, hub->name, hub); } void uncacheHub(struct trackHub *hub) /* take this trackHub out of the trackHub hash */ { if ( trackHubHash == NULL) return; hashMustRemove(trackHubHash, hub->name); } struct trackHub *trackHubOpen(char *url, char *hubName) /* Open up a track hub from url. Reads and parses hub.txt and the genomesFile. * The hubName is generally just the asciified ID number. */ { struct trackHub *hub = grabHashedHub(hubName); if (hub != NULL) return hub; struct lineFile *lf = udcWrapShortLineFile(url, NULL, MAX_HUB_TRACKDB_FILE_SIZE); struct hash *hubRa = raNextRecord(lf); if (hubRa == NULL) errAbort("empty %s in trackHubOpen", url); // no errAbort when more records in hub.txt file: user can stuff // trackDb into it /* Allocate hub and fill in settings field and url. */ AllocVar(hub); hub->url = cloneString(url); hub->name = cloneString(hubName); hub->settings = hubRa; /* Fill in required fields from settings. */ trackHubRequiredSetting(hub, "hub"); trackHubRequiredSetting(hub, "email"); hub->shortLabel = trackHubRequiredSetting(hub, "shortLabel"); hub->longLabel = trackHubRequiredSetting(hub, "longLabel"); boolean isOneFile = (trackHubSetting(hub, "useOneFile") != NULL); char *ourFile = NULL; if (isOneFile) { ourFile = url; char *root = strrchr(url, '/'); if (root) ourFile = root + 1; hub->genomesFile = cloneString(ourFile); } else hub->genomesFile = trackHubRequiredSetting(hub, "genomesFile"); hub->email = trackHubSetting(hub, "email"); hub->version = trackHubSetting(hub, "version"); // default to current version hub->level = trackHubSetting(hub, "level"); // "core" or "all" char *descriptionUrl = trackHubSetting(hub, "descriptionUrl"); if (descriptionUrl != NULL) hub->descriptionUrl = trackHubRelativeUrl(hub->url, descriptionUrl); lineFileClose(&lf); char *genomesUrl = trackHubRelativeUrl(hub->url, hub->genomesFile); hub->genomeHash = hashNew(8); hub->genomeList = trackHubGenomeReadRa(genomesUrl, hub, ourFile); freez(&genomesUrl); cacheHub(hub); return hub; } void trackHubClose(struct trackHub **pHub) /* Close up and free resources from hub. */ { struct trackHub *hub = *pHub; if (hub != NULL) { trackHubGenomeFreeList(hub); freeMem(hub->url); hashFree(&hub->settings); hashFree(&hub->genomeHash); uncacheHub(hub); freez(pHub); } } void trackHubGenomeFree(struct trackHubGenome **pGenome) /* Free up genome info. */ { struct trackHubGenome *genome = *pGenome; if (genome != NULL) { freeMem(genome->name); freeMem(genome->trackDbFile); freez(pGenome); } } void trackHubGenomeFreeList(struct trackHub *hub) /* Free a list of dynamically allocated trackHubGenome's */ { struct trackHubGenome *el, *next; for (el = hub->genomeList; el != NULL; el = next) { next = el->next; if (el->twoBitPath != NULL) deleteAssembly(el->name, el, hub); trackHubGenomeFree(&el); } hub->genomeList = NULL; } static char *requiredSetting(struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdb, char *setting) /* Fetch setting or give an error message, a little more specific than the * error message from trackDbRequiredSetting(). */ { char *val = trackDbSetting(tdb, setting); if (val == NULL) errAbort("Missing required '%s' setting in hub %s genome %s track %s. Add a '%s' line to the track stanza.", setting, hub->url, genome->name, tdb->track, setting); return val; } static void forbidSetting(struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdb, char *setting) /* Abort if forbidden setting found. */ { if (trackDbSetting(tdb, setting)) errAbort("Forbidden setting '%s' in hub %s genome %s track %s", setting, hub->url, genome->name, tdb->track); } static void expandOneUrl(struct hash *settingsHash, char *hubUrl, char *variable) { struct hashEl *hel = hashLookup(settingsHash, variable); if (hel != NULL) { char *oldVal = hel->val; hel->val = trackHubRelativeUrl(hubUrl, oldVal); char *newPath = (char *) hel->val; if (hasProtocol(hubUrl) && !hasProtocol(newPath)) { // allow local url with udc.localDir path prefix char *prefix = cfgOption("udc.localDir"); if (!(prefix && startsWith(prefix, newPath))) { errAbort("setting %s local URL %s not allowed with non-local host URL %s", variable, newPath, hubUrl); } } freeMem(oldVal); } } static void expandBigDataUrl(struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdb) /* Expand bigDataUrls so that no longer relative to genome->trackDbFile */ { struct hashEl *hel; struct hashCookie cookie = hashFirst(tdb->settingsHash); while ((hel = hashNext(&cookie)) != NULL) { char *name = hel->name; if (trackSettingIsFile(name)) expandOneUrl(tdb->settingsHash, genome->trackDbFile, name); } } struct trackHubGenome *trackHubFindGenome(struct trackHub *hub, char *genomeName) /* Return trackHubGenome of given name associated with hub. Return NULL if no * such genome. Check genomeName without hub prefix to see if this hub * is attached to an assembly hub.*/ { struct trackHubGenome *ret = hashFindVal(hub->genomeHash, genomeName); if (ret == NULL) ret = hashFindVal(hub->genomeHash, hubConnectSkipHubPrefix(genomeName)); return ret; } static void requireBarChartBars(struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdb) /* Fetch setting(s) or give an error message */ { if (!trackDbSetting(tdb, BAR_CHART_CATEGORY_URL) && !trackDbSetting(tdb, BAR_CHART_CATEGORY_LABELS)) errAbort("BarChart track '%s' is missing either %s or %s setting. Please add one of those settings to the appropriate stanza", tdb->track, BAR_CHART_CATEGORY_LABELS, BAR_CHART_CATEGORY_URL); } +boolean trackHubBigNetEnabled() +/* Return TRUE if the bigNet track type is turned on. Off unless hg.conf says + * bigNet=on. Everything that accepts or advertises the type asks this. */ +{ +return cfgOptionBooleanDefault("bigNet", FALSE); +} + static void validateOneTrack( struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdb) /* Validate a track's trackDb entry. */ { /* Check for existence of fields required in all tracks */ requiredSetting(hub, genome, tdb, "shortLabel"); char *shortLabel = trackDbSetting(tdb, "shortLabel"); memSwapChar(shortLabel, strlen(shortLabel), '\t', ' '); requiredSetting(hub, genome, tdb, "longLabel"); char *longLabel = trackDbSetting(tdb, "longLabel"); memSwapChar(longLabel, strlen(longLabel), '\t', ' '); /* Forbid any dangerous settings that should not be allowed */ forbidSetting(hub, genome, tdb, "idInUrlSql"); // subtracks is not NULL if a track said we were its parent if (tdb->subtracks != NULL) { boolean isSuper = FALSE; char *superTrack = trackDbSetting(tdb, "superTrack"); if ((superTrack != NULL) && startsWith("on", superTrack)) isSuper = TRUE; if (!(trackDbSetting(tdb, "compositeTrack") || trackDbSetting(tdb, "container") || isSuper)) { errAbort("Parent track %s is not compositeTrack, container, or superTrack in hub %s genome %s", tdb->track, hub->url, genome->name); } } else { /* Check type field. */ char *type = requiredSetting(hub, genome, tdb, "type"); char *quickLifted = trackDbSetting(tdb, "quickLifted"); if (! ( isCustomComposite(tdb) || (quickLifted != NULL))) { if (startsWithWord("mathWig", type) ) { requiredSetting(hub, genome, tdb, "mathDataUrl"); } else { // **** DANGER, WILL ROBINSON! **** // There is also code in hubCheck.c that checks track names. // both places must be changed or common code created. if (!(startsWithWord("wig", type)|| startsWithWord("bedGraph", type))) { if (!(startsWithWord("bigWig", type) || startsWithWord("bigBed", type) || startsWithWord("pslSnake", type) || startsWithWord("halSnake", type) || startsWithWord("vcfTabix", type) || startsWithWord("vcfPhasedTrio", type) || startsWithWord("bigPsl", type) || startsWithWord("bigMaf", type) || startsWithWord("longTabix", type) || startsWithWord("bigGenePred", type) || startsWithWord("bigNarrowPeak", type) || startsWithWord("bigChain", type) || + (startsWithWord("bigNet", type) && trackHubBigNetEnabled()) || startsWithWord("bigLolly", type) || startsWithWord("bigBaseView", type) || startsWithWord("bigRmsk", type) || startsWithWord("bigBarChart", type) || startsWithWord("bigInteract", type) || startsWithWord("bigMethyl", type) || startsWithWord("hic", type) || startsWithWord("bigDbSnp", type) || startsWithWord("instaPort", type) || startsWithWord("bam", type))) { errAbort("Unsupported type '%s' in hub %s genome %s track %s", type, hub->url, genome->name, tdb->track); } requiredSetting(hub, genome, tdb, "bigDataUrl"); } } if (sameString("barChart", type) || sameString("bigBarChart", type)) requireBarChartBars(hub, genome, tdb); if (sameString("vcfPhasedTrio", type)) requiredSetting(hub, genome, tdb, VCF_PHASED_CHILD_SAMPLE_SETTING); } } } static void markContainers( struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdbList) /* Mark containers that are parents, or have them. */ { struct hash *hash = hashNew(0); struct trackDb *tdb; // add all the track names to a hash for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { if (hashLookup(hash, tdb->track)) errAbort("Track %s appears more than once in genome %s. " "Track identifiers have to be unique. Please check your track hub files, " "especially the 'track' lines. " "The most likely reason for this error is that you duplicated a " "'track' identifier. Hub URL: %s", tdb->track, genome->name, hub->url); hashAdd(hash, tdb->track, tdb); } // go through and find the container tracks for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { char *parentLine = trackDbLocalSetting(tdb, "parent"); // maybe it's a child of a supertrack? if (parentLine == NULL) { parentLine = trackDbLocalSetting(tdb, "superTrack"); if ((parentLine != NULL) && startsWith("on", parentLine)) parentLine = NULL; } if (parentLine != NULL) { char *parentName = cloneFirstWord(parentLine); if (sameString(parentName, tdb->track)) errAbort("Track %s lists itself as its own parent in hub %s genome %s", tdb->track, hub->url, genome->name); struct trackDb *parent = hashFindVal(hash, parentName); if (parent == NULL) errAbort("Parent %s of track %s doesn't exist in hub %s genome %s", parentName, tdb->track, hub->url, genome->name); // mark the parent as a container parent->subtracks = tdb; // ugh...do this so requiredSetting looks at parent // in the case of views. We clear this after // validating anyway tdb->parent = parent; freeMem(parentName); } } hashFree(&hash); } static void validateTracks( struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdbList) /* Make sure a hub track list has the right settings and its parents exist. */ { // mark the containers by setting their subtracks pointer markContainers(hub, genome, tdbList); /* Loop through list checking tags */ struct trackDb *tdb; for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) { validateOneTrack(hub, genome, tdb); } errCatchEnd(errCatch); if (errCatch->gotError) { tdb->errMessage = cloneString(errCatch->message->string); } // clear these two pointers which we set in markContainers tdb->subtracks = NULL; tdb->parent = NULL; } } struct trackDb *trackHubTracksForGenome(struct trackHub *hub, struct trackHubGenome *genome, struct dyString *incFiles, boolean *foundFirstGenome) /* Get list of tracks associated with genome. Check that it only is composed of legal * types. Do a few other quick checks to catch errors early. If incFiles is not NULL, * put the list of included files in there. Only the first example of a genome * gets to populate groups, the others get a group for the trackHub. */ { struct lineFile *lf = udcWrapShortLineFile(genome->trackDbFile, NULL, MAX_HUB_TRACKDB_FILE_SIZE); struct trackDb *tdbList = trackDbFromOpenRa(lf, NULL, incFiles); lineFileClose(&lf); char *tabMetaName = hashFindVal(genome->settingsHash, "metaTab"); char *absTabName = NULL; if (tabMetaName) absTabName = trackHubRelativeUrl(hub->url, tabMetaName); char *tagStormName = hashFindVal(genome->settingsHash, "metaDb"); char *absStormName = NULL; if (tagStormName) absStormName = trackHubRelativeUrl(hub->url, tagStormName); /* Make bigDataUrls more absolute rather than relative to genome.ra dir */ struct trackDb *tdb; for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { expandBigDataUrl(hub, genome, tdb); if (absStormName) hashReplace(tdb->settingsHash, "metaDb", absStormName); if (absTabName) hashReplace(tdb->settingsHash, "metaTab", absTabName); } validateTracks(hub, genome, tdbList); trackDbAddTableField(tdbList); if (!isEmpty(hub->name)) trackHubAddNamePrefix(hub->name, tdbList); if ((genome->twoBitPath != NULL) && (*foundFirstGenome == FALSE)) *foundFirstGenome = TRUE; else if ((genome->groups != NULL) && hubsCanAddGroups()) tdbListAddHubToGroup(hub->name, tdbList); else trackHubAddGroupName(hub->name, tdbList); for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { trackDbFieldsFromSettings(tdb); trackDbPolish(tdb); } return tdbList; } static void reprefixString(char **pString, char *prefix) /* Replace *pString with prefix + *pString, freeing * whatever was in *pString before. */ { char *oldName = *pString; *pString = catTwoStrings(prefix, oldName); freeMem(oldName); } static void addPrefixToSetting(struct hash *settings, char *key, char *prefix) /* Given a settings hash, which is string valued. Old values will be freed. */ { struct hashEl *hel = hashLookup(settings, key); if (hel != NULL) reprefixString((char **)&hel->val, prefix); } static void trackDbListAddNamePrefix(struct trackDb *tdbList, char *prefix) /* Surgically alter tdbList so that it works as if every track was * renamed so as to add a prefix to it's name. */ { struct trackDb *tdb; for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { addPrefixToSetting(tdb->settingsHash, "track", prefix); addPrefixToSetting(tdb->settingsHash, "parent", prefix); reprefixString(&tdb->track, prefix); if (tdb->table != NULL) reprefixString(&tdb->table, prefix); } } void trackHubAddNamePrefix(char *hubName, struct trackDb *tdbList) /* For a hub named "hub_1" add the prefix "hub_1_" to each track and parent field. */ { char namePrefix[PATH_LEN]; safef(namePrefix, sizeof(namePrefix), "%s_", hubName); trackDbListAddNamePrefix(tdbList, namePrefix); } char *trackHubGetHubName(char *name) /* Get the hub_#_ prefix in a hub name. */ { if ((name == NULL) || !startsWith("hub_", name)) return NULL; char *ret = cloneString(name); char *under = strchr(&ret[4], '_'); if (under) { *under = 0; return ret; } return NULL; } char *trackHubSkipHubName(char *name) /* Skip the hub_#_ prefix in a hub name. */ { if ((name == NULL) || !startsWith("hub_", name)) return name; return strchr(&name[4], '_') + 1; } struct trackDb *findTdbByBareName(struct trackDb *tdbList, char *bareName) /* Recursively search tdbList (and subtracks) for a tdb whose bare track name matches. */ { struct trackDb *tdb; for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { if (sameString(trackHubSkipHubName(tdb->track), bareName)) return tdb; struct trackDb *found = findTdbByBareName(tdb->subtracks, bareName); if (found != NULL) return found; } return NULL; } void trackHubAddGroupName(char *hubName, struct trackDb *tdbList) /* Add group tag that references the hubs symbolic name. */ { struct trackDb *tdb; for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { tdb->grp = cloneString(hubName); hashReplace(tdb->settingsHash, "group", tdb->grp); } } static char *trackHubDescriptionText(char *trackDbFile, struct trackDb *tdb) /* Fetch the text of tdb->track's html description page, or NULL if it has none. */ { /* html setting should always be set because we set it at load time */ char *htmlName = trackDbSetting(tdb, "html"); if (htmlName == NULL) return NULL; char *simpleName = hubConnectSkipHubPrefix(htmlName); char *url = trackHubRelativeUrl(trackDbFile, simpleName); char buffer[10*1024]; char *fixedUrl = url; if (!endsWith(url, ".html")) { safef(buffer, sizeof buffer, "%s.html", url); fixedUrl = buffer; } char *html = udcFileReadAllIfExists(fixedUrl, NULL, 0, NULL); freez(&url); return html; } void trackHubAddOneDescription(char *trackDbFile, struct trackDb *tdb) /* Fetch tdb->track's html description and store in tdb->html. */ { char *html = trackHubDescriptionText(trackDbFile, tdb); if (html == NULL) return; /* no page of its own, so leave any it inherited alone */ tdb->html = htmlSanitize(html); freeMem(html); } struct slName *trackHubDescriptionRemovals(char *trackDbFile, struct trackDb *tdb) /* Return a list of messages naming the parts of tdb's description page that we do not * print, or NULL if we print all of it. */ { char *html = trackHubDescriptionText(trackDbFile, tdb); struct slName *removed = NULL; char *clean = htmlSanitizeReport(html, &removed); freeMem(html); freeMem(clean); return removed; } void trackHubAddDescription(char *trackDbFile, struct trackDb *tdb) /* Fetch tdb->track's html description (or nearest ancestor's non-empty description) * and store in tdb->html. */ { trackHubAddOneDescription(trackDbFile, tdb); if (isEmpty(tdb->html)) { struct trackDb *parent; for (parent = tdb->parent; isEmpty(tdb->html) && parent != NULL; parent = parent->parent) { trackHubAddOneDescription(trackDbFile, parent); if (isNotEmpty(parent->html)) tdb->html = cloneString(parent->html); } } } void trackHubFixName(char *name) /* Change all characters other than alphanumeric, dash, and underbar * to underbar. */ { if (name == NULL) return; char *in = name; char c; for(; (c = *in) != 0; in++) { if (c == ' ') break; if (!(isalnum(c) || c == '-' || c == '_')) *in = '_'; } } static void polishOneTrack( struct trackHub *hub, struct trackDb *bt, struct hash *hash) /* Get rid of special characters in track name, squirrel away a copy * of the original name for html retrieval, make sure there aren't * two tracks with the same name. */ { char *polished = trackDbLocalSetting(bt, "polished"); if (polished != NULL) return; trackDbAddSetting(bt, "polished", "polished"); char *htmlName = trackDbSetting(bt, "html"); /* if the user didn't specify an html variable, set it to be the original * track name */ if (htmlName == NULL) trackDbAddSetting(bt, "html", bt->track); trackHubFixName(bt->track); if (hashLookup(hash, bt->track) != NULL) errAbort("more than one track called %s in hub %s\n", bt->track, hub->url); hashStore(hash, bt->track); } void trackHubPolishTrackNames(struct trackHub *hub, struct trackDb *tdbList) /* Remove all the special characters from trackHub track names. */ { struct trackDb *next, *tdb; struct hash *nameHash = hashNew(5); for (tdb = tdbList; tdb != NULL; tdb = next) { if (tdb->parent != NULL) polishOneTrack(hub, tdb->parent, nameHash); next = tdb->next; polishOneTrack(hub, tdb, nameHash); if (tdb->subtracks != NULL) { trackHubPolishTrackNames(hub, tdb->subtracks); } } } void trackHubFindPos(struct cart *cart, char *db, char *term, struct hgPositions *hgp, boolean measureTiming) /* Look for term in track hubs. Update hgp if found */ { findBigBedPosInTdbList(cart, db, hubCollectTracks(db, NULL), term, hgp, NULL, measureTiming); } static void parseBlatPcrParams(char *database, char *type, char *setting, char **pHost, char **pPort, char **pGenomeDataDir) /* parser parameters for either blat or pcr */ { char *conf = trimSpaces(cloneString(setting)); int numWords = chopByWhite(conf, NULL, 5); if ((numWords < 2) || (numWords > 4)) errAbort("invalid configuration for hub %s server, expect 2 or 4 arguments: %s", type, setting); char *words[4]; chopByWhite(conf, words, numWords); *pHost = words[0]; *pPort = words[1]; if (numWords > 2) { if (!sameString(words[2], "dynamic")) errAbort("invalid configuration for hub %s server, third argument should be 'dynamic' or omitted, got: %s", type, words[2]); *pGenomeDataDir = words[3]; } else *pGenomeDataDir = NULL; } boolean trackHubGetPcrParams(char *database, char **pHost, char **pPort, char **pGenomeDataDir) /* Get the isPcr params from a trackHub genome. */ { char *type = "isPcr"; char *setting = trackHubAssemblyField(database, type); parseBlatPcrParams(database, type, setting, pHost, pPort, pGenomeDataDir); return TRUE; } boolean trackHubGetBlatParams(char *database, boolean isTrans, char **pHost, char **pPort, char **pGenomeDataDir) { char *type = isTrans ? "transBlat" : "blat"; char *setting = trackHubAssemblyField(database, type); if (setting == NULL) return FALSE; parseBlatPcrParams(database, type, setting, pHost, pPort, pGenomeDataDir); return TRUE; } void hubCheckBigDataUrl(struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdb) /* Check remote file exists and is of correct type. Wrap this in error catcher */ { -char *relativeUrl = trackDbSetting(tdb, "bigDataUrl"); -if (relativeUrl != NULL) +/* trackHubTracksForGenome() has already run expandBigDataUrl() on this tdb, so + * bigDataUrl and bigDataIndex are no longer relative to genome->trackDbFile and must + * be used as they are. Resolving them a second time here prepended the hub directory + * twice whenever a local hub was reached by a relative path, so that + * "hubCheck out/hub.txt" looked for out/out/hg19/x.bb and reported it missing. */ +char *bigDataUrl = trackDbSetting(tdb, "bigDataUrl"); +if (bigDataUrl != NULL) { char *type = trackDbRequiredSetting(tdb, "type"); - char *bigDataUrl = trackHubRelativeUrl(genome->trackDbFile, relativeUrl); - - char *bigDataIndex = NULL; - char *relIdxUrl = trackDbSetting(tdb, "bigDataIndex"); - if (relIdxUrl != NULL) - bigDataIndex = trackHubRelativeUrl(genome->trackDbFile, relIdxUrl); + char *bigDataIndex = trackDbSetting(tdb, "bigDataIndex"); verbose(2, "checking %s.%s type %s at %s\n", genome->name, tdb->track, type, bigDataUrl); if (startsWithWord("bigWig", type)) { /* Just open and close to verify file exists and is correct type. */ struct bbiFile *bbi = bigWigFileOpen(bigDataUrl); bbiFileClose(&bbi); } else if (startsWithWord("bigNarrowPeak", type) || startsWithWord("bigBed", type) || startsWithWord("bigGenePred", type) || startsWithWord("bigPsl", type)|| - startsWithWord("bigChain", type)|| startsWithWord("bigMaf", type) || + startsWithWord("bigChain", type)|| startsWithWord("bigNet", type) || + startsWithWord("bigMaf", type) || startsWithWord("bigBarChart", type) || startsWithWord("bigInteract", type) || startsWithWord("bigLolly", type) || startsWithWord("bigRmsk",type) || startsWithWord("bigMethyl", type)) { /* Just open and close to verify file exists and is correct type. */ struct bbiFile *bbi = bigBedFileOpen(bigDataUrl); char *typeString = cloneString(type); nextWord(&typeString); if (startsWithWord("bigBed", type) && (typeString != NULL)) { unsigned numFields = sqlUnsigned(nextWord(&typeString)); if (numFields > bbi->fieldCount) errAbort("bigBed file '%s' has %d fields, but track \"%s\" declares 'type bigBed %d'. Either regenerate the bigBed with the correct number of fields, or change the type line to match the file's field count.", bigDataUrl, bbi->fieldCount, trackHubSkipHubName(tdb->track), numFields); } bbiFileClose(&bbi); } else if (startsWithWord("vcfTabix", type) || startsWithWord("vcfPhasedTrio", type)) { /* Just open and close to verify file exists and is correct type. */ struct vcfFile *vcf = vcfTabixFileAndIndexMayOpen(bigDataUrl, bigDataIndex, NULL, 0, 0, 1, 1); if (vcf == NULL) // Warnings already indicated whether the tabix file is missing etc. errAbort("Couldn't open %s and/or its tabix index (.tbi) file for track %s. " "Both the .vcf.gz file and a matching .vcf.gz.tbi index must be publicly accessible " "at the same URL path. Generate the index with: tabix -p vcf yourFile.vcf.gz " "See https://genome.ucsc.edu/goldenPath/help/vcf.html", bigDataUrl, trackHubSkipHubName(tdb->track)); vcfFileFree(&vcf); } else if (startsWithWord("bam", type)) { bamFileAndIndexMustExist(bigDataUrl, bigDataIndex); } else if (startsWithWord("longTabix", type)) { struct bedTabixFile *btf = bedTabixFileMayOpen(bigDataUrl, NULL, 0, 0); if (btf == NULL) errAbort("Couldn't open %s and/or its tabix index (.tbi) file for track %s. " "Both the file and a matching .tbi index must be publicly accessible " "at the same URL path.", bigDataUrl, trackHubSkipHubName(tdb->track)); bedTabixFileClose(&btf); } else if (startsWithWord("halSnake", type)) { #ifdef USE_HAL char *errString; int handle = halOpenLOD(bigDataUrl, &errString); if (handle < 0) errAbort("HAL open error: %s", errString); if (halClose(handle, &errString) < 0) errAbort("HAL close error: %s", errString); #else warn("Note: can not check track %s, validation of halSnake tracks not compiled into this copy of hubCheck", trackHubSkipHubName(tdb->track)); #endif } else if (startsWithWord("hic", type)) { struct hicMeta *header; char *errString = hicLoadHeader(bigDataUrl, &header, genome->name); if (errString != NULL) errAbort("hic file error: %s", errString); } else errAbort("unrecognized type %s in genome %s track %s", type, genome->name, tdb->track); - freez(&bigDataUrl); } } static void outHubHeader(struct dyString *dy, char *db) // output a track hub header { dyStringPrintf(dy,"hub quickLiftHub%s\n\ shortLabel QuickLift from %s\n\ longLabel QuickLift from %s\n\ useOneFile on\n\ email genome-www@soe.ucsc.edu\n\n", db, db, db); dyStringPrintf(dy,"genome %s\n\n", db); } static char *getHubName(struct cart *cart, char *db) // get the name of the hub to use for quickLifted tracks { struct tempName hubTn; char buffer[4096]; #define quickLiftCartName "hubQuickLift" safef(buffer, sizeof buffer, "%s-%s", quickLiftCartName, db); char *hubName = cartOptionalString(cart, buffer); int fd = -1; // we don't reuse userdata paths since they are in save sessions if ((hubName != NULL) && strstr(hubName, "userdata")) hubName = NULL; if ((hubName == NULL) || ((fd = open(hubName, 0)) < 0)) { trashDirDateFile(&hubTn, "quickLift", "hub", ".txt"); hubName = cloneString(hubTn.forCgi); cartSetString(cart, buffer, hubName); } return hubName; } struct quickLiftStanza /* One track stanza parsed out of a quickLift hub file. */ { struct quickLiftStanza *next; char *name; /* bare track name */ char *parent; /* bare parent track name, or NULL */ struct dyString *text; /* full stanza text including final newline */ }; static char *firstWordClone(char *s) /* Return a clone of the first whitespace-delimited word of s, or NULL. */ { s = skipLeadingSpaces(s); if (isEmpty(s)) return NULL; char *sp = skipToSpaces(s); int len = (sp != NULL) ? (sp - s) : (int)strlen(s); return cloneStringZ(s, len); } static struct quickLiftStanza *readStanzas(struct lineFile *lf, struct dyString *header) /* Read track stanzas out of a quickLift hub file, or out of a string of freshly * generated stanzas. Everything ahead of the first track line goes into header. * A container is many stanzas here, one per track line, not a single block. */ { struct quickLiftStanza *stanzaList = NULL; struct quickLiftStanza *cur = NULL; char *line; int lineSize; while (lineFileNext(lf, &line, &lineSize)) { char *trim = skipLeadingSpaces(line); if (startsWithWord("track", trim)) { AllocVar(cur); cur->text = dyStringNew(0); cur->name = firstWordClone(trim + 5); slAddHead(&stanzaList, cur); } else if (cur != NULL && startsWithWord("parent", trim)) { if (cur->parent == NULL) cur->parent = firstWordClone(trim + 6); } struct dyString *target = (cur != NULL) ? cur->text : header; dyStringAppend(target, line); dyStringAppendC(target, '\n'); } slReverse(&stanzaList); return stanzaList; } static void freeStanzas(struct quickLiftStanza **pList) /* Free a list of stanzas. */ { struct quickLiftStanza *s; for (s = *pList; s != NULL; s = s->next) { dyStringFree(&s->text); freeMem(s->name); freeMem(s->parent); } slFreeList(pList); } boolean quickLiftHubRemoveTrack(struct cart *cart, char *sourceDb, char *trackName) /* Remove a track stanza from the quickLift hub file for sourceDb, along with * any descendant stanzas (transitively) whose parent is being removed. * Returns TRUE if at least one stanza was removed. */ { char buffer[4096]; safef(buffer, sizeof buffer, "%s-%s", quickLiftCartName, sourceDb); char *filename = cartOptionalString(cart, buffer); if (filename == NULL || !isServerUserFilePath(filename)) return FALSE; struct lineFile *lf = lineFileMayOpen(filename, TRUE); if (lf == NULL) return FALSE; char *bareName = trackHubSkipHubName(trackName); struct dyString *header = dyStringNew(0); /* Pass 1: read the file into a header + list of stanzas, recording each * stanza's name and (if any) parent. */ struct quickLiftStanza *stanzaList = readStanzas(lf, header); lineFileClose(&lf); /* Build a removal set: start with the named track, then iterate adding any * stanza whose parent is already in the set, until the set is stable. */ struct hash *removeSet = hashNew(0); hashStore(removeSet, bareName); boolean changed = TRUE; while (changed) { changed = FALSE; struct quickLiftStanza *s; for (s = stanzaList; s != NULL; s = s->next) { if (s->name == NULL || s->parent == NULL) continue; if (hashLookup(removeSet, s->name) != NULL) continue; if (hashLookup(removeSet, s->parent) != NULL) { hashStore(removeSet, s->name); changed = TRUE; } } } boolean removedAny = FALSE; struct dyString *out = dyStringNew(0); struct quickLiftStanza *s; for (s = stanzaList; s != NULL; s = s->next) { if (s->name != NULL && hashLookup(removeSet, s->name) != NULL) removedAny = TRUE; else dyStringAppend(out, s->text->string); } if (removedAny) { FILE *f = mustOpen(filename, "w"); chmod(filename, 0666); fputs(header->string, f); fputs(out->string, f); fclose(f); } dyStringFree(&header); dyStringFree(&out); hashFree(&removeSet); freeStanzas(&stanzaList); return removedAny; } static char *vettedTracks[] = /* tracks that have been tested with quickLift */ { "decipherContainer", "decipherSnvs", "omimLocation", "omimAvSnp", "ncbiRefSeq", "clinvar", "clinvarSubLolly", "pubs", "pubsBlat", "pubsMarkerBand", "pubsMarkerSnp", "pubsMarkerGene", "refSeqComposite", //"gtexGeneV8", }; static boolean isVetted(char *track) /* Is this a track that's been tested with quickLift? If not we don't want to do the special name handling on the track. */ { if (startsWith("wgEncodeGencode", track)) return TRUE; // NCBI RefSeq composite subtracks (ncbiRefSeqCurated/Predicted/Other/Psl/ // Select/Hgmd/Historical, ncbiOrtho) and UCSC RefSeq (refGene) have been // validated through quickLift; let their native hgc handlers fire. if (startsWith("ncbiRefSeq", track) || startsWith("ncbiOrtho", track) || sameString("refGene", track)) return TRUE; static bool inited = FALSE; static struct hash *vettedHash = NULL; if (!inited) { vettedHash = newHash(10); int ii; int len = sizeof(vettedTracks) / sizeof(char *); for(ii = 0; ii < len; ii++) hashStore(vettedHash, vettedTracks[ii]); } return hashLookup(vettedHash, track) != NULL; } static bool isParentVisible(struct cart *cart, struct trackDb *tdb) // Are this track's parents visible? { if (tdb->parent == NULL) return TRUE; if (!isParentVisible(cart, tdb->parent)) return FALSE; char *cartVis = cartOptionalString(cart, tdb->parent->track); boolean vis; if (cartVis != NULL) vis = differentString(cartVis, "hide"); else if (tdbIsSuperTrack(tdb->parent)) vis = tdb->parent->isShow; else vis = tdb->parent->visibility != tvHide; return vis; } static bool subtrackEnabledInTdb(struct trackDb *subTdb) /* Return TRUE unless the subtrack was declared with "subTrack ... off". */ { bool enabled = TRUE; char *words[2]; char *setting; if ((setting = trackDbLocalSetting(subTdb, "parent")) != NULL) { if (chopLine(cloneString(setting), words) >= 2) if (sameString(words[1], "off")) enabled = FALSE; } else return subTdb->visibility != tvHide; return enabled; } static bool isSubtrackVisible(struct cart *cart, struct trackDb *tdb) /* Has this subtrack not been deselected in hgTrackUi or declared with * * "subTrack ... off"? -- assumes composite track is visible. */ { boolean overrideComposite = (NULL != cartOptionalString(cart, tdb->track)); bool enabledInTdb = subtrackEnabledInTdb(tdb); char option[1024]; safef(option, sizeof(option), "%s_sel", tdb->track); boolean enabled = cartUsualBoolean(cart, option, enabledInTdb); if (overrideComposite) enabled = TRUE; return enabled; } static void dumpTdbAndChildren(struct cart *cart, struct dyString *dy, struct trackDb *tdb) /* Put a trackDb entry into a dyString, stepping up the tree for some variables. */ { struct hashCookie cookie = hashFirst(tdb->settingsHash); struct hashEl *hel; char *cartVis = cartOptionalString(cart, tdb->track); if (cartVis != NULL) tdb->visibility = hTvFromString(cartVis); dyStringPrintf(dy, "visibility %s\n", hStringFromTv(tdb->visibility)); if (tdbIsSuperTrack(tdb->parent)) dyStringPrintf(dy, "parent %s\n", trackHubSkipHubName(tdb->parent->track)); while ((hel = hashNext(&cookie)) != NULL) { if (sameString(hel->name, "parent")) { char buffer[1024]; safef(buffer, sizeof buffer, "%s_sel", tdb->track); char *cartSelected = cartOptionalString(cart, buffer); if (cartSelected != NULL) { char *str = (sameWord(cartSelected, "on") || atoi(cartSelected) > 0) ? "on" : "off"; dyStringPrintf(dy, "parent %s %s\n", trackHubSkipHubName(tdb->parent->track), str); } else dyStringPrintf(dy, "%s %s\n", hel->name, trackHubSkipHubName(((char *)hel->val))); } else if (sameString(hel->name, "html")) dyStringPrintf(dy, "%s %s\n", hel->name, trackHubSkipHubName((char *)hel->val)); else if (differentString(hel->name, "track") && differentString(hel->name, "visibility")) dyStringPrintf(dy, "%s %s\n", hel->name, ((char *)hel->val)); } if (tdb->subtracks) { for (tdb = tdb->subtracks; tdb; tdb = tdb->next) { char *track = trackHubSkipHubName(tdb->track); dyStringPrintf(dy, "\ntrack %s\nquickLifted on\n", track); if (!isVetted(track)) dyStringPrintf(dy, "avoidHandler on\n"); dumpTdbAndChildren(cart, dy, tdb); } } } struct dyString *trackDbString(struct cart *cart, struct trackDb *tdb) /* Convert a trackDb entry into a dyString. */ { struct dyString *dy; // add a note that the name based handler shouldn't be used on this track // add a note that this is a quickLifted track so the browser will accept tracks that aren't big* dy = dyStringNew(200); char *track = trackHubSkipHubName(tdb->track); dyStringPrintf(dy, "track %s\nquickLifted on\n", track); if (tdbIsSuperTrack(tdb)) { dyStringPrintf(dy, "superTrack on show\n"); } if (!isVetted(track)) dyStringPrintf(dy, "avoidHandler on\n"); dumpTdbAndChildren(cart, dy, tdb); return dy; } - static boolean validateOneTdb(char *db, struct trackDb *tdb, struct trackDb **badList) + static boolean isAlignmentType(char *type) + /* The alignment types quickLift can lift. These are newer than the rest of quickLift and + * are gated in hg.conf, so quickLiftAlignmentsEnabled decides whether one may enter the + * hub. This is the only door: quickLiftUrl and quickLiftDb, the pair every lift path + * keys off, are written by the quickLift hub writer and by nothing else. */ + { + // trackDb types are matched without regard to case since that's how the rest of the + // browser reads them (some trackDb stanzas say "bigbed" rather than "bigBed"). + return startsWithNoCase("bigPsl", type) || + startsWithNoCase("bigChain", type) || + startsWithNoCase("bigMaf", type) || + startsWithNoCase("wigMaf", type) || + sameWord("chain", type) || + startsWithNoCase("chain ", type) || + sameWord("psl", type) || + startsWithNoCase("psl ", type); + } + + static boolean validateOneTdb(struct cart *cart, char *db, struct trackDb *tdb, struct trackDb **badList) /* Make sure the tdb is a track type we grok. badList may be NULL to validate * silently (no user-facing complaint about non-liftable types). */ { // trackDb types are matched without regard to case since that's how the rest of the // browser reads them (some trackDb stanzas say "bigbed" rather than "bigBed"). if (sameString("cytoBandIdeo", trackHubSkipHubName(tdb->track)) || !( startsWithNoCase("bigBed", tdb->type) || \ startsWithNoCase("bigWig", tdb->type) || \ startsWithNoCase("bigDbSnp", tdb->type) || \ startsWithNoCase("bigGenePred", tdb->type) || \ startsWithNoCase("gvf", tdb->type) || \ startsWithNoCase("genePred", tdb->type) || \ startsWithNoCase("narrowPeak", tdb->type) || \ startsWithNoCase("broadPeak", tdb->type) || \ startsWithNoCase("bigLolly", tdb->type) || \ + (startsWithNoCase("bigNet", tdb->type) && trackHubBigNetEnabled()) || \ + (isAlignmentType(tdb->type) && quickLiftAlignmentsEnabled(cart)) || \ sameWord("bed", tdb->type) || startsWithNoCase("bed ", tdb->type))) { if (badList != NULL) slAddHead(badList, tdb); return FALSE; } // make sure we have a bigDataUrl if (startsWithNoCase("bigBed", tdb->type) || \ + startsWithNoCase("bigNet", tdb->type) || \ + startsWithNoCase("bigPsl", tdb->type) || \ + startsWithNoCase("bigChain", tdb->type) || \ + startsWithNoCase("bigMaf", tdb->type) || \ startsWithNoCase("bigWig", tdb->type)) { char *fileName = cloneString(trackDbSetting(tdb, "bigDataUrl")); if (fileName == NULL) { struct sqlConnection *conn = hAllocConnTrack(db, tdb); fileName = bbiNameFromSettingOrTable(tdb, conn, tdb->table); hashAdd(tdb->settingsHash, "bigDataUrl", fileName); hFreeConn(&conn); } } return TRUE; } static struct trackDb * validateTdbChildren(struct cart *cart, char *db, struct trackDb *tdb, struct trackDb **badList) /* return a list of the children that can be quick lifted */ { struct trackDb *validTdbs = NULL; struct trackDb *nextTdb; unsigned count = 0; if (tdb->subtracks) // this is a view, descend again { struct trackDb *view = tdb; for (; view; view = nextTdb) { nextTdb = view->next; view->subtracks = validateTdbChildren(cart, db,view->subtracks, badList); if (view->subtracks != NULL) { slAddHead(&validTdbs, view); if (view->visibility) count++; } } } else { for(; tdb; tdb = nextTdb) { nextTdb = tdb->next; boolean visible = isParentVisible(cart, tdb) && isSubtrackVisible(cart, tdb); // Lift all siblings of a visible subtrack, but only complain about // non-liftable ones the user actually asked for (visible ones). - if (validateOneTdb(db, tdb, visible ? badList : NULL)) + if (validateOneTdb(cart, db, tdb, visible ? badList : NULL)) { slAddHead(&validTdbs, tdb); if (visible) count++; } } } if (count) return validTdbs; return NULL; } static boolean validateTdb(struct cart *cart, char *db, struct trackDb *tdb, struct trackDb **badList) // make sure we only output track types that can // be quickLifted. Return true if we any tracks survive { if (tdb->subtracks) { tdb->subtracks = validateTdbChildren(cart, db, tdb->subtracks, badList); if (tdb->subtracks == NULL) return FALSE; return TRUE; } - return validateOneTdb(db, tdb, badList); + return validateOneTdb(cart, db, tdb, badList); } static void outTrack(struct dyString *out, struct cart *cart, struct trackDb *tdb, double priority) /* Set priority and output track to hub. */ { char buffer[1024]; safef(buffer, sizeof buffer, "%g", priority); hashReplace(tdb->settingsHash, "priority", cloneString(buffer)); struct dyString *dy = trackDbString(cart, tdb); dyStringPrintf(out, "%s\n", dy->string); dyStringFree(&dy); } static boolean checkCartVisibility(struct cart *cart, struct trackDb *tdb) { char *cartVis = cartOptionalString(cart, tdb->track); if (cartVis != NULL) tdb->visibility = hTvFromString(cartVis); return (tdb->visibility != tvHide); } static boolean isFromQuickLiftHub(struct trackDb *tdb) /* True if this tdb came from a quickLift hub (already a lifted shadow track). * Such tracks must not be lifted again. */ { return trackDbSetting(tdb, "quickLiftUrl") != NULL || trackDbSetting(tdb, "quickLifted") != NULL; } static void walkTree(struct dyString *out, char *db, struct cart *cart, struct trackDb *tdb, struct trackDb **badList) /* walk tree looking for visible tracks to output to hub. Skip tracks that already * came from a quickLift hub. Every visible track is written, whether or not it is * already in the hub file: the caller merges these stanzas over the old ones, so a * track that was lifted before gets its current state rather than the one it had * when it was first lifted. */ { struct hash *haveSuper = newHash(0); struct trackDb *tdbNext = NULL; // The priority written to the hub is the track's rank in the source list, which the // caller has sorted on group priority and then track priority. The rank has to count // every track we walk past, not just the ones we output: tracks accumulate in the hub // file across requests, so the number a track gets must depend only on how the source // is laid out, never on which request it happened to be added in. // // The source priority itself will not do. All lifted tracks land in one group on the // target (trackHubAddGroupName rewrites the group of every hub track), so a priority // that only orders within a source group is being compared across groups. double rank = 0; for(; tdb; tdb = tdbNext) { tdbNext = tdb->next; rank += 1; if (isFromQuickLiftHub(tdb)) continue; boolean isVisible = FALSE; if (tdb->parent == NULL) isVisible = checkCartVisibility(cart, tdb); else if (isParentVisible(cart, tdb) && isSubtrackVisible(cart, tdb)) // child of supertrack { if (hashLookup(haveSuper, tdb->parent->track) == NULL) // output yet? { //if (checkCartVisibility(cart, tdb->parent)) { tdb->parent->visibility = hTvFromString("tvShow"); // a superTrack is not in the list we are walking, so it has no rank // of its own. Slot it just above the first child that brought it in. outTrack(out, cart, tdb->parent, rank - 0.5); hashStore(haveSuper, tdb->parent->track); } } isVisible = checkCartVisibility(cart, tdb); } if (isVisible && validateTdb(cart, db, tdb, badList)) { hashRemove(tdb->settingsHash, "superTrack"); // this gets inherited by subTracks(?) // is this a custom track? char *tdbType = trackDbSetting(tdb, "tdbType"); if (tdbType != NULL) { hashReplace(tdb->settingsHash, "type", tdbType); hashReplace(tdb->settingsHash, "shortLabel", trackDbSetting(tdb, "name")); hashReplace(tdb->settingsHash, "longLabel", trackDbSetting(tdb, "description")); } outTrack(out, cart, tdb, rank); } } } static void writeMergedHubFile(char *filename, char *db, struct dyString *newContent) /* Write the hub file from the stanzas we just generated plus whatever was already * in the file. A generated stanza replaces the old stanza of the same track, in * the slot the old one held, so the file keeps parents ahead of their children. * A track in the file that we did not generate this time is kept as it was, so * tracks still accumulate across lifts in one session. */ { struct dyString *header = dyStringNew(0); struct quickLiftStanza *oldList = NULL; struct lineFile *lf = lineFileMayOpen(filename, TRUE); if (lf != NULL) { oldList = readStanzas(lf, header); lineFileClose(&lf); } /* The generated stanzas have no header of their own; scratch collects nothing. */ struct dyString *scratch = dyStringNew(0); lf = lineFileOnString("quickLift stanzas", TRUE, cloneString(newContent->string)); struct quickLiftStanza *newList = readStanzas(lf, scratch); lineFileClose(&lf); struct hash *newByName = newHash(8); struct quickLiftStanza *s; for (s = newList; s != NULL; s = s->next) { if ((s->name != NULL) && (hashLookup(newByName, s->name) == NULL)) hashAdd(newByName, s->name, s); } struct dyString *out = dyStringNew(0); if (isEmpty(header->string)) outHubHeader(out, trackHubSkipHubName(db)); else dyStringAppend(out, header->string); struct hash *written = newHash(8); for (s = oldList; s != NULL; s = s->next) { struct quickLiftStanza *fresh = (s->name == NULL) ? NULL : hashFindVal(newByName, s->name); if (fresh == NULL) dyStringAppend(out, s->text->string); else if (hashLookup(written, fresh->name) == NULL) { dyStringAppend(out, fresh->text->string); hashStore(written, fresh->name); } } for (s = newList; s != NULL; s = s->next) { if ((s->name != NULL) && (hashLookup(written, s->name) != NULL)) continue; dyStringAppend(out, s->text->string); if (s->name != NULL) hashStore(written, s->name); } /* Write a temporary file and rename it into place. The target assembly reads this * same file, and rewriting it in place would show a reader a truncated hub. */ char tmpName[PATH_LEN]; safef(tmpName, sizeof tmpName, "%s.tmp", filename); FILE *f = mustOpen(tmpName, "w"); fputs(out->string, f); carefulClose(&f); chmod(tmpName, 0666); mustRename(tmpName, filename); dyStringFree(&out); dyStringFree(&header); dyStringFree(&scratch); hashFree(&newByName); hashFree(&written); freeStanzas(&oldList); freeStanzas(&newList); } static int cmpPriority(const void *va, const void *vb) /* Compare to sort based on priority; use shortLabel as secondary sort key. */ { const struct trackDb *a = *((struct trackDb **)va); const struct trackDb *b = *((struct trackDb **)vb); float dif = 0; dif = a->groupPriority - b->groupPriority; if (dif == 0) dif = a->priority - b->priority; if (dif < 0) return -1; else if (dif == 0.0) /* secondary sort on label */ return strcasecmp(a->shortLabel, b->shortLabel); else return 1; } char *trackHubBuild(char *db, struct cart *cart, struct trackDb **badList) /* Build a track hub using trackDb and the cart. If a hub file already exists * for db (i.e. earlier quickLift work in the same session), merge the new track * stanzas into it: a track that is being lifted again gets the state it has now, * and a track that is in the file but is not visible on the source any more is * left alone. */ { struct trackDb *tdbList, *tdb; struct grp *grpList; cartTrackDbInit(cart, &tdbList, &grpList, FALSE); struct hash *groupHash = newHash(0); struct grp *grp; for(grp = grpList; grp; grp = grp->next) hashAdd(groupHash, grp->name, grp); for(tdb = tdbList; tdb; tdb = tdb->next) { grp = hashFindVal(groupHash, tdb->grp); tdb->groupPriority = grp->priority; } slSort(&tdbList, cmpPriority); char *filename = getHubName(cart, db); struct dyString *newContent = dyStringNew(0); walkTree(newContent, db, cart, tdbList, badList); writeMergedHubFile(filename, db, newContent); dyStringFree(&newContent); return cloneString(filename); } struct grp *trackHubGetGrps() /* Get the groups defined by attached track hubs. */ { return trackHubGrps; } struct trackDb *trackHubAddTracksGenome(struct trackHubGenome *hubGenome) /* Load up stuff from data hub and return list. */ { /* Load trackDb.ra file and make it into proper trackDb tree */ struct trackDb *tdbList = NULL; if (hubGenome != NULL) { boolean doCache = trackDbCacheOn(); if (doCache) { // we have to open the trackDb file to get the udc cache to check for an update struct udcFile *checkCache = udcFileMayOpen(hubGenome->trackDbFile, NULL); if (checkCache != NULL) { time_t time = udcUpdateTime(checkCache); udcFileClose(&checkCache); struct trackDb *cacheTdb = trackDbHubCache(hubGenome->trackDbFile, time); if (cacheTdb != NULL) return cacheTdb; } memCheckPoint(); // we want to know how much memory is used to build the tdbList } struct dyString *incFiles = newDyString(4096); boolean foundFirstGenome = FALSE; tdbList = trackHubTracksForGenome(hubGenome->trackHub, hubGenome, incFiles, &foundFirstGenome); tdbList = trackDbLinkUpGenerations(tdbList); tdbList = trackDbPolishAfterLinkup(tdbList, hubGenome->name); trackDbPrioritizeContainerItems(tdbList); trackHubPolishTrackNames(hubGenome->trackHub, tdbList); if (doCache) trackDbHubCloneTdbListToSharedMem(hubGenome->trackDbFile, tdbList, memCheckPoint(), incFiles->string); } return tdbList; }