d0ed51d077718213c0b843a89fd6f36ac32f1804 braney Fri Sep 4 11:39:28 2026 -0700 quickLift: lift alignments, and let psl and bigPsl tracks into the hub Every type quickLift handled so far is one set of coordinates on one genome, so the lift is a call to remapBlockedBed. An alignment carries coordinates on both sides at once, with a block start on each, so moving the genome side means splitting and trimming blocks while keeping the other side lined up with them. quickLiftPsl does that with pieces that were already in the tree: pick the chain with liftOverChainForRange, the same one the bed path picks, turn it into a mapping alignment with chainToPsl plus pslSwap, then pslTransMap. The mapping alignment is kept per chain, since rebuilding it for every item costs nothing at gene zoom and a great deal zoomed out. Two things pslTransMap does had to be undone. It recounts match and mismatch off the blocks, which would read every lifted alignment as a perfect match and draw every item at full shade, so quickLiftPslCounts puts the original counts back, scaled by how much of the alignment survived. And it leaves a protein alignment in nucleotide space, which the base alignment view rejects, so quickLiftPslBackToProtein returns the query side to protein units when the lift did not split a codon. quickLiftChainHash hands out the chains for a reference range for callers that collected their items some other way. validateOneTdb now accepts psl and bigPsl, and bigPsl joins bigBed and bigWig in the bigDataUrl fill-in. refs #38249 diff --git src/hg/lib/trackHub.c src/hg/lib/trackHub.c index ca08288f003..264bbb41fa4 100644 --- src/hg/lib/trackHub.c +++ src/hg/lib/trackHub.c @@ -1,2312 +1,2316 @@ /* Copyright (C) 2014 The Regents of the University of California * See kent/LICENSE or http://genome.ucsc.edu/license/ for licensing information. */ /* trackHub - supports collections of tracks hosted on a remote site. * The basic layout of a data hub is: * hub.txt - contains information about the hub itself * genomes.txt - says which genomes are supported by hub * Contains file name of trackDb.txt for each genome * trackDb.txt - contains a stanza for each track. Stanzas * are in a subset of the usual trackDb format. * How you use the routines here most commonly is as so: * struct trackHub *hub = trackHubOpen(hubRaUrl); * struct trackHubGenome *hubGenome = trackHubFindGenome(hub, "hg19"); * struct trackDb *tdbList = trackHubTracksForGenome(hub, hubGenome); * // do something with tdbList * trackHubClose(&hub); * Note that the tdbList returned does not have the parent/subtrack pointers set. * It is just a simple list of tracks, not a tree. */ #include "common.h" #include "linefile.h" #include "hash.h" #include "options.h" #include "udc.h" #include "ra.h" #include "filePath.h" #include "htmlPage.h" #include "trackDb.h" #include "htmlSanitize.h" #include "trackHub.h" #include "errCatch.h" #include "hgBam.h" #include "bigWig.h" #include "bigBed.h" #include "barChartUi.h" #include "hdb.h" #include "chromInfo.h" #include "grp.h" #include "twoBit.h" #include "dbDb.h" #include "net.h" #include "bbiFile.h" #include "bPlusTree.h" #include "hgFind.h" #include "hubConnect.h" #include "trix.h" #include "vcf.h" #include "vcfUi.h" #include "htmshell.h" #include "bigBedFind.h" #include "customComposite.h" #include "interactUi.h" #include "bedTabix.h" #include "hic.h" #include "hui.h" #include "chromAlias.h" #include "trashDir.h" #include "hgConfig.h" #include "cartTrackDb.h" #include "quickLift.h" #include "portable.h" #ifdef USE_HAL #include "halBlockViz.h" #endif struct grp *trackHubGrps = NULL; // global with grps loaded from track hubs static struct hash *hubCladeHash; // mapping of clade name to hub pointer static struct hash *hubAssemblyHash; // mapping of assembly name to genome struct static struct hash *hubAssemblyUndecoratedHash; // mapping of undecorated assembly name to genome struct static struct hash *hubOrgHash; // mapping from organism name to hub pointer static struct trackHub *globalAssemblyHubList; // list of trackHubs in the user's cart static struct hash *trackHubHash; static boolean isValidSeqNameChar(char c) /* Return TRUE if c is a valid character for a sequence name: [A-Za-z0-9._-]. */ { return isalnum((unsigned char)c) || c == '.' || c == '_' || c == '-'; } boolean trackHubIsValidSeqName(char *name) /* Return TRUE if name is a valid sequence name: non-empty, starts with a * letter or digit, and contains only [A-Za-z0-9._-]. */ { if (!name || !name[0]) return FALSE; if (!isalnum((unsigned char)name[0])) return FALSE; char *p; for (p = name; *p; p++) if (!isValidSeqNameChar(*p)) return FALSE; return TRUE; } static void tdbListAddHubToGroup(char *hubName, struct trackDb *tdbList) /* Prepend hub name to group name for every tdb. */ { struct trackDb *tdb; for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { char buffer[4096]; char *grp = trackDbSetting(tdb, "group"); if (grp == NULL) tdb->grp = cloneString(hubName); else { safef(buffer, sizeof buffer, "%s_%s", hubName, grp); tdb->grp = cloneString(buffer); } hashReplace(tdb->settingsHash, "group", tdb->grp); } } char *trackHubRelativeUrl(char *hubUrl, char *path) /* Return full path (in URL form if it's a remote hub) given * path possibly relative to hubUrl. Do a freeMem of result * when done. */ { /* If path itself is a URL then just return a copy of it. */ if (hasProtocol(path)) return cloneString(path); /* If it's a remote hub, let html path expander handle it. */ if (hasProtocol(hubUrl) && !startsWith("/gbdb",path)) return expandUrlOnBase(hubUrl, path); /* If we got to here hub is local, and so is path. Do standard * path parsing. */ return pathRelativeToFile(hubUrl, path); } static void badGenomeStanza(struct lineFile *lf) /* Put up semi-informative error message about a genome stanza being bad. */ { errAbort("Genome stanza should have at least two lines, one with 'genome' and one with 'trackDb'\n" "Bad stanza format ending line %d of %s", lf->lineIx, lf->fileName); } char *trackHubCladeToGenome(char *clade) /* Given a track hub clade(hub name) return the default genome. */ { if (hubCladeHash == NULL) return FALSE; struct hashEl *hel = hashLookup(hubCladeHash, clade); if (hel == NULL) return FALSE; struct trackHub *trackHub = hel->val; struct trackHubGenome *hubGenome = trackHub->genomeList; for(; hubGenome; hubGenome=hubGenome->next) if (hubGenome->twoBitPath != NULL) return hubGenome->organism ; return NULL; } struct trackHubGenome *trackHubGetGenomeUndecorated(char *database) /* Get the genome structure for an undecorated genome name. */ { if (hubAssemblyUndecoratedHash == NULL) return NULL; struct hashEl *hel = hashLookup(hubAssemblyUndecoratedHash, database); if (hel == NULL) return NULL; return (struct trackHubGenome *)hel->val; } struct trackHubGenome *trackHubGetGenome(char *database) /* get genome structure for an assembly in a trackHub */ { if (hubAssemblyHash == NULL) errAbort("An error occurred while adding the custom track. This may be due to a mismatch" " between the db= parameter and the genome selected at the top of the page." " If you are having trouble resolving this error please contact us at" " genome-www@soe.ucsc.edu."); struct hashEl *hel = hashLookup(hubAssemblyHash, database); if (hel == NULL) return NULL; return (struct trackHubGenome *)hel->val; } boolean trackHubDatabase(char *database) /* Is this an assembly from an Assembly Data hub? */ { if (hubAssemblyHash == NULL) return FALSE; return trackHubGetGenome(database) != NULL; } char *trackHubDatabaseToGenome(char *db) /* get a database name that is either a genome database or a trackHub * database, return the genome assembly */ { return trackHubDatabase(db) ? trackHubAssemblyField(db, "genome") : db; } char *trackHubAssemblyField(char *database, char *field) /* Get data field from a assembly data hub. */ { struct trackHubGenome *genome = trackHubGetGenome(database); if (genome == NULL) return NULL; char *ret = hashFindVal(genome->settingsHash, field); return cloneString(ret); } static struct dbDb *makeDbDbFromAssemblyGenome(struct trackHubGenome *hubGenome) /* Make a dbdb struture from a single assembly hub database. */ { struct dbDb *db; AllocVar(db); db->genome = cloneString(hubGenome->organism); db->organism = cloneString(hubGenome->organism); db->name = cloneString(hubGenome->name); db->active = TRUE; if (hubGenome->description != NULL) db->description = cloneString(hubGenome->description); else db->description = cloneString(""); char *orderKey = hashFindVal(hubGenome->settingsHash, "orderKey"); if (orderKey != NULL) db->orderKey = sqlUnsigned(orderKey); db->defaultPos = cloneString(hubGenome->defaultPos); return db; } struct dbDb *trackHubDbDbFromAssemblyDb(char *database) /* Return a dbDb structure for just this database. */ { struct trackHubGenome *genome = trackHubGetGenome(database); if (genome == NULL) return NULL; return makeDbDbFromAssemblyGenome(genome); } struct slPair *trackHubGetCladeLabels() /* Get a list of labels describing the loaded assembly data hubs. */ { if (globalAssemblyHubList == NULL) return NULL; struct slPair *clade, *cladeList = NULL; struct trackHub *trackHub = globalAssemblyHubList; for(;trackHub; trackHub = trackHub->next) { AllocVar(clade); slAddHead(&cladeList, clade); clade->name = cloneString(trackHub->name); clade->val = cloneString(trackHub->shortLabel); } return cladeList; } struct dbDb *trackHubGetPcrServers() /* Look through attached trackHubs to see which of them have "isPcr" line in them. */ { struct dbDb *db, *dbList = NULL; if (globalAssemblyHubList != NULL) { struct trackHub *trackHub = globalAssemblyHubList; for(;trackHub; trackHub = trackHub->next) { struct trackHubGenome *hubGenome = trackHub->genomeList; for(; hubGenome; hubGenome = hubGenome->next) { if (hashFindVal(hubGenome->settingsHash,"isPcr") != NULL) { db = makeDbDbFromAssemblyGenome(hubGenome); slAddHead(&dbList, db); } } } } slReverse(&dbList); slSort(&dbList, hDbDbCmpOrderKey); return dbList; } static struct dbDb *getDbDbs(char *clade, boolean blatEnabled) /* Get a list of struct dbDbs from track hubs. Only get blat enabled ones if asked */ { struct dbDb *db, *dbList = NULL; if (globalAssemblyHubList != NULL) { struct trackHub *trackHub = globalAssemblyHubList; for(;trackHub; trackHub = trackHub->next) { if ((clade != NULL) && differentString(clade, trackHub->name)) continue; struct trackHubGenome *hubGenome = trackHub->genomeList; for(; hubGenome; hubGenome = hubGenome->next) { boolean blatCheck = !blatEnabled || ((hashFindVal(hubGenome->settingsHash,"transBlat") != NULL) || (hashFindVal(hubGenome->settingsHash,"blat") != NULL)); if ( blatCheck && (hubGenome->twoBitPath != NULL)) { db = makeDbDbFromAssemblyGenome(hubGenome); slAddHead(&dbList, db); } } } } slReverse(&dbList); slSort(&dbList, hDbDbCmpOrderKey); return dbList; } struct dbDb *trackHubGetBlatDbDbs() /* Get a list of connected track hubs that have blat servers */ { return getDbDbs(NULL, TRUE); } struct dbDb *trackHubGetDbDbs(char *clade) /* Get a list of dbDb structures for all the tracks in this clade/hub. */ { return getDbDbs(clade, FALSE); } struct slPair *trackHubDbDbToValueLabel(struct dbDb *hubDbDbList) /* Given a trackHub (list of) track hub dbDb which may be missing some info, * return an slPair of value and label suitable for making a select/menu option. */ { struct dbDb *dbDb; struct slPair *pairList = NULL; for (dbDb = hubDbDbList; dbDb != NULL; dbDb = dbDb->next) { char *db = dbDb->name; if (isEmpty(db)) db = dbDb->genome; char *label = dbDb->description; if (isEmpty(label)) label = trackHubSkipHubName(db); slAddHead(&pairList, slPairNew(db, cloneString(label))); } slReverse(&pairList); return pairList; } struct slName *trackHubAllChromNames(char *database) /* Return a list of all the chrom names in this assembly hub database. */ /* Free with slFreeList. */ { struct trackHubGenome *genome = trackHubGetGenome(database); if (genome == NULL) return NULL; struct slName *chromList = twoBitSeqNames(genome->twoBitPath); return chromList; } int trackHubChromCount(char *database) /* Return number of chromosomes in a assembly data hub. */ { struct slName *chromList = trackHubAllChromNames(database); int num = slCount(chromList); slFreeList(&chromList); return num; } char *trackHubDefaultChrom(char *database) /* Return the default chromosome for this track hub assembly. */ { struct slName *chromList = trackHubAllChromNames(database); if (chromList == NULL) return NULL; char *defaultName = cloneString( chromList->name); slFreeList(&chromList); return defaultName; } static struct twoBitFile *openTwoBit(struct trackHubGenome *genome) /* Open a twoBit file that may or may not have a bpt index. */ { if (genome->twoBitBptUrl) return twoBitOpenExternalBptIndex(genome->twoBitPath, genome->twoBitBptUrl); else return twoBitOpen(genome->twoBitPath); } struct chromInfo *trackHubMaybeChromInfo(char *database, char *chrom) /* Return a chromInfo structure for just this chrom in this database. The database * may be decorated with a hub_<id>_ prefix or undecorated. * Return NULL if chrom doesn't exist. */ { struct trackHubGenome *genome = NULL; if (hubAssemblyHash != NULL) genome = trackHubGetGenome(database); if (genome == NULL) genome = trackHubGetGenomeUndecorated(trackHubSkipHubName(database)); if (genome == NULL) return NULL; if (genome->tbf == NULL) genome->tbf = openTwoBit(genome); if (!twoBitIsSequence(genome->tbf, chrom)) return NULL; struct chromInfo *ci; AllocVar(ci); ci->chrom = cloneString(chrom); ci->fileName = genome->twoBitPath; ci->size = twoBitSeqSize(genome->tbf, chrom); return ci; } struct chromInfo *trackHubChromInfo(char *database, char *chrom) /* Return a chromInfo structure for just this chrom in this database. * errAbort if chrom doesn't exist. */ { struct chromInfo *ci = trackHubMaybeChromInfo(database, chrom); if (ci == NULL) errAbort("%s is not in %s", chrom, database); return ci; } static char *assemblyHubGenomeSetting(char *database, char *tagName) /* see if this assembly hub has specified tagName, return url if present * returns NULL when not present */ { struct trackHubGenome *genome = trackHubGetGenome(database); if (genome == NULL) return NULL; char *fileName = hashFindVal(genome->settingsHash, tagName); char *absFileName = NULL; if (fileName) absFileName = trackHubRelativeUrl((genome->trackHub)->url, fileName); if (absFileName) { hashReplace(genome->settingsHash, tagName, absFileName); fileName = absFileName; } return fileName; } char *trackHubChromSizes(char *database) /* see if this assembly hub has a chrom.sizes file, return url if present * returns NULL when not present */ { return assemblyHubGenomeSetting(database, "chromSizes"); } char *trackHubAliasBbFile(char *database) /* see if this assembly hub has an alias bigBed file, return url if present * returns NULL when not present */ { return assemblyHubGenomeSetting(database, "chromAliasBb"); } char *trackHubAliasFile(char *database) /* see if this assembly hub has an alias file, return url if present * returns NULL when not present */ { return assemblyHubGenomeSetting(database, "chromAlias"); } struct chromInfo *trackHubAllChromInfo(char *database) /* Return a chromInfo structure for all the chroms in this database. */ { struct trackHubGenome *genome = trackHubGetGenome(database); if (genome == NULL) return NULL; if (genome->tbf == NULL) genome->tbf = openTwoBit(genome); struct chromInfo *ci, *ciList = NULL; struct slName *chromList = twoBitSeqNames(genome->twoBitPath); for(; chromList; chromList = chromList->next) { AllocVar(ci); ci->chrom = cloneString(chromList->name); ci->fileName = cloneString(genome->twoBitPath); ci->size = twoBitSeqSize(genome->tbf, chromList->name); slAddHead(&ciList, ci); } slFreeList(&chromList); return ciList; } static char *getRequiredGrpSetting(struct hash *hash, char *name, struct lineFile *lf) /* Grab a group setting out of the group hash. errAbort if not found. */ { char *str; if ((str = hashFindVal(hash, name)) == NULL) errAbort("missing required setting '%s' for group on line %d in file %s\n", name, lf->lineIx, lf->fileName); return str; } static void checkHubIdName(char *type, char *name) /* Abort if name holds a character that is not valid in an identifier. A hub * identifier - a genome name, a group name - is printed into dozens of URLs, * form values and attributes all over the CGIs, so the check belongs here rather than at every * one of those places. Real names use letters, digits, underscore, dot and dash. */ { if (strchr(name, '<') || strchr(name, '>') || strchr(name, '"') || strchr(name, '\'') || strchr(name, '&')) errAbort("Bad %s name: \"%s\". The characters < > \" ' and & are not allowed in a %s name.", type, name, type); } struct grp *readGroupRa(char *groupFileName) /* Read in the ra file that describes the groups in an assembly hub. */ { if (groupFileName == NULL) return NULL; struct hash *ra; struct grp *list = NULL; struct lineFile *lf = udcWrapShortLineFile(groupFileName, NULL, MAX_HUB_GROUP_FILE_SIZE); while ((ra = raNextRecord(lf)) != NULL) { char *str; struct grp *grp; AllocVar(grp); slAddHead(&list, grp); grp->name = cloneString(getRequiredGrpSetting(ra, "name", lf)); checkHubIdName("group", grp->name); grp->label = cloneString(getRequiredGrpSetting(ra, "label", lf)); grp->priority = BIGDOUBLE; str = hashFindVal(ra, "priority"); if (str != NULL) grp->priority = atof(str); str = hashFindVal(ra, "defaultIsClosed"); if ((str != NULL) && (sameString("on",str) || sameString("1", str))) grp->defaultIsClosed = 1; hashFree(&ra); } if (list) slSort(&list, grpCmpPriorityLabel); lineFileClose(&lf); return list; } struct grp *trackHubLoadGroups(char *database) /* Load the grp structures for this track hub database. */ { struct trackHubGenome *genome = trackHubGetGenome(database); if (genome == NULL) return NULL; struct grp *list = readGroupRa(genome->groups); return list; } char *trackHubGenomeNameToDb(char *genome) /* Return assembly name given a genome name if one exists, otherwise NULL. */ { struct hashEl *hel; if ((hubOrgHash != NULL) && (hel = hashLookup(hubOrgHash, genome)) != NULL) { struct trackHub *hub = hel->val; struct trackHubGenome *genomeList = hub->genomeList; for(; genomeList; genomeList=genomeList->next) if ((genomeList->organism != NULL ) && sameString(genomeList->organism, genome)) return genomeList->name; } return NULL; } char *trackHubAssemblyClade(char *genome) /* Return the clade/hub_name that contains this genome. */ { struct hashEl *hel; if ((hubOrgHash != NULL) && (hel = hashLookup(hubOrgHash, genome)) != NULL) { struct trackHub *hub = hel->val; return cloneString(hub->name); } return NULL; } static void deleteAssembly(char *name, struct trackHubGenome *genome, struct trackHub *hub) /* delete this assembly from the assembly caches */ { hashRemove(hubCladeHash, hub->name); slRemoveEl(&globalAssemblyHubList, hub); hashRemove(hubOrgHash, genome->organism); hashRemove(hubAssemblyHash, genome->name); } static void addAssembly(char *name, struct trackHubGenome *genome, struct trackHub *hub) /* Add a new assembly hub database to our global list. */ { struct hashEl *hel; if (hubCladeHash == NULL) hubCladeHash = newHash(5); if ((hel = hashLookup(hubCladeHash, hub->name)) == NULL) { hashAdd(hubCladeHash, hub->name, hub); slAddHead(&globalAssemblyHubList, hub); } if (hubOrgHash == NULL) hubOrgHash = newHash(5); if ((hel = hashLookup(hubOrgHash, genome->organism)) == NULL) { hashAdd(hubOrgHash, genome->organism, hub); } if (hubAssemblyHash == NULL) hubAssemblyHash = newHash(5); if ((hel = hashLookup(hubAssemblyHash, genome->name)) == NULL) hashAdd(hubAssemblyHash, genome->name, genome); if (hubAssemblyUndecoratedHash == NULL) hubAssemblyUndecoratedHash = newHash(5); if ((hel = hashLookup(hubAssemblyUndecoratedHash, trackHubSkipHubName(genome->name))) == NULL) hashAdd(hubAssemblyUndecoratedHash, trackHubSkipHubName(genome->name), genome); } static char *addHubName(char *base, char *hubName) { if (base == NULL) return NULL; char buffer[4096]; if (isNotEmpty(hubName)) safef(buffer, sizeof(buffer), "%s_%s", hubName, base); else safef(buffer, sizeof(buffer), "%s", base); return cloneString(buffer); } static int genomeOrderKeyCmp(const void *va, const void *vb) /* Compare to sort based on order key */ { const struct trackHubGenome *a = *((struct trackHubGenome **)va); const struct trackHubGenome *b = *((struct trackHubGenome **)vb); if (b->orderKey > a->orderKey) return -1; else if (b->orderKey < a->orderKey) return 1; else return 0; } static struct trackHubGenome *trackHubGenomeReadRa(char *url, struct trackHub *hub, char *singleFile) /* Read in a genome.ra format url and return it as a list of trackHubGenomes. * Also add it to hash, which is keyed by genome. */ { struct lineFile *lf = udcWrapShortLineFile(url, NULL, MAX_HUB_GENOME_FILE_SIZE); struct trackHubGenome *list = NULL, *el; struct hash *hash = hub->genomeHash; struct hash *ra; while ((ra = raNextRecord(lf)) != NULL) { // allow that trackDb+hub+genome is in one single file if (hashFindVal(ra, "hub")) continue; if (hashFindVal(ra, "track")) break; char *twoBitPath = hReplaceGbdb(hashFindVal(ra, "twoBitPath")); char *twoBitBptUrl = hReplaceGbdb(hashFindVal(ra, "twoBitBptUrl")); char *genome, *trackDb; if (twoBitPath != NULL) genome = addHubName(hashFindVal(ra, "genome"), hub->name); else genome = hashFindVal(ra, "genome"); if (hub->defaultDb == NULL) hub->defaultDb = genome; if (genome == NULL) badGenomeStanza(lf); if (hasWhiteSpace(genome)) errAbort("Bad genome name: \"%s\". Only alpha-numeric characters and \"_\" are allowed ([A-Za-z0-9_]).", genome); checkHubIdName("genome", genome); if (hashLookup(hash, genome) != NULL) errAbort("Duplicate genome %s in stanza ending line %d of %s", genome, lf->lineIx, lf->fileName); if (singleFile == NULL) { trackDb = hashFindVal(ra, "trackDb"); if (trackDb == NULL) badGenomeStanza(lf); } else trackDb = singleFile; AllocVar(el); el->name = cloneString(genome); el->trackDbFile = trackHubRelativeUrl(url, trackDb); el->trackHub = hub; hashAdd(hash, el->name, el); hashAdd(hash, hubConnectSkipHubPrefix(el->name), el); slAddHead(&list, el); char *orderKey = hashFindVal(ra, "orderKey"); if (orderKey != NULL) el->orderKey = sqlUnsigned(orderKey); char *groups = hashFindVal(ra, "groups"); if (twoBitPath != NULL) { el->description = hashFindVal(ra, "description"); char *organism = hashFindVal(ra, "organism"); if (organism == NULL) errAbort("must have 'organism' set in assembly hub in stanza ending line %d of %s", lf->lineIx, lf->fileName); el->organism = addHubName(organism, hub->name); hashReplace(ra, "organism", el->organism); el->defaultPos = hashFindVal(ra, "defaultPos"); if (el->defaultPos == NULL) errAbort("must have 'defaultPos' set in assembly hub in stanza ending line %d of %s", lf->lineIx, lf->fileName); el->twoBitPath = trackHubRelativeUrl(url, twoBitPath); if (twoBitBptUrl != NULL) el->twoBitBptUrl = trackHubRelativeUrl(url, twoBitBptUrl); char *htmlPath = hashFindVal(ra, "htmlPath"); if (htmlPath != NULL) hashReplace(ra, "htmlPath",trackHubRelativeUrl(url, htmlPath)); if (groups != NULL) el->groups = trackHubRelativeUrl(url, groups); addAssembly(genome, el, hub); } else { if (groups != NULL) { el->groups = trackHubRelativeUrl(url, groups); } } el->settingsHash = ra; hashAdd(ra, "hubName", hub->shortLabel); el->chromAuthority = hashFindVal(ra, "chromAuthority"); } /* Clean up and go home. */ lineFileClose(&lf); slReverse(&list); slSort(&list, genomeOrderKeyCmp); return list; } char *trackHubSetting(struct trackHub *hub, char *name) /* Return setting if it exists, otherwise NULL. */ { return hashFindVal(hub->settings, name); } char *trackHubRequiredSetting(struct trackHub *hub, char *name) /* Return named setting. Abort with error message if not found. */ { char *val = trackHubSetting(hub, name); if (val == NULL) errAbort("Missing required setting '%s' from %s", name, hub->url); return val; } struct trackHub *grabHashedHub(char *hubName) /* see if a trackHub with this name is in the cache */ { if ( trackHubHash == NULL) trackHubHash = newHash(5); return (struct trackHub *)hashFindVal(trackHubHash, hubName); } static void cacheHub(struct trackHub *hub) { /* put this trackHub in the trackHub hash */ if ( trackHubHash == NULL) trackHubHash = newHash(5); hashAdd(trackHubHash, hub->name, hub); } void uncacheHub(struct trackHub *hub) /* take this trackHub out of the trackHub hash */ { if ( trackHubHash == NULL) return; hashMustRemove(trackHubHash, hub->name); } struct trackHub *trackHubOpen(char *url, char *hubName) /* Open up a track hub from url. Reads and parses hub.txt and the genomesFile. * The hubName is generally just the asciified ID number. */ { struct trackHub *hub = grabHashedHub(hubName); if (hub != NULL) return hub; struct lineFile *lf = udcWrapShortLineFile(url, NULL, MAX_HUB_TRACKDB_FILE_SIZE); struct hash *hubRa = raNextRecord(lf); if (hubRa == NULL) errAbort("empty %s in trackHubOpen", url); // no errAbort when more records in hub.txt file: user can stuff // trackDb into it /* Allocate hub and fill in settings field and url. */ AllocVar(hub); hub->url = cloneString(url); hub->name = cloneString(hubName); hub->settings = hubRa; /* Fill in required fields from settings. */ trackHubRequiredSetting(hub, "hub"); trackHubRequiredSetting(hub, "email"); hub->shortLabel = trackHubRequiredSetting(hub, "shortLabel"); hub->longLabel = trackHubRequiredSetting(hub, "longLabel"); boolean isOneFile = (trackHubSetting(hub, "useOneFile") != NULL); char *ourFile = NULL; if (isOneFile) { ourFile = url; char *root = strrchr(url, '/'); if (root) ourFile = root + 1; hub->genomesFile = cloneString(ourFile); } else hub->genomesFile = trackHubRequiredSetting(hub, "genomesFile"); hub->email = trackHubSetting(hub, "email"); hub->version = trackHubSetting(hub, "version"); // default to current version hub->level = trackHubSetting(hub, "level"); // "core" or "all" char *descriptionUrl = trackHubSetting(hub, "descriptionUrl"); if (descriptionUrl != NULL) hub->descriptionUrl = trackHubRelativeUrl(hub->url, descriptionUrl); lineFileClose(&lf); char *genomesUrl = trackHubRelativeUrl(hub->url, hub->genomesFile); hub->genomeHash = hashNew(8); hub->genomeList = trackHubGenomeReadRa(genomesUrl, hub, ourFile); freez(&genomesUrl); cacheHub(hub); return hub; } void trackHubClose(struct trackHub **pHub) /* Close up and free resources from hub. */ { struct trackHub *hub = *pHub; if (hub != NULL) { trackHubGenomeFreeList(hub); freeMem(hub->url); hashFree(&hub->settings); hashFree(&hub->genomeHash); uncacheHub(hub); freez(pHub); } } void trackHubGenomeFree(struct trackHubGenome **pGenome) /* Free up genome info. */ { struct trackHubGenome *genome = *pGenome; if (genome != NULL) { freeMem(genome->name); freeMem(genome->trackDbFile); freez(pGenome); } } void trackHubGenomeFreeList(struct trackHub *hub) /* Free a list of dynamically allocated trackHubGenome's */ { struct trackHubGenome *el, *next; for (el = hub->genomeList; el != NULL; el = next) { next = el->next; if (el->twoBitPath != NULL) deleteAssembly(el->name, el, hub); trackHubGenomeFree(&el); } hub->genomeList = NULL; } static char *requiredSetting(struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdb, char *setting) /* Fetch setting or give an error message, a little more specific than the * error message from trackDbRequiredSetting(). */ { char *val = trackDbSetting(tdb, setting); if (val == NULL) errAbort("Missing required '%s' setting in hub %s genome %s track %s. Add a '%s' line to the track stanza.", setting, hub->url, genome->name, tdb->track, setting); return val; } static void forbidSetting(struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdb, char *setting) /* Abort if forbidden setting found. */ { if (trackDbSetting(tdb, setting)) errAbort("Forbidden setting '%s' in hub %s genome %s track %s", setting, hub->url, genome->name, tdb->track); } static void expandOneUrl(struct hash *settingsHash, char *hubUrl, char *variable) { struct hashEl *hel = hashLookup(settingsHash, variable); if (hel != NULL) { char *oldVal = hel->val; hel->val = trackHubRelativeUrl(hubUrl, oldVal); char *newPath = (char *) hel->val; if (hasProtocol(hubUrl) && !hasProtocol(newPath)) { // allow local url with udc.localDir path prefix char *prefix = cfgOption("udc.localDir"); if (!(prefix && startsWith(prefix, newPath))) { errAbort("setting %s local URL %s not allowed with non-local host URL %s", variable, newPath, hubUrl); } } freeMem(oldVal); } } static void expandBigDataUrl(struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdb) /* Expand bigDataUrls so that no longer relative to genome->trackDbFile */ { struct hashEl *hel; struct hashCookie cookie = hashFirst(tdb->settingsHash); while ((hel = hashNext(&cookie)) != NULL) { char *name = hel->name; if (trackSettingIsFile(name)) expandOneUrl(tdb->settingsHash, genome->trackDbFile, name); } } struct trackHubGenome *trackHubFindGenome(struct trackHub *hub, char *genomeName) /* Return trackHubGenome of given name associated with hub. Return NULL if no * such genome. Check genomeName without hub prefix to see if this hub * is attached to an assembly hub.*/ { struct trackHubGenome *ret = hashFindVal(hub->genomeHash, genomeName); if (ret == NULL) ret = hashFindVal(hub->genomeHash, hubConnectSkipHubPrefix(genomeName)); return ret; } static void requireBarChartBars(struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdb) /* Fetch setting(s) or give an error message */ { if (!trackDbSetting(tdb, BAR_CHART_CATEGORY_URL) && !trackDbSetting(tdb, BAR_CHART_CATEGORY_LABELS)) errAbort("BarChart track '%s' is missing either %s or %s setting. Please add one of those settings to the appropriate stanza", tdb->track, BAR_CHART_CATEGORY_LABELS, BAR_CHART_CATEGORY_URL); } static void validateOneTrack( struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdb) /* Validate a track's trackDb entry. */ { /* Check for existence of fields required in all tracks */ requiredSetting(hub, genome, tdb, "shortLabel"); char *shortLabel = trackDbSetting(tdb, "shortLabel"); memSwapChar(shortLabel, strlen(shortLabel), '\t', ' '); requiredSetting(hub, genome, tdb, "longLabel"); char *longLabel = trackDbSetting(tdb, "longLabel"); memSwapChar(longLabel, strlen(longLabel), '\t', ' '); /* Forbid any dangerous settings that should not be allowed */ forbidSetting(hub, genome, tdb, "idInUrlSql"); // subtracks is not NULL if a track said we were its parent if (tdb->subtracks != NULL) { boolean isSuper = FALSE; char *superTrack = trackDbSetting(tdb, "superTrack"); if ((superTrack != NULL) && startsWith("on", superTrack)) isSuper = TRUE; if (!(trackDbSetting(tdb, "compositeTrack") || trackDbSetting(tdb, "container") || isSuper)) { errAbort("Parent track %s is not compositeTrack, container, or superTrack in hub %s genome %s", tdb->track, hub->url, genome->name); } } else { /* Check type field. */ char *type = requiredSetting(hub, genome, tdb, "type"); char *quickLifted = trackDbSetting(tdb, "quickLifted"); if (! ( isCustomComposite(tdb) || (quickLifted != NULL))) { if (startsWithWord("mathWig", type) ) { requiredSetting(hub, genome, tdb, "mathDataUrl"); } else { // **** DANGER, WILL ROBINSON! **** // There is also code in hubCheck.c that checks track names. // both places must be changed or common code created. if (!(startsWithWord("wig", type)|| startsWithWord("bedGraph", type))) { if (!(startsWithWord("bigWig", type) || startsWithWord("bigBed", type) || startsWithWord("pslSnake", type) || startsWithWord("halSnake", type) || startsWithWord("vcfTabix", type) || startsWithWord("vcfPhasedTrio", type) || startsWithWord("bigPsl", type) || startsWithWord("bigMaf", type) || startsWithWord("longTabix", type) || startsWithWord("bigGenePred", type) || startsWithWord("bigNarrowPeak", type) || startsWithWord("bigChain", type) || startsWithWord("bigLolly", type) || startsWithWord("bigBaseView", type) || startsWithWord("bigRmsk", type) || startsWithWord("bigBarChart", type) || startsWithWord("bigInteract", type) || startsWithWord("bigMethyl", type) || startsWithWord("hic", type) || startsWithWord("bigDbSnp", type) || startsWithWord("instaPort", type) || startsWithWord("bam", type))) { errAbort("Unsupported type '%s' in hub %s genome %s track %s", type, hub->url, genome->name, tdb->track); } requiredSetting(hub, genome, tdb, "bigDataUrl"); } } if (sameString("barChart", type) || sameString("bigBarChart", type)) requireBarChartBars(hub, genome, tdb); if (sameString("vcfPhasedTrio", type)) requiredSetting(hub, genome, tdb, VCF_PHASED_CHILD_SAMPLE_SETTING); } } } static void markContainers( struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdbList) /* Mark containers that are parents, or have them. */ { struct hash *hash = hashNew(0); struct trackDb *tdb; // add all the track names to a hash for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { if (hashLookup(hash, tdb->track)) errAbort("Track %s appears more than once in genome %s. " "Track identifiers have to be unique. Please check your track hub files, " "especially the 'track' lines. " "The most likely reason for this error is that you duplicated a " "'track' identifier. Hub URL: %s", tdb->track, genome->name, hub->url); hashAdd(hash, tdb->track, tdb); } // go through and find the container tracks for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { char *parentLine = trackDbLocalSetting(tdb, "parent"); // maybe it's a child of a supertrack? if (parentLine == NULL) { parentLine = trackDbLocalSetting(tdb, "superTrack"); if ((parentLine != NULL) && startsWith("on", parentLine)) parentLine = NULL; } if (parentLine != NULL) { char *parentName = cloneFirstWord(parentLine); if (sameString(parentName, tdb->track)) errAbort("Track %s lists itself as its own parent in hub %s genome %s", tdb->track, hub->url, genome->name); struct trackDb *parent = hashFindVal(hash, parentName); if (parent == NULL) errAbort("Parent %s of track %s doesn't exist in hub %s genome %s", parentName, tdb->track, hub->url, genome->name); // mark the parent as a container parent->subtracks = tdb; // ugh...do this so requiredSetting looks at parent // in the case of views. We clear this after // validating anyway tdb->parent = parent; freeMem(parentName); } } hashFree(&hash); } static void validateTracks( struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdbList) /* Make sure a hub track list has the right settings and its parents exist. */ { // mark the containers by setting their subtracks pointer markContainers(hub, genome, tdbList); /* Loop through list checking tags */ struct trackDb *tdb; for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) { validateOneTrack(hub, genome, tdb); } errCatchEnd(errCatch); if (errCatch->gotError) { tdb->errMessage = cloneString(errCatch->message->string); } // clear these two pointers which we set in markContainers tdb->subtracks = NULL; tdb->parent = NULL; } } struct trackDb *trackHubTracksForGenome(struct trackHub *hub, struct trackHubGenome *genome, struct dyString *incFiles, boolean *foundFirstGenome) /* Get list of tracks associated with genome. Check that it only is composed of legal * types. Do a few other quick checks to catch errors early. If incFiles is not NULL, * put the list of included files in there. Only the first example of a genome * gets to populate groups, the others get a group for the trackHub. */ { struct lineFile *lf = udcWrapShortLineFile(genome->trackDbFile, NULL, MAX_HUB_TRACKDB_FILE_SIZE); struct trackDb *tdbList = trackDbFromOpenRa(lf, NULL, incFiles); lineFileClose(&lf); char *tabMetaName = hashFindVal(genome->settingsHash, "metaTab"); char *absTabName = NULL; if (tabMetaName) absTabName = trackHubRelativeUrl(hub->url, tabMetaName); char *tagStormName = hashFindVal(genome->settingsHash, "metaDb"); char *absStormName = NULL; if (tagStormName) absStormName = trackHubRelativeUrl(hub->url, tagStormName); /* Make bigDataUrls more absolute rather than relative to genome.ra dir */ struct trackDb *tdb; for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { expandBigDataUrl(hub, genome, tdb); if (absStormName) hashReplace(tdb->settingsHash, "metaDb", absStormName); if (absTabName) hashReplace(tdb->settingsHash, "metaTab", absTabName); } validateTracks(hub, genome, tdbList); trackDbAddTableField(tdbList); if (!isEmpty(hub->name)) trackHubAddNamePrefix(hub->name, tdbList); if ((genome->twoBitPath != NULL) && (*foundFirstGenome == FALSE)) *foundFirstGenome = TRUE; else if ((genome->groups != NULL) && hubsCanAddGroups()) tdbListAddHubToGroup(hub->name, tdbList); else trackHubAddGroupName(hub->name, tdbList); for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { trackDbFieldsFromSettings(tdb); trackDbPolish(tdb); } return tdbList; } static void reprefixString(char **pString, char *prefix) /* Replace *pString with prefix + *pString, freeing * whatever was in *pString before. */ { char *oldName = *pString; *pString = catTwoStrings(prefix, oldName); freeMem(oldName); } static void addPrefixToSetting(struct hash *settings, char *key, char *prefix) /* Given a settings hash, which is string valued. Old values will be freed. */ { struct hashEl *hel = hashLookup(settings, key); if (hel != NULL) reprefixString((char **)&hel->val, prefix); } static void trackDbListAddNamePrefix(struct trackDb *tdbList, char *prefix) /* Surgically alter tdbList so that it works as if every track was * renamed so as to add a prefix to it's name. */ { struct trackDb *tdb; for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { addPrefixToSetting(tdb->settingsHash, "track", prefix); addPrefixToSetting(tdb->settingsHash, "parent", prefix); reprefixString(&tdb->track, prefix); if (tdb->table != NULL) reprefixString(&tdb->table, prefix); } } void trackHubAddNamePrefix(char *hubName, struct trackDb *tdbList) /* For a hub named "hub_1" add the prefix "hub_1_" to each track and parent field. */ { char namePrefix[PATH_LEN]; safef(namePrefix, sizeof(namePrefix), "%s_", hubName); trackDbListAddNamePrefix(tdbList, namePrefix); } char *trackHubGetHubName(char *name) /* Get the hub_#_ prefix in a hub name. */ { if ((name == NULL) || !startsWith("hub_", name)) return NULL; char *ret = cloneString(name); char *under = strchr(&ret[4], '_'); if (under) { *under = 0; return ret; } return NULL; } char *trackHubSkipHubName(char *name) /* Skip the hub_#_ prefix in a hub name. */ { if ((name == NULL) || !startsWith("hub_", name)) return name; return strchr(&name[4], '_') + 1; } struct trackDb *findTdbByBareName(struct trackDb *tdbList, char *bareName) /* Recursively search tdbList (and subtracks) for a tdb whose bare track name matches. */ { struct trackDb *tdb; for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { if (sameString(trackHubSkipHubName(tdb->track), bareName)) return tdb; struct trackDb *found = findTdbByBareName(tdb->subtracks, bareName); if (found != NULL) return found; } return NULL; } void trackHubAddGroupName(char *hubName, struct trackDb *tdbList) /* Add group tag that references the hubs symbolic name. */ { struct trackDb *tdb; for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { tdb->grp = cloneString(hubName); hashReplace(tdb->settingsHash, "group", tdb->grp); } } static char *trackHubDescriptionText(char *trackDbFile, struct trackDb *tdb) /* Fetch the text of tdb->track's html description page, or NULL if it has none. */ { /* html setting should always be set because we set it at load time */ char *htmlName = trackDbSetting(tdb, "html"); if (htmlName == NULL) return NULL; char *simpleName = hubConnectSkipHubPrefix(htmlName); char *url = trackHubRelativeUrl(trackDbFile, simpleName); char buffer[10*1024]; char *fixedUrl = url; if (!endsWith(url, ".html")) { safef(buffer, sizeof buffer, "%s.html", url); fixedUrl = buffer; } char *html = udcFileReadAllIfExists(fixedUrl, NULL, 0, NULL); freez(&url); return html; } void trackHubAddOneDescription(char *trackDbFile, struct trackDb *tdb) /* Fetch tdb->track's html description and store in tdb->html. */ { char *html = trackHubDescriptionText(trackDbFile, tdb); if (html == NULL) return; /* no page of its own, so leave any it inherited alone */ tdb->html = htmlSanitize(html); freeMem(html); } struct slName *trackHubDescriptionRemovals(char *trackDbFile, struct trackDb *tdb) /* Return a list of messages naming the parts of tdb's description page that we do not * print, or NULL if we print all of it. */ { char *html = trackHubDescriptionText(trackDbFile, tdb); struct slName *removed = NULL; char *clean = htmlSanitizeReport(html, &removed); freeMem(html); freeMem(clean); return removed; } void trackHubAddDescription(char *trackDbFile, struct trackDb *tdb) /* Fetch tdb->track's html description (or nearest ancestor's non-empty description) * and store in tdb->html. */ { trackHubAddOneDescription(trackDbFile, tdb); if (isEmpty(tdb->html)) { struct trackDb *parent; for (parent = tdb->parent; isEmpty(tdb->html) && parent != NULL; parent = parent->parent) { trackHubAddOneDescription(trackDbFile, parent); if (isNotEmpty(parent->html)) tdb->html = cloneString(parent->html); } } } void trackHubFixName(char *name) /* Change all characters other than alphanumeric, dash, and underbar * to underbar. */ { if (name == NULL) return; char *in = name; char c; for(; (c = *in) != 0; in++) { if (c == ' ') break; if (!(isalnum(c) || c == '-' || c == '_')) *in = '_'; } } static void polishOneTrack( struct trackHub *hub, struct trackDb *bt, struct hash *hash) /* Get rid of special characters in track name, squirrel away a copy * of the original name for html retrieval, make sure there aren't * two tracks with the same name. */ { char *polished = trackDbLocalSetting(bt, "polished"); if (polished != NULL) return; trackDbAddSetting(bt, "polished", "polished"); char *htmlName = trackDbSetting(bt, "html"); /* if the user didn't specify an html variable, set it to be the original * track name */ if (htmlName == NULL) trackDbAddSetting(bt, "html", bt->track); trackHubFixName(bt->track); if (hashLookup(hash, bt->track) != NULL) errAbort("more than one track called %s in hub %s\n", bt->track, hub->url); hashStore(hash, bt->track); } void trackHubPolishTrackNames(struct trackHub *hub, struct trackDb *tdbList) /* Remove all the special characters from trackHub track names. */ { struct trackDb *next, *tdb; struct hash *nameHash = hashNew(5); for (tdb = tdbList; tdb != NULL; tdb = next) { if (tdb->parent != NULL) polishOneTrack(hub, tdb->parent, nameHash); next = tdb->next; polishOneTrack(hub, tdb, nameHash); if (tdb->subtracks != NULL) { trackHubPolishTrackNames(hub, tdb->subtracks); } } } void trackHubFindPos(struct cart *cart, char *db, char *term, struct hgPositions *hgp, boolean measureTiming) /* Look for term in track hubs. Update hgp if found */ { findBigBedPosInTdbList(cart, db, hubCollectTracks(db, NULL), term, hgp, NULL, measureTiming); } static void parseBlatPcrParams(char *database, char *type, char *setting, char **pHost, char **pPort, char **pGenomeDataDir) /* parser parameters for either blat or pcr */ { char *conf = trimSpaces(cloneString(setting)); int numWords = chopByWhite(conf, NULL, 5); if ((numWords < 2) || (numWords > 4)) errAbort("invalid configuration for hub %s server, expect 2 or 4 arguments: %s", type, setting); char *words[4]; chopByWhite(conf, words, numWords); *pHost = words[0]; *pPort = words[1]; if (numWords > 2) { if (!sameString(words[2], "dynamic")) errAbort("invalid configuration for hub %s server, third argument should be 'dynamic' or omitted, got: %s", type, words[2]); *pGenomeDataDir = words[3]; } else *pGenomeDataDir = NULL; } boolean trackHubGetPcrParams(char *database, char **pHost, char **pPort, char **pGenomeDataDir) /* Get the isPcr params from a trackHub genome. */ { char *type = "isPcr"; char *setting = trackHubAssemblyField(database, type); parseBlatPcrParams(database, type, setting, pHost, pPort, pGenomeDataDir); return TRUE; } boolean trackHubGetBlatParams(char *database, boolean isTrans, char **pHost, char **pPort, char **pGenomeDataDir) { char *type = isTrans ? "transBlat" : "blat"; char *setting = trackHubAssemblyField(database, type); if (setting == NULL) return FALSE; parseBlatPcrParams(database, type, setting, pHost, pPort, pGenomeDataDir); return TRUE; } void hubCheckBigDataUrl(struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdb) /* Check remote file exists and is of correct type. Wrap this in error catcher */ { char *relativeUrl = trackDbSetting(tdb, "bigDataUrl"); if (relativeUrl != NULL) { char *type = trackDbRequiredSetting(tdb, "type"); char *bigDataUrl = trackHubRelativeUrl(genome->trackDbFile, relativeUrl); char *bigDataIndex = NULL; char *relIdxUrl = trackDbSetting(tdb, "bigDataIndex"); if (relIdxUrl != NULL) bigDataIndex = trackHubRelativeUrl(genome->trackDbFile, relIdxUrl); verbose(2, "checking %s.%s type %s at %s\n", genome->name, tdb->track, type, bigDataUrl); if (startsWithWord("bigWig", type)) { /* Just open and close to verify file exists and is correct type. */ struct bbiFile *bbi = bigWigFileOpen(bigDataUrl); bbiFileClose(&bbi); } else if (startsWithWord("bigNarrowPeak", type) || startsWithWord("bigBed", type) || startsWithWord("bigGenePred", type) || startsWithWord("bigPsl", type)|| startsWithWord("bigChain", type)|| startsWithWord("bigMaf", type) || startsWithWord("bigBarChart", type) || startsWithWord("bigInteract", type) || startsWithWord("bigLolly", type) || startsWithWord("bigRmsk",type) || startsWithWord("bigMethyl", type)) { /* Just open and close to verify file exists and is correct type. */ struct bbiFile *bbi = bigBedFileOpen(bigDataUrl); char *typeString = cloneString(type); nextWord(&typeString); if (startsWithWord("bigBed", type) && (typeString != NULL)) { unsigned numFields = sqlUnsigned(nextWord(&typeString)); if (numFields > bbi->fieldCount) errAbort("bigBed file '%s' has %d fields, but track \"%s\" declares 'type bigBed %d'. Either regenerate the bigBed with the correct number of fields, or change the type line to match the file's field count.", bigDataUrl, bbi->fieldCount, trackHubSkipHubName(tdb->track), numFields); } bbiFileClose(&bbi); } else if (startsWithWord("vcfTabix", type) || startsWithWord("vcfPhasedTrio", type)) { /* Just open and close to verify file exists and is correct type. */ struct vcfFile *vcf = vcfTabixFileAndIndexMayOpen(bigDataUrl, bigDataIndex, NULL, 0, 0, 1, 1); if (vcf == NULL) // Warnings already indicated whether the tabix file is missing etc. errAbort("Couldn't open %s and/or its tabix index (.tbi) file for track %s. " "Both the .vcf.gz file and a matching .vcf.gz.tbi index must be publicly accessible " "at the same URL path. Generate the index with: tabix -p vcf yourFile.vcf.gz " "See https://genome.ucsc.edu/goldenPath/help/vcf.html", bigDataUrl, trackHubSkipHubName(tdb->track)); vcfFileFree(&vcf); } else if (startsWithWord("bam", type)) { bamFileAndIndexMustExist(bigDataUrl, bigDataIndex); } else if (startsWithWord("longTabix", type)) { struct bedTabixFile *btf = bedTabixFileMayOpen(bigDataUrl, NULL, 0, 0); if (btf == NULL) errAbort("Couldn't open %s and/or its tabix index (.tbi) file for track %s. " "Both the file and a matching .tbi index must be publicly accessible " "at the same URL path.", bigDataUrl, trackHubSkipHubName(tdb->track)); bedTabixFileClose(&btf); } else if (startsWithWord("halSnake", type)) { #ifdef USE_HAL char *errString; int handle = halOpenLOD(bigDataUrl, &errString); if (handle < 0) errAbort("HAL open error: %s", errString); if (halClose(handle, &errString) < 0) errAbort("HAL close error: %s", errString); #else warn("Note: can not check track %s, validation of halSnake tracks not compiled into this copy of hubCheck", trackHubSkipHubName(tdb->track)); #endif } else if (startsWithWord("hic", type)) { struct hicMeta *header; char *errString = hicLoadHeader(bigDataUrl, &header, genome->name); if (errString != NULL) errAbort("hic file error: %s", errString); } else errAbort("unrecognized type %s in genome %s track %s", type, genome->name, tdb->track); freez(&bigDataUrl); } } static void outHubHeader(struct dyString *dy, char *db) // output a track hub header { dyStringPrintf(dy,"hub quickLiftHub%s\n\ shortLabel QuickLift from %s\n\ longLabel QuickLift from %s\n\ useOneFile on\n\ email genome-www@soe.ucsc.edu\n\n", db, db, db); dyStringPrintf(dy,"genome %s\n\n", db); } static char *getHubName(struct cart *cart, char *db) // get the name of the hub to use for quickLifted tracks { struct tempName hubTn; char buffer[4096]; #define quickLiftCartName "hubQuickLift" safef(buffer, sizeof buffer, "%s-%s", quickLiftCartName, db); char *hubName = cartOptionalString(cart, buffer); int fd = -1; // we don't reuse userdata paths since they are in save sessions if ((hubName != NULL) && strstr(hubName, "userdata")) hubName = NULL; if ((hubName == NULL) || ((fd = open(hubName, 0)) < 0)) { trashDirDateFile(&hubTn, "quickLift", "hub", ".txt"); hubName = cloneString(hubTn.forCgi); cartSetString(cart, buffer, hubName); } return hubName; } struct quickLiftStanza /* One track stanza parsed out of a quickLift hub file. */ { struct quickLiftStanza *next; char *name; /* bare track name */ char *parent; /* bare parent track name, or NULL */ struct dyString *text; /* full stanza text including final newline */ }; static char *firstWordClone(char *s) /* Return a clone of the first whitespace-delimited word of s, or NULL. */ { s = skipLeadingSpaces(s); if (isEmpty(s)) return NULL; char *sp = skipToSpaces(s); int len = (sp != NULL) ? (sp - s) : (int)strlen(s); return cloneStringZ(s, len); } static struct quickLiftStanza *readStanzas(struct lineFile *lf, struct dyString *header) /* Read track stanzas out of a quickLift hub file, or out of a string of freshly * generated stanzas. Everything ahead of the first track line goes into header. * A container is many stanzas here, one per track line, not a single block. */ { struct quickLiftStanza *stanzaList = NULL; struct quickLiftStanza *cur = NULL; char *line; int lineSize; while (lineFileNext(lf, &line, &lineSize)) { char *trim = skipLeadingSpaces(line); if (startsWithWord("track", trim)) { AllocVar(cur); cur->text = dyStringNew(0); cur->name = firstWordClone(trim + 5); slAddHead(&stanzaList, cur); } else if (cur != NULL && startsWithWord("parent", trim)) { if (cur->parent == NULL) cur->parent = firstWordClone(trim + 6); } struct dyString *target = (cur != NULL) ? cur->text : header; dyStringAppend(target, line); dyStringAppendC(target, '\n'); } slReverse(&stanzaList); return stanzaList; } static void freeStanzas(struct quickLiftStanza **pList) /* Free a list of stanzas. */ { struct quickLiftStanza *s; for (s = *pList; s != NULL; s = s->next) { dyStringFree(&s->text); freeMem(s->name); freeMem(s->parent); } slFreeList(pList); } boolean quickLiftHubRemoveTrack(struct cart *cart, char *sourceDb, char *trackName) /* Remove a track stanza from the quickLift hub file for sourceDb, along with * any descendant stanzas (transitively) whose parent is being removed. * Returns TRUE if at least one stanza was removed. */ { char buffer[4096]; safef(buffer, sizeof buffer, "%s-%s", quickLiftCartName, sourceDb); char *filename = cartOptionalString(cart, buffer); if (filename == NULL || !isServerUserFilePath(filename)) return FALSE; struct lineFile *lf = lineFileMayOpen(filename, TRUE); if (lf == NULL) return FALSE; char *bareName = trackHubSkipHubName(trackName); struct dyString *header = dyStringNew(0); /* Pass 1: read the file into a header + list of stanzas, recording each * stanza's name and (if any) parent. */ struct quickLiftStanza *stanzaList = readStanzas(lf, header); lineFileClose(&lf); /* Build a removal set: start with the named track, then iterate adding any * stanza whose parent is already in the set, until the set is stable. */ struct hash *removeSet = hashNew(0); hashStore(removeSet, bareName); boolean changed = TRUE; while (changed) { changed = FALSE; struct quickLiftStanza *s; for (s = stanzaList; s != NULL; s = s->next) { if (s->name == NULL || s->parent == NULL) continue; if (hashLookup(removeSet, s->name) != NULL) continue; if (hashLookup(removeSet, s->parent) != NULL) { hashStore(removeSet, s->name); changed = TRUE; } } } boolean removedAny = FALSE; struct dyString *out = dyStringNew(0); struct quickLiftStanza *s; for (s = stanzaList; s != NULL; s = s->next) { if (s->name != NULL && hashLookup(removeSet, s->name) != NULL) removedAny = TRUE; else dyStringAppend(out, s->text->string); } if (removedAny) { FILE *f = mustOpen(filename, "w"); chmod(filename, 0666); fputs(header->string, f); fputs(out->string, f); fclose(f); } dyStringFree(&header); dyStringFree(&out); hashFree(&removeSet); freeStanzas(&stanzaList); return removedAny; } static char *vettedTracks[] = /* tracks that have been tested with quickLift */ { "decipherContainer", "decipherSnvs", "omimLocation", "omimAvSnp", "ncbiRefSeq", "clinvar", "clinvarSubLolly", "pubs", "pubsBlat", "pubsMarkerBand", "pubsMarkerSnp", "pubsMarkerGene", "refSeqComposite", //"gtexGeneV8", }; static boolean isVetted(char *track) /* Is this a track that's been tested with quickLift? If not we don't want to do the special name handling on the track. */ { if (startsWith("wgEncodeGencode", track)) return TRUE; // NCBI RefSeq composite subtracks (ncbiRefSeqCurated/Predicted/Other/Psl/ // Select/Hgmd/Historical, ncbiOrtho) and UCSC RefSeq (refGene) have been // validated through quickLift; let their native hgc handlers fire. if (startsWith("ncbiRefSeq", track) || startsWith("ncbiOrtho", track) || sameString("refGene", track)) return TRUE; static bool inited = FALSE; static struct hash *vettedHash = NULL; if (!inited) { vettedHash = newHash(10); int ii; int len = sizeof(vettedTracks) / sizeof(char *); for(ii = 0; ii < len; ii++) hashStore(vettedHash, vettedTracks[ii]); } return hashLookup(vettedHash, track) != NULL; } static bool isParentVisible(struct cart *cart, struct trackDb *tdb) // Are this track's parents visible? { if (tdb->parent == NULL) return TRUE; if (!isParentVisible(cart, tdb->parent)) return FALSE; char *cartVis = cartOptionalString(cart, tdb->parent->track); boolean vis; if (cartVis != NULL) vis = differentString(cartVis, "hide"); else if (tdbIsSuperTrack(tdb->parent)) vis = tdb->parent->isShow; else vis = tdb->parent->visibility != tvHide; return vis; } static bool subtrackEnabledInTdb(struct trackDb *subTdb) /* Return TRUE unless the subtrack was declared with "subTrack ... off". */ { bool enabled = TRUE; char *words[2]; char *setting; if ((setting = trackDbLocalSetting(subTdb, "parent")) != NULL) { if (chopLine(cloneString(setting), words) >= 2) if (sameString(words[1], "off")) enabled = FALSE; } else return subTdb->visibility != tvHide; return enabled; } static bool isSubtrackVisible(struct cart *cart, struct trackDb *tdb) /* Has this subtrack not been deselected in hgTrackUi or declared with * * "subTrack ... off"? -- assumes composite track is visible. */ { boolean overrideComposite = (NULL != cartOptionalString(cart, tdb->track)); bool enabledInTdb = subtrackEnabledInTdb(tdb); char option[1024]; safef(option, sizeof(option), "%s_sel", tdb->track); boolean enabled = cartUsualBoolean(cart, option, enabledInTdb); if (overrideComposite) enabled = TRUE; return enabled; } static void dumpTdbAndChildren(struct cart *cart, struct dyString *dy, struct trackDb *tdb) /* Put a trackDb entry into a dyString, stepping up the tree for some variables. */ { struct hashCookie cookie = hashFirst(tdb->settingsHash); struct hashEl *hel; char *cartVis = cartOptionalString(cart, tdb->track); if (cartVis != NULL) tdb->visibility = hTvFromString(cartVis); dyStringPrintf(dy, "visibility %s\n", hStringFromTv(tdb->visibility)); if (tdbIsSuperTrack(tdb->parent)) dyStringPrintf(dy, "parent %s\n", trackHubSkipHubName(tdb->parent->track)); while ((hel = hashNext(&cookie)) != NULL) { if (sameString(hel->name, "parent")) { char buffer[1024]; safef(buffer, sizeof buffer, "%s_sel", tdb->track); char *cartSelected = cartOptionalString(cart, buffer); if (cartSelected != NULL) { char *str = (sameWord(cartSelected, "on") || atoi(cartSelected) > 0) ? "on" : "off"; dyStringPrintf(dy, "parent %s %s\n", trackHubSkipHubName(tdb->parent->track), str); } else dyStringPrintf(dy, "%s %s\n", hel->name, trackHubSkipHubName(((char *)hel->val))); } else if (sameString(hel->name, "html")) dyStringPrintf(dy, "%s %s\n", hel->name, trackHubSkipHubName((char *)hel->val)); else if (differentString(hel->name, "track") && differentString(hel->name, "visibility")) dyStringPrintf(dy, "%s %s\n", hel->name, ((char *)hel->val)); } if (tdb->subtracks) { for (tdb = tdb->subtracks; tdb; tdb = tdb->next) { char *track = trackHubSkipHubName(tdb->track); dyStringPrintf(dy, "\ntrack %s\nquickLifted on\n", track); if (!isVetted(track)) dyStringPrintf(dy, "avoidHandler on\n"); dumpTdbAndChildren(cart, dy, tdb); } } } struct dyString *trackDbString(struct cart *cart, struct trackDb *tdb) /* Convert a trackDb entry into a dyString. */ { struct dyString *dy; // add a note that the name based handler shouldn't be used on this track // add a note that this is a quickLifted track so the browser will accept tracks that aren't big* dy = dyStringNew(200); char *track = trackHubSkipHubName(tdb->track); dyStringPrintf(dy, "track %s\nquickLifted on\n", track); if (tdbIsSuperTrack(tdb)) { dyStringPrintf(dy, "superTrack on show\n"); } if (!isVetted(track)) dyStringPrintf(dy, "avoidHandler on\n"); dumpTdbAndChildren(cart, dy, tdb); return dy; } static boolean validateOneTdb(char *db, struct trackDb *tdb, struct trackDb **badList) /* Make sure the tdb is a track type we grok. badList may be NULL to validate * silently (no user-facing complaint about non-liftable types). */ { // trackDb types are matched without regard to case since that's how the rest of the // browser reads them (some trackDb stanzas say "bigbed" rather than "bigBed"). if (sameString("cytoBandIdeo", trackHubSkipHubName(tdb->track)) || !( startsWithNoCase("bigBed", tdb->type) || \ startsWithNoCase("bigWig", tdb->type) || \ startsWithNoCase("bigDbSnp", tdb->type) || \ startsWithNoCase("bigGenePred", tdb->type) || \ startsWithNoCase("gvf", tdb->type) || \ startsWithNoCase("genePred", tdb->type) || \ startsWithNoCase("narrowPeak", tdb->type) || \ startsWithNoCase("broadPeak", tdb->type) || \ startsWithNoCase("bigLolly", tdb->type) || \ + startsWithNoCase("bigPsl", tdb->type) || \ + sameWord("psl", tdb->type) || + startsWithNoCase("psl ", tdb->type) || sameWord("bed", tdb->type) || startsWithNoCase("bed ", tdb->type))) { if (badList != NULL) slAddHead(badList, tdb); return FALSE; } // make sure we have a bigDataUrl if (startsWithNoCase("bigBed", tdb->type) || \ + startsWithNoCase("bigPsl", tdb->type) || \ startsWithNoCase("bigWig", tdb->type)) { char *fileName = cloneString(trackDbSetting(tdb, "bigDataUrl")); if (fileName == NULL) { struct sqlConnection *conn = hAllocConnTrack(db, tdb); fileName = bbiNameFromSettingOrTable(tdb, conn, tdb->table); hashAdd(tdb->settingsHash, "bigDataUrl", fileName); hFreeConn(&conn); } } return TRUE; } static struct trackDb * validateTdbChildren(struct cart *cart, char *db, struct trackDb *tdb, struct trackDb **badList) /* return a list of the children that can be quick lifted */ { struct trackDb *validTdbs = NULL; struct trackDb *nextTdb; unsigned count = 0; if (tdb->subtracks) // this is a view, descend again { struct trackDb *view = tdb; for (; view; view = nextTdb) { nextTdb = view->next; view->subtracks = validateTdbChildren(cart, db,view->subtracks, badList); if (view->subtracks != NULL) { slAddHead(&validTdbs, view); if (view->visibility) count++; } } } else { for(; tdb; tdb = nextTdb) { nextTdb = tdb->next; boolean visible = isParentVisible(cart, tdb) && isSubtrackVisible(cart, tdb); // Lift all siblings of a visible subtrack, but only complain about // non-liftable ones the user actually asked for (visible ones). if (validateOneTdb(db, tdb, visible ? badList : NULL)) { slAddHead(&validTdbs, tdb); if (visible) count++; } } } if (count) return validTdbs; return NULL; } static boolean validateTdb(struct cart *cart, char *db, struct trackDb *tdb, struct trackDb **badList) // make sure we only output track types that can // be quickLifted. Return true if we any tracks survive { if (tdb->subtracks) { tdb->subtracks = validateTdbChildren(cart, db, tdb->subtracks, badList); if (tdb->subtracks == NULL) return FALSE; return TRUE; } return validateOneTdb(db, tdb, badList); } static void outTrack(struct dyString *out, struct cart *cart, struct trackDb *tdb, double priority) /* Set priority and output track to hub. */ { char buffer[1024]; safef(buffer, sizeof buffer, "%g", priority); hashReplace(tdb->settingsHash, "priority", cloneString(buffer)); struct dyString *dy = trackDbString(cart, tdb); dyStringPrintf(out, "%s\n", dy->string); dyStringFree(&dy); } static boolean checkCartVisibility(struct cart *cart, struct trackDb *tdb) { char *cartVis = cartOptionalString(cart, tdb->track); if (cartVis != NULL) tdb->visibility = hTvFromString(cartVis); return (tdb->visibility != tvHide); } static boolean isFromQuickLiftHub(struct trackDb *tdb) /* True if this tdb came from a quickLift hub (already a lifted shadow track). * Such tracks must not be lifted again. */ { return trackDbSetting(tdb, "quickLiftUrl") != NULL || trackDbSetting(tdb, "quickLifted") != NULL; } static void walkTree(struct dyString *out, char *db, struct cart *cart, struct trackDb *tdb, struct trackDb **badList) /* walk tree looking for visible tracks to output to hub. Skip tracks that already * came from a quickLift hub. Every visible track is written, whether or not it is * already in the hub file: the caller merges these stanzas over the old ones, so a * track that was lifted before gets its current state rather than the one it had * when it was first lifted. */ { struct hash *haveSuper = newHash(0); struct trackDb *tdbNext = NULL; // The priority written to the hub is the track's rank in the source list, which the // caller has sorted on group priority and then track priority. The rank has to count // every track we walk past, not just the ones we output: tracks accumulate in the hub // file across requests, so the number a track gets must depend only on how the source // is laid out, never on which request it happened to be added in. // // The source priority itself will not do. All lifted tracks land in one group on the // target (trackHubAddGroupName rewrites the group of every hub track), so a priority // that only orders within a source group is being compared across groups. double rank = 0; for(; tdb; tdb = tdbNext) { tdbNext = tdb->next; rank += 1; if (isFromQuickLiftHub(tdb)) continue; boolean isVisible = FALSE; if (tdb->parent == NULL) isVisible = checkCartVisibility(cart, tdb); else if (isParentVisible(cart, tdb) && isSubtrackVisible(cart, tdb)) // child of supertrack { if (hashLookup(haveSuper, tdb->parent->track) == NULL) // output yet? { //if (checkCartVisibility(cart, tdb->parent)) { tdb->parent->visibility = hTvFromString("tvShow"); // a superTrack is not in the list we are walking, so it has no rank // of its own. Slot it just above the first child that brought it in. outTrack(out, cart, tdb->parent, rank - 0.5); hashStore(haveSuper, tdb->parent->track); } } isVisible = checkCartVisibility(cart, tdb); } if (isVisible && validateTdb(cart, db, tdb, badList)) { hashRemove(tdb->settingsHash, "superTrack"); // this gets inherited by subTracks(?) // is this a custom track? char *tdbType = trackDbSetting(tdb, "tdbType"); if (tdbType != NULL) { hashReplace(tdb->settingsHash, "type", tdbType); hashReplace(tdb->settingsHash, "shortLabel", trackDbSetting(tdb, "name")); hashReplace(tdb->settingsHash, "longLabel", trackDbSetting(tdb, "description")); } outTrack(out, cart, tdb, rank); } } } static void writeMergedHubFile(char *filename, char *db, struct dyString *newContent) /* Write the hub file from the stanzas we just generated plus whatever was already * in the file. A generated stanza replaces the old stanza of the same track, in * the slot the old one held, so the file keeps parents ahead of their children. * A track in the file that we did not generate this time is kept as it was, so * tracks still accumulate across lifts in one session. */ { struct dyString *header = dyStringNew(0); struct quickLiftStanza *oldList = NULL; struct lineFile *lf = lineFileMayOpen(filename, TRUE); if (lf != NULL) { oldList = readStanzas(lf, header); lineFileClose(&lf); } /* The generated stanzas have no header of their own; scratch collects nothing. */ struct dyString *scratch = dyStringNew(0); lf = lineFileOnString("quickLift stanzas", TRUE, cloneString(newContent->string)); struct quickLiftStanza *newList = readStanzas(lf, scratch); lineFileClose(&lf); struct hash *newByName = newHash(8); struct quickLiftStanza *s; for (s = newList; s != NULL; s = s->next) { if ((s->name != NULL) && (hashLookup(newByName, s->name) == NULL)) hashAdd(newByName, s->name, s); } struct dyString *out = dyStringNew(0); if (isEmpty(header->string)) outHubHeader(out, trackHubSkipHubName(db)); else dyStringAppend(out, header->string); struct hash *written = newHash(8); for (s = oldList; s != NULL; s = s->next) { struct quickLiftStanza *fresh = (s->name == NULL) ? NULL : hashFindVal(newByName, s->name); if (fresh == NULL) dyStringAppend(out, s->text->string); else if (hashLookup(written, fresh->name) == NULL) { dyStringAppend(out, fresh->text->string); hashStore(written, fresh->name); } } for (s = newList; s != NULL; s = s->next) { if ((s->name != NULL) && (hashLookup(written, s->name) != NULL)) continue; dyStringAppend(out, s->text->string); if (s->name != NULL) hashStore(written, s->name); } /* Write a temporary file and rename it into place. The target assembly reads this * same file, and rewriting it in place would show a reader a truncated hub. */ char tmpName[PATH_LEN]; safef(tmpName, sizeof tmpName, "%s.tmp", filename); FILE *f = mustOpen(tmpName, "w"); fputs(out->string, f); carefulClose(&f); chmod(tmpName, 0666); mustRename(tmpName, filename); dyStringFree(&out); dyStringFree(&header); dyStringFree(&scratch); hashFree(&newByName); hashFree(&written); freeStanzas(&oldList); freeStanzas(&newList); } static int cmpPriority(const void *va, const void *vb) /* Compare to sort based on priority; use shortLabel as secondary sort key. */ { const struct trackDb *a = *((struct trackDb **)va); const struct trackDb *b = *((struct trackDb **)vb); float dif = 0; dif = a->groupPriority - b->groupPriority; if (dif == 0) dif = a->priority - b->priority; if (dif < 0) return -1; else if (dif == 0.0) /* secondary sort on label */ return strcasecmp(a->shortLabel, b->shortLabel); else return 1; } char *trackHubBuild(char *db, struct cart *cart, struct trackDb **badList) /* Build a track hub using trackDb and the cart. If a hub file already exists * for db (i.e. earlier quickLift work in the same session), merge the new track * stanzas into it: a track that is being lifted again gets the state it has now, * and a track that is in the file but is not visible on the source any more is * left alone. */ { struct trackDb *tdbList, *tdb; struct grp *grpList; cartTrackDbInit(cart, &tdbList, &grpList, FALSE); struct hash *groupHash = newHash(0); struct grp *grp; for(grp = grpList; grp; grp = grp->next) hashAdd(groupHash, grp->name, grp); for(tdb = tdbList; tdb; tdb = tdb->next) { grp = hashFindVal(groupHash, tdb->grp); tdb->groupPriority = grp->priority; } slSort(&tdbList, cmpPriority); char *filename = getHubName(cart, db); struct dyString *newContent = dyStringNew(0); walkTree(newContent, db, cart, tdbList, badList); writeMergedHubFile(filename, db, newContent); dyStringFree(&newContent); return cloneString(filename); } struct grp *trackHubGetGrps() /* Get the groups defined by attached track hubs. */ { return trackHubGrps; } struct trackDb *trackHubAddTracksGenome(struct trackHubGenome *hubGenome) /* Load up stuff from data hub and return list. */ { /* Load trackDb.ra file and make it into proper trackDb tree */ struct trackDb *tdbList = NULL; if (hubGenome != NULL) { boolean doCache = trackDbCacheOn(); if (doCache) { // we have to open the trackDb file to get the udc cache to check for an update struct udcFile *checkCache = udcFileMayOpen(hubGenome->trackDbFile, NULL); if (checkCache != NULL) { time_t time = udcUpdateTime(checkCache); udcFileClose(&checkCache); struct trackDb *cacheTdb = trackDbHubCache(hubGenome->trackDbFile, time); if (cacheTdb != NULL) return cacheTdb; } memCheckPoint(); // we want to know how much memory is used to build the tdbList } struct dyString *incFiles = newDyString(4096); boolean foundFirstGenome = FALSE; tdbList = trackHubTracksForGenome(hubGenome->trackHub, hubGenome, incFiles, &foundFirstGenome); tdbList = trackDbLinkUpGenerations(tdbList); tdbList = trackDbPolishAfterLinkup(tdbList, hubGenome->name); trackDbPrioritizeContainerItems(tdbList); trackHubPolishTrackNames(hubGenome->trackHub, tdbList); if (doCache) trackDbHubCloneTdbListToSharedMem(hubGenome->trackDbFile, tdbList, memCheckPoint(), incFiles->string); } return tdbList; }