f001e39d1561c86eb6832c556888e73dc200f8d0 braney Thu Aug 27 14:10:09 2026 -0700 quickLift: let a second lift update a track already in the hub, refs #38198 The hub writer skipped any track whose name was already in the hub file. The first lift writes a container together with all of its children, including the ones switched off, so those names were taken from then on. A later lift skipped the container and everything inside it: a subtrack switched on after the first lift never reached the target, and a visibility change on a track that had already been lifted never reached it either. walkTree no longer takes the set of names already in the file and no longer skips a track for being there. It writes every visible track into a dyString, and the new writeMergedHubFile merges that against the file: a generated stanza replaces the old stanza of the same track, in the slot the old one held, so parents stay ahead of their children; a track in the file that was not generated this time is kept as it was, so tracks still accumulate across lifts in one session. The skip could not simply be dropped because a duplicate track name makes hub loading abort, and replacing in place keeps names unique. The stanza parser inside quickLiftHubRemoveTrack is now readStanzas, shared by both paths. A container is many stanzas there, one per track line, which is what makes the per-track replacement work. The file is written to a temporary name and renamed into place. Every lift now rewrites the whole file and the target assembly reads that same file, so an in-place rewrite could hand a reader a truncated hub. outTrack also frees the dyString it gets from trackDbString. diff --git src/hg/lib/trackHub.c src/hg/lib/trackHub.c index 5efc8689c61..7689dc8ff9a 100644 --- src/hg/lib/trackHub.c +++ src/hg/lib/trackHub.c @@ -46,30 +46,31 @@ #include "trix.h" #include "vcf.h" #include "vcfUi.h" #include "htmshell.h" #include "bigBedFind.h" #include "customComposite.h" #include "interactUi.h" #include "bedTabix.h" #include "hic.h" #include "hui.h" #include "chromAlias.h" #include "trashDir.h" #include "hgConfig.h" #include "cartTrackDb.h" #include "quickLift.h" +#include "portable.h" #ifdef USE_HAL #include "halBlockViz.h" #endif struct grp *trackHubGrps = NULL; // global with grps loaded from track hubs static struct hash *hubCladeHash; // mapping of clade name to hub pointer static struct hash *hubAssemblyHash; // mapping of assembly name to genome struct static struct hash *hubAssemblyUndecoratedHash; // mapping of undecorated assembly name to genome struct static struct hash *hubOrgHash; // mapping from organism name to hub pointer static struct trackHub *globalAssemblyHubList; // list of trackHubs in the user's cart static struct hash *trackHubHash; static boolean isValidSeqNameChar(char c) /* Return TRUE if c is a valid character for a sequence name: [A-Za-z0-9._-]. */ @@ -1548,39 +1549,39 @@ #endif } else if (startsWithWord("hic", type)) { struct hicMeta *header; char *errString = hicLoadHeader(bigDataUrl, &header, genome->name); if (errString != NULL) errAbort("hic file error: %s", errString); } else errAbort("unrecognized type %s in genome %s track %s", type, genome->name, tdb->track); freez(&bigDataUrl); } } -static void outHubHeader(FILE *f, char *db) +static void outHubHeader(struct dyString *dy, char *db) // output a track hub header { -fprintf(f,"hub quickLiftHub%s\n\ +dyStringPrintf(dy,"hub quickLiftHub%s\n\ shortLabel QuickLift from %s\n\ longLabel QuickLift from %s\n\ useOneFile on\n\ email genome-www@soe.ucsc.edu\n\n", db, db, db); -fprintf(f,"genome %s\n\n", db); +dyStringPrintf(dy,"genome %s\n\n", db); } static char *getHubName(struct cart *cart, char *db) // get the name of the hub to use for quickLifted tracks { struct tempName hubTn; char buffer[4096]; #define quickLiftCartName "hubQuickLift" safef(buffer, sizeof buffer, "%s-%s", quickLiftCartName, db); char *hubName = cartOptionalString(cart, buffer); int fd = -1; // we don't reuse userdata paths since they are in save sessions if ((hubName != NULL) && strstr(hubName, "userdata")) hubName = NULL; @@ -1603,75 +1604,98 @@ char *parent; /* bare parent track name, or NULL */ struct dyString *text; /* full stanza text including final newline */ }; static char *firstWordClone(char *s) /* Return a clone of the first whitespace-delimited word of s, or NULL. */ { s = skipLeadingSpaces(s); if (isEmpty(s)) return NULL; char *sp = skipToSpaces(s); int len = (sp != NULL) ? (sp - s) : (int)strlen(s); return cloneStringZ(s, len); } -boolean quickLiftHubRemoveTrack(struct cart *cart, char *sourceDb, char *trackName) -/* Remove a track stanza from the quickLift hub file for sourceDb, along with - * any descendant stanzas (transitively) whose parent is being removed. - * Returns TRUE if at least one stanza was removed. */ +static struct quickLiftStanza *readStanzas(struct lineFile *lf, struct dyString *header) +/* Read track stanzas out of a quickLift hub file, or out of a string of freshly + * generated stanzas. Everything ahead of the first track line goes into header. + * A container is many stanzas here, one per track line, not a single block. */ { -char buffer[4096]; -safef(buffer, sizeof buffer, "%s-%s", quickLiftCartName, sourceDb); -char *filename = cartOptionalString(cart, buffer); -if (filename == NULL || !isServerUserFilePath(filename)) - return FALSE; - -struct lineFile *lf = lineFileMayOpen(filename, TRUE); -if (lf == NULL) - return FALSE; - -char *bareName = trackHubSkipHubName(trackName); -struct dyString *header = dyStringNew(0); struct quickLiftStanza *stanzaList = NULL; struct quickLiftStanza *cur = NULL; char *line; int lineSize; -/* Pass 1: read the file into a header + list of stanzas, recording each - * stanza's name and (if any) parent. */ while (lineFileNext(lf, &line, &lineSize)) { char *trim = skipLeadingSpaces(line); if (startsWithWord("track", trim)) { AllocVar(cur); cur->text = dyStringNew(0); cur->name = firstWordClone(trim + 5); slAddHead(&stanzaList, cur); } else if (cur != NULL && startsWithWord("parent", trim)) { if (cur->parent == NULL) cur->parent = firstWordClone(trim + 6); } struct dyString *target = (cur != NULL) ? cur->text : header; dyStringAppend(target, line); dyStringAppendC(target, '\n'); } slReverse(&stanzaList); +return stanzaList; +} + +static void freeStanzas(struct quickLiftStanza **pList) +/* Free a list of stanzas. */ +{ +struct quickLiftStanza *s; +for (s = *pList; s != NULL; s = s->next) + { + dyStringFree(&s->text); + freeMem(s->name); + freeMem(s->parent); + } +slFreeList(pList); +} + +boolean quickLiftHubRemoveTrack(struct cart *cart, char *sourceDb, char *trackName) +/* Remove a track stanza from the quickLift hub file for sourceDb, along with + * any descendant stanzas (transitively) whose parent is being removed. + * Returns TRUE if at least one stanza was removed. */ +{ +char buffer[4096]; +safef(buffer, sizeof buffer, "%s-%s", quickLiftCartName, sourceDb); +char *filename = cartOptionalString(cart, buffer); +if (filename == NULL || !isServerUserFilePath(filename)) + return FALSE; + +struct lineFile *lf = lineFileMayOpen(filename, TRUE); +if (lf == NULL) + return FALSE; + +char *bareName = trackHubSkipHubName(trackName); +struct dyString *header = dyStringNew(0); + +/* Pass 1: read the file into a header + list of stanzas, recording each + * stanza's name and (if any) parent. */ +struct quickLiftStanza *stanzaList = readStanzas(lf, header); lineFileClose(&lf); /* Build a removal set: start with the named track, then iterate adding any * stanza whose parent is already in the set, until the set is stable. */ struct hash *removeSet = hashNew(0); hashStore(removeSet, bareName); boolean changed = TRUE; while (changed) { changed = FALSE; struct quickLiftStanza *s; for (s = stanzaList; s != NULL; s = s->next) { if (s->name == NULL || s->parent == NULL) continue; @@ -1696,37 +1720,31 @@ dyStringAppend(out, s->text->string); } if (removedAny) { FILE *f = mustOpen(filename, "w"); chmod(filename, 0666); fputs(header->string, f); fputs(out->string, f); fclose(f); } dyStringFree(&header); dyStringFree(&out); hashFree(&removeSet); -for (s = stanzaList; s != NULL; s = s->next) - { - dyStringFree(&s->text); - freeMem(s->name); - freeMem(s->parent); - } -slFreeList(&stanzaList); +freeStanzas(&stanzaList); return removedAny; } static char *vettedTracks[] = /* tracks that have been tested with quickLift */ { "decipherContainer", "decipherSnvs", "omimLocation", "omimAvSnp", "ncbiRefSeq", "clinvar", "clinvarSubLolly", "pubs", "pubsBlat", @@ -1984,207 +2002,251 @@ // make sure we only output track types that can // be quickLifted. Return true if we any tracks survive { if (tdb->subtracks) { tdb->subtracks = validateTdbChildren(cart, db, tdb->subtracks, badList); if (tdb->subtracks == NULL) return FALSE; return TRUE; } return validateOneTdb(db, tdb, badList); } -static void outTrack(FILE *f, struct cart *cart, struct trackDb *tdb, double priority) +static void outTrack(struct dyString *out, struct cart *cart, struct trackDb *tdb, double priority) /* Set priority and output track to hub. */ { char buffer[1024]; safef(buffer, sizeof buffer, "%g", priority); hashReplace(tdb->settingsHash, "priority", cloneString(buffer)); struct dyString *dy = trackDbString(cart, tdb); -fprintf(f, "%s\n", dy->string); +dyStringPrintf(out, "%s\n", dy->string); +dyStringFree(&dy); } static boolean checkCartVisibility(struct cart *cart, struct trackDb *tdb) { char *cartVis = cartOptionalString(cart, tdb->track); if (cartVis != NULL) tdb->visibility = hTvFromString(cartVis); return (tdb->visibility != tvHide); } static boolean isFromQuickLiftHub(struct trackDb *tdb) /* True if this tdb came from a quickLift hub (already a lifted shadow track). * Such tracks must not be lifted again. */ { return trackDbSetting(tdb, "quickLiftUrl") != NULL || trackDbSetting(tdb, "quickLifted") != NULL; } -static void walkTree(FILE *f, char *db, struct cart *cart, struct trackDb *tdb, struct trackDb **badList, struct hash *existingTracks) +static void walkTree(struct dyString *out, char *db, struct cart *cart, struct trackDb *tdb, struct trackDb **badList) /* walk tree looking for visible tracks to output to hub. Skip tracks that already - * came from a quickLift hub, and skip tracks whose name is already present in - * the existing hub file. */ + * came from a quickLift hub. Every visible track is written, whether or not it is + * already in the hub file: the caller merges these stanzas over the old ones, so a + * track that was lifted before gets its current state rather than the one it had + * when it was first lifted. */ { struct hash *haveSuper = newHash(0); struct trackDb *tdbNext = NULL; // The priority written to the hub is the track's rank in the source list, which the // caller has sorted on group priority and then track priority. The rank has to count -// every track we walk past, not just the ones we output: the hub file is appended to -// across requests, so the number a track gets must depend only on how the source is -// laid out, never on which request it happened to be added in. +// every track we walk past, not just the ones we output: tracks accumulate in the hub +// file across requests, so the number a track gets must depend only on how the source +// is laid out, never on which request it happened to be added in. // // The source priority itself will not do. All lifted tracks land in one group on the // target (trackHubAddGroupName rewrites the group of every hub track), so a priority // that only orders within a source group is being compared across groups. double rank = 0; for(; tdb; tdb = tdbNext) { tdbNext = tdb->next; rank += 1; if (isFromQuickLiftHub(tdb)) continue; - if (existingTracks != NULL && - hashLookup(existingTracks, trackHubSkipHubName(tdb->track)) != NULL) - continue; - boolean isVisible = FALSE; if (tdb->parent == NULL) isVisible = checkCartVisibility(cart, tdb); else if (isParentVisible(cart, tdb) && isSubtrackVisible(cart, tdb)) // child of supertrack { if (hashLookup(haveSuper, tdb->parent->track) == NULL) // output yet? { //if (checkCartVisibility(cart, tdb->parent)) - { - char *bareParent = trackHubSkipHubName(tdb->parent->track); - if (existingTracks == NULL || - hashLookup(existingTracks, bareParent) == NULL) { tdb->parent->visibility = hTvFromString("tvShow"); // a superTrack is not in the list we are walking, so it has no rank // of its own. Slot it just above the first child that brought it in. - outTrack(f, cart, tdb->parent, rank - 0.5); - } + outTrack(out, cart, tdb->parent, rank - 0.5); hashStore(haveSuper, tdb->parent->track); } } isVisible = checkCartVisibility(cart, tdb); } if (isVisible && validateTdb(cart, db, tdb, badList)) { hashRemove(tdb->settingsHash, "superTrack"); // this gets inherited by subTracks(?) // is this a custom track? char *tdbType = trackDbSetting(tdb, "tdbType"); if (tdbType != NULL) { hashReplace(tdb->settingsHash, "type", tdbType); hashReplace(tdb->settingsHash, "shortLabel", trackDbSetting(tdb, "name")); hashReplace(tdb->settingsHash, "longLabel", trackDbSetting(tdb, "description")); } - outTrack(f, cart, tdb, rank); + outTrack(out, cart, tdb, rank); } } } -static void readExistingHubTracks(char *filename, struct hash *trackNames) -/* Scan an existing quickLift hub file and populate trackNames with the set of - * track names already present. */ +static void writeMergedHubFile(char *filename, char *db, struct dyString *newContent) +/* Write the hub file from the stanzas we just generated plus whatever was already + * in the file. A generated stanza replaces the old stanza of the same track, in + * the slot the old one held, so the file keeps parents ahead of their children. + * A track in the file that we did not generate this time is kept as it was, so + * tracks still accumulate across lifts in one session. */ { +struct dyString *header = dyStringNew(0); +struct quickLiftStanza *oldList = NULL; struct lineFile *lf = lineFileMayOpen(filename, TRUE); if (lf != NULL) { - char *line; - while (lineFileNextReal(lf, &line)) + oldList = readStanzas(lf, header); + lineFileClose(&lf); + } + +/* The generated stanzas have no header of their own; scratch collects nothing. */ +struct dyString *scratch = dyStringNew(0); +lf = lineFileOnString("quickLift stanzas", TRUE, cloneString(newContent->string)); +struct quickLiftStanza *newList = readStanzas(lf, scratch); +lineFileClose(&lf); + +struct hash *newByName = newHash(8); +struct quickLiftStanza *s; +for (s = newList; s != NULL; s = s->next) { - if (startsWithWord("track", line)) + if ((s->name != NULL) && (hashLookup(newByName, s->name) == NULL)) + hashAdd(newByName, s->name, s); + } + +struct dyString *out = dyStringNew(0); +if (isEmpty(header->string)) + outHubHeader(out, trackHubSkipHubName(db)); +else + dyStringAppend(out, header->string); + +struct hash *written = newHash(8); +for (s = oldList; s != NULL; s = s->next) { - char *name = skipLeadingSpaces(line + 5); - if (isNotEmpty(name)) - hashStoreName(trackNames, cloneString(firstWordInLine(name))); + struct quickLiftStanza *fresh = (s->name == NULL) ? NULL : hashFindVal(newByName, s->name); + if (fresh == NULL) + dyStringAppend(out, s->text->string); + else if (hashLookup(written, fresh->name) == NULL) + { + dyStringAppend(out, fresh->text->string); + hashStore(written, fresh->name); } } - lineFileClose(&lf); + +for (s = newList; s != NULL; s = s->next) + { + if ((s->name != NULL) && (hashLookup(written, s->name) != NULL)) + continue; + dyStringAppend(out, s->text->string); + if (s->name != NULL) + hashStore(written, s->name); } + +/* Write a temporary file and rename it into place. The target assembly reads this + * same file, and rewriting it in place would show a reader a truncated hub. */ +char tmpName[PATH_LEN]; +safef(tmpName, sizeof tmpName, "%s.tmp", filename); +FILE *f = mustOpen(tmpName, "w"); +fputs(out->string, f); +carefulClose(&f); +chmod(tmpName, 0666); +mustRename(tmpName, filename); + +dyStringFree(&out); +dyStringFree(&header); +dyStringFree(&scratch); +hashFree(&newByName); +hashFree(&written); +freeStanzas(&oldList); +freeStanzas(&newList); } static int cmpPriority(const void *va, const void *vb) /* Compare to sort based on priority; use shortLabel as secondary sort key. */ { const struct trackDb *a = *((struct trackDb **)va); const struct trackDb *b = *((struct trackDb **)vb); float dif = 0; dif = a->groupPriority - b->groupPriority; if (dif == 0) dif = a->priority - b->priority; if (dif < 0) return -1; else if (dif == 0.0) /* secondary sort on label */ return strcasecmp(a->shortLabel, b->shortLabel); else return 1; } char *trackHubBuild(char *db, struct cart *cart, struct trackDb **badList) /* Build a track hub using trackDb and the cart. If a hub file already exists - * for db (i.e. earlier quickLift work in the same session), append new track - * stanzas to it instead of overwriting, and skip tracks that are already in - * the file. */ + * for db (i.e. earlier quickLift work in the same session), merge the new track + * stanzas into it: a track that is being lifted again gets the state it has now, + * and a track that is in the file but is not visible on the source any more is + * left alone. */ { struct trackDb *tdbList, *tdb; struct grp *grpList; cartTrackDbInit(cart, &tdbList, &grpList, FALSE); struct hash *groupHash = newHash(0); struct grp *grp; for(grp = grpList; grp; grp = grp->next) hashAdd(groupHash, grp->name, grp); for(tdb = tdbList; tdb; tdb = tdb->next) { grp = hashFindVal(groupHash, tdb->grp); tdb->groupPriority = grp->priority; } slSort(&tdbList, cmpPriority); char *filename = getHubName(cart, db); -struct hash *existingTracks = newHash(8); -readExistingHubTracks(filename, existingTracks); -boolean hubExists = (hashNumEntries(existingTracks) > 0); - -FILE *f = mustOpen(filename, hubExists ? "a" : "w"); -chmod(filename, 0666); -if (!hubExists) - outHubHeader(f, trackHubSkipHubName(db)); - -walkTree(f, db, cart, tdbList, badList, existingTracks); -fclose(f); +struct dyString *newContent = dyStringNew(0); +walkTree(newContent, db, cart, tdbList, badList); +writeMergedHubFile(filename, db, newContent); +dyStringFree(&newContent); return cloneString(filename); } struct grp *trackHubGetGrps() /* Get the groups defined by attached track hubs. */ { return trackHubGrps; } struct trackDb *trackHubAddTracksGenome(struct trackHubGenome *hubGenome) /* Load up stuff from data hub and return list. */ { /* Load trackDb.ra file and make it into proper trackDb tree */ struct trackDb *tdbList = NULL;