0afbe51b59a42ea8fd8e008b35dcb49159e8fcc6 hiram Tue Aug 18 13:48:14 2026 -0700 efficiency for hubClone to avoid reloading files repeatedly and fix error in reading include statements with trailing # comments refs #38147 diff --git src/hg/utils/hubClone/hubClone.c src/hg/utils/hubClone/hubClone.c index b69b07614ae..5fdfbdb4b62 100644 --- src/hg/utils/hubClone/hubClone.c +++ src/hg/utils/hubClone/hubClone.c @@ -10,31 +10,33 @@ #include "errCatch.h" #include "ra.h" #include "hui.h" #include "pipeline.h" void usage() /* Explain usage and exit. */ { errAbort( "hubClone - Clone the remote hub text files to a local copy in newDirectoryName, fixing up bigDataUrls to remote location if necessary\n" "usage:\n" " hubClone http://url/to/hub.txt\n" "options:\n" " -udcDir=/dir/to/udcCache Path to udc directory\n" " -download Download data files in addition to the hub configuration files\n" - " -skipMissingAssemblies Skip assemblies whose trackDb.txt files are missing instead of aborting\n" + " -skipMissingAssemblies Skip assemblies whose trackDb.txt files are missing instead of\n" + " aborting; also skips (with a warning) any include file that\n" + " can't be opened, instead of aborting the whole clone\n" ); } /* Command line validation table. */ static struct optionSpec options[] = { {"udcDir", OPTION_STRING}, {"download", OPTION_BOOLEAN}, {"skipMissingAssemblies", OPTION_BOOLEAN}, {NULL, 0}, }; /* Simple structure to hold genome info when doing manual parsing */ struct simpleGenome { struct simpleGenome *next; @@ -146,95 +148,125 @@ if (isNotEmpty(downloadDir)) { dyStringClear(fname); char *relName = strrchr(hel->val,'/'); if (relName != NULL) { relName = relName + 1; dyStringPrintf(fname, "%s%s", downloadDir, relName); } else { relName = hel->val; dyStringPrintf(fname, "%s%s", downloadDir, (char *)hel->val); } fprintf(out, "%s %s\n", hel->name, relName); + + // don't re-download big data files that are already present from + // a previous clone of this hub (like "wget -nc", but done ourselves + // since -nc is not honored by wget when -O is also given) + if (fileExists(dyStringContents(fname))) + { + verbose(1, "skipping already-downloaded file: %s\n", dyStringContents(fname)); + } + else + { char *cmd[] = {"wget", "-q", "-O", dyStringContents(fname), urlToData, NULL}; // use pipelineNoAbort so the loop continues if a url is typo'd or something, // but still warn the user struct pipeline *pl = pipelineOpen1(cmd, pipelineWrite | pipelineNoAbort, "/dev/null", NULL, 0); int ret = pipelineWait(pl); if (ret != 0) { warn("wget failed for url: %s", urlToData); } } + } else fprintf(out, "%s %s\n", hel->name, urlToData); } else fprintf(out, "%s %s\n", hel->name, (char *)hel->val); } } fprintf(out, "\n"); hashElFreeList(&helList); } void printGenericStanza(struct hash *stanza, FILE *out, char *baseUrl) /* print a hash to out */ { struct hashEl *hel, *helList = hashElListHash(stanza); for (hel = helList; hel != NULL; hel = hel->next) { fprintf(out, "%s %s\n", hel->name, (char *)hel->val); } fprintf(out,"\n"); } -void printOneFile(char *url, FILE *f, boolean oneFile, char *downloadDir) -/* printOneFile: pass a stanza to appropriate printer */ +void printOneFile(char *url, FILE *f, boolean oneFile, char *downloadDir, boolean skipErrors) +/* printOneFile: pass a stanza to appropriate printer. If skipErrors is set, an include + * file that can't be opened (missing, typo'd URL, etc.) is skipped with a warning instead + * of aborting the whole clone -- useful for pushing on through a very large hub. */ { struct lineFile *lf; struct hash *stanza; struct hashEl *includeFile; lf = udcWrapShortLineFile(url, NULL, MAX_HUB_TRACKDB_FILE_SIZE); while ((stanza = raNextRecord(lf)) != NULL) { if (hashLookup(stanza, "hub")) { printHubStanza(stanza, f, url); } else if (hashLookup(stanza, "genome")) { printGenomeStanza(stanza, f, url, oneFile); } else if (hashLookup(stanza, "track")) { printTrackDbStanza(stanza, f, url, downloadDir); } else { // if there's an include file then open and print the include file includeFile = hashLookup(stanza, "include"); if (includeFile != NULL) { for(; includeFile; includeFile = includeFile->next) { - char *newUrl = trackHubRelativeUrl(url, includeFile->val); - printOneFile(newUrl, f, oneFile, downloadDir); + // ra format only treats a whole line starting with '#' as a comment, + // so a trailing "include foo.txt # some comment" line would otherwise + // pass the comment through as part of the filename; keep just the + // first word + char *incFileName = cloneString(includeFile->val); + firstWordInLine(incFileName); + char *newUrl = trackHubRelativeUrl(url, incFileName); + if (skipErrors) + { + struct errCatch *errCatch = errCatchNew(); + if (errCatchStart(errCatch)) + printOneFile(newUrl, f, oneFile, downloadDir, skipErrors); + errCatchEnd(errCatch); + if (errCatch->gotError) + warn("skipping include file %s: %s", newUrl, errCatch->message->string); + errCatchFree(&errCatch); + } + else + printOneFile(newUrl, f, oneFile, downloadDir, skipErrors); } } else printGenericStanza(stanza, f, url); } } lineFileClose(&lf); freeHash(&stanza); } struct trackHub *readHubFromUrl(char *hubUrl) /* readHubUrl: errCatch around trackHubOpen */ { struct trackHub *hub = NULL; struct errCatch *errCatch = errCatchNew(); @@ -249,36 +281,37 @@ FILE *createPathAndFile(char *path) /* if path contains parent directories that don't exist, create them first before opening file */ { char *copy = cloneString(path); if (stringIn("/", copy)) { chopSuffixAt(copy, '/'); makeDirs(copy); // now make the real file return mustOpen(path, "w"); } return mustOpen(path, "w"); } -void createWriteAndCloseFile(char *fileName, char *url, boolean useOneFile, char *downloadDir) +void createWriteAndCloseFile(char *fileName, char *url, boolean useOneFile, char *downloadDir, + boolean skipErrors) /* Wrapper around a couple lines */ { FILE *f; f = createPathAndFile(fileName); -printOneFile(url, f, useOneFile, downloadDir); +printOneFile(url, f, useOneFile, downloadDir, skipErrors); carefulClose(&f); } boolean canAccessUrl(char *url) /* Check if a URL can be accessed by attempting to open it */ { struct errCatch *errCatch = errCatchNew(); boolean canAccess = FALSE; if (errCatchStart(errCatch)) { struct lineFile *lf = udcWrapShortLineFile(url, NULL, MAX_HUB_TRACKDB_FILE_SIZE); if (lf != NULL) { canAccess = TRUE; @@ -424,47 +457,47 @@ char *genomesFile = parseHubTxtForGenomesFile(hubUrl, &hubName, &oneFile); polishHubName(hubName); if (oneFile) { // For useOneFile hubs, we still need to try trackHubOpen since everything is in one file // Fall through to standard processing hub = readHubFromUrl(hubUrl); if (hub == NULL) errAbort("error opening %s", hubUrl); makeDirs(hubName); path = catTwoStrings(hubName, catTwoStrings("/", hubFileName)); f = mustOpen(path, "w"); if (download) dyStringPrintf(downloadDir, "%s/", hubName); - printOneFile(hubUrl, f, oneFile, dyStringContents(downloadDir)); + printOneFile(hubUrl, f, oneFile, dyStringContents(downloadDir), skipMissingAssemblies); carefulClose(&f); return; } if (genomesFile == NULL) errAbort("No genomesFile found in hub.txt"); genomesUrl = trackHubRelativeUrl(hubUrl, genomesFile); struct simpleGenome *genomeList = parseGenomesTxt(genomesUrl); if (genomeList == NULL) errAbort("No genomes found in %s", genomesUrl); // Write hub.txt path = catTwoStrings(hubName, catTwoStrings("/", hubFileName)); - createWriteAndCloseFile(path, hubUrl, FALSE, dyStringContents(downloadDir)); + createWriteAndCloseFile(path, hubUrl, FALSE, dyStringContents(downloadDir), skipMissingAssemblies); // Track which genomes to skip struct hash *skipGenomes = hashNew(0); // Process each genome, checking accessibility genomesFileName = catTwoStrings(hubName, catTwoStrings("/", genomesFile)); char *genomePath = cloneString(genomesFileName); chopSuffixAt(genomePath, '/'); struct simpleGenome *sg; for (sg = genomeList; sg != NULL; sg = sg->next) { char *trackDbUrl = trackHubRelativeUrl(genomesUrl, sg->trackDbPath); char *genomeName = sg->name; @@ -481,92 +514,95 @@ } // Make correct directory structure and write trackDb genomesDir = catTwoStrings(genomePath, catTwoStrings("/", genomeName)); if (download) { dyStringClear(downloadDir); dyStringPrintf(downloadDir, "%s/%s/", hubName, genomeName); } tdbFileName = strrchr(sg->trackDbPath, '/'); if (tdbFileName != NULL) tdbFileName += 1; else tdbFileName = sg->trackDbPath; tdbFilePath = catTwoStrings(genomesDir, catTwoStrings("/", tdbFileName)); - createWriteAndCloseFile(tdbFilePath, trackDbUrl, FALSE, dyStringContents(downloadDir)); + createWriteAndCloseFile(tdbFilePath, trackDbUrl, FALSE, dyStringContents(downloadDir), + skipMissingAssemblies); } // Write genomes.txt, filtering out skipped genomes printGenomesTxtFiltered(genomesUrl, genomesFileName, skipGenomes); hashFree(&skipGenomes); return; } // Standard mode: use trackHubOpen hub = readHubFromUrl(hubUrl); if (hub == NULL) errAbort("error opening %s", hubUrl); hubName = cloneString((char *)hashFindVal(hub->settings, "hub")); polishHubName(hubName); if (trackHubSetting(hub, "useOneFile")) { oneFile = TRUE; makeDirs(hubName); path = catTwoStrings(hubName, catTwoStrings("/", hubFileName)); f = mustOpen(path, "w"); if (download) { dyStringPrintf(downloadDir, "%s/", hubName); } - printOneFile(hubUrl, f, oneFile, dyStringContents(downloadDir)); + printOneFile(hubUrl, f, oneFile, dyStringContents(downloadDir), skipMissingAssemblies); carefulClose(&f); } else { genome = hub->genomeList; if (genome == NULL) errAbort("error opening %s file", hub->genomesFile); path = catTwoStrings(hubName, catTwoStrings("/", hubFileName)); - createWriteAndCloseFile(path, hubUrl, oneFile, dyStringContents(downloadDir)); + createWriteAndCloseFile(path, hubUrl, oneFile, dyStringContents(downloadDir), skipMissingAssemblies); genomesUrl = trackHubRelativeUrl(hub->url, hub->genomesFile); genomesFileName = catTwoStrings(hubName, catTwoStrings("/", hub->genomesFile)); char *genomePath = cloneString(genomesFileName); chopSuffixAt(genomePath, '/'); // used later for making the right directory structure - createWriteAndCloseFile(genomesFileName, genomesUrl, oneFile, dyStringContents(downloadDir)); + createWriteAndCloseFile(genomesFileName, genomesUrl, oneFile, dyStringContents(downloadDir), + skipMissingAssemblies); for (; genome != NULL; genome = genome->next) { if (startsWith("_", genome->name)) // assembly hubs have a leading '_' genome->name += 1; // make correct directory strucutre genomesDir = catTwoStrings(genomePath, catTwoStrings("/", genome->name)); if (download) { dyStringClear(downloadDir); dyStringPrintf(downloadDir, "%s/%s/", hubName, genome->name); } tdbFileName = strrchr(genome->trackDbFile, '/') + 1; tdbFilePath = catTwoStrings(genomesDir, catTwoStrings("/", tdbFileName)); - createWriteAndCloseFile(tdbFilePath, genome->trackDbFile, oneFile, dyStringContents(downloadDir)); + createWriteAndCloseFile(tdbFilePath, genome->trackDbFile, oneFile, dyStringContents(downloadDir), + skipMissingAssemblies); } } } int main(int argc, char *argv[]) /* Process command line. */ { optionInit(&argc, argv, options); if (argc < 2) usage(); setUdcCacheDir(); udcSetDefaultDir(optionVal("udcDir", udcDefaultDir())); hubClone(argv[1], optionExists("download"), optionExists("skipMissingAssemblies")); return 0; }