7ac21bf1c3ef2b54beec22d23f6a6d389e229d92 galt Thu Feb 21 02:25:27 2019 -0800 fixing it to handle 0 custom tracks to backup by skipping the backup button. Fixed the show to match the make and correctly deal with both complete custom-trash-table types as well as newer bigDataUrl cts. Also fixed a bug where the origTrackLine is missing from CTs generated by Table Browser. Currently just try to detect the missing value, and if tdbType is wig then feed in a simple track line for generic wiggles. diff --git src/hg/hgSession/backup.c src/hg/hgSession/backup.c index ea89aaf..7564b6b 100644 --- src/hg/hgSession/backup.c +++ src/hg/hgSession/backup.c @@ -377,41 +377,73 @@ } return result; } /* hold extra data that does not fit in the track struct itself, * but which is track-specific */ struct ctExtra { struct ctExtra *next; char *name; // important to identity of the track in hgCustom char *browserLines; char *trackLine; int tableRows; unsigned long tableDataLength; + char *bigDataUrl; }; struct downloadResults { struct downloadResults *next; char *db; char *ctPath; // path to ct file in trash struct customTrack *cts; // tracks struct ctExtra *ctExtras; // parallel additional info to the tracks }; + +void addFieldToTrackLine(struct dyString *dy, struct customTrack *track, char *field, boolean isRequired) +/* add optional or required field and its value to a track line */ +{ +char *val = hashFindVal(track->tdb->settingsHash, field); +if (val) + { + dyStringPrintf(dy, " %s='%s'", field, val); + } +else + { + if (isRequired) + errAbort("required field %s is missing from custom track settings", field); + } +} + +char *fabricateWigTrackline(struct customTrack *track) +/* make up a missing trackline for a ct exported from Table Browser as a wig */ +{ +struct dyString *dy = dyStringNew(1024); +dyStringPrintf(dy, "track "); +addFieldToTrackLine(dy, track,"name", TRUE); // Required +dyStringPrintf(dy, " type='wiggle_0'"); +addFieldToTrackLine(dy, track,"description", FALSE); +addFieldToTrackLine(dy, track,"visibility", FALSE); +addFieldToTrackLine(dy, track,"priority", FALSE); +addFieldToTrackLine(dy, track,"altColor", FALSE); +return dyStringCannibalize(&dy); +} + + struct downloadResults *processCtsForDownloadInternals(char *contents, char **pTrackHubsVar) /* Process the saved session cart contents, * looking for custom-tracks which are db-specific. */ { struct downloadResults *resultList = NULL; if (!contents) return resultList; char *contentsToChop = cloneString(contents); char *namePt = contentsToChop; struct sqlConnection *ctConn = hAllocConn(CUSTOM_TRASH); @@ -456,54 +488,68 @@ // what to save in ct output // name is important because hgCustom identifies by this string char *name = hashFindVal(track->tdb->settingsHash, "name"); extra->name = cloneString(name); char *browserLines = hashFindVal(track->tdb->settingsHash, "browserLines"); // to handle multiple-lines replace semi-colon ";" with newline. if (browserLines) replaceChar(browserLines,';','\n'); extra->browserLines = cloneString(browserLines); char *origTrackLine = hashFindVal(track->tdb->settingsHash, "origTrackLine"); extra->trackLine = cloneString(origTrackLine); + if (!extra->trackLine) // Table Browser creates some wiggles without an origTrackLine + { + char *tdbType = hashFindVal(track->tdb->settingsHash, "tdbType"); + if (tdbType && startsWith("wig ", tdbType)) + { + extra->trackLine = fabricateWigTrackline(track); + } + } // is it weird that the loader customFactoryTestExistence() did not do this for me? char *wibFilePath = hashFindVal(track->tdb->settingsHash, "wibFile"); if (wibFilePath && fileExists(wibFilePath)) { track->wibFile = wibFilePath; } char *mafFilePath = hashFindVal(track->tdb->settingsHash, "mafFile"); if (mafFilePath && fileExists(mafFilePath)) { track->wibFile = mafFilePath; } // old vcf type (not vcfTabix) char *vcfFilePath = hashFindVal(track->tdb->settingsHash, "vcfFile"); if (vcfFilePath && fileExists(vcfFilePath)) { track->wibFile = vcfFilePath; } if (track->dbTrack && track->dbDataLoad && track->dbTableName) { extra->tableRows = sqlTableSizeIfExists(ctConn, track->dbTableName); extra->tableDataLength = sqlTableDataSizeFromSchema(ctConn, CUSTOM_TRASH, track->dbTableName); } + + + char *bigDataUrl = hashFindVal(track->tdb->settingsHash, "bigDataUrl"); + if (bigDataUrl) + extra->bigDataUrl = bigDataUrl; + slAddHead(&result->ctExtras,extra); } slReverse(&result->ctExtras); } slAddHead(&resultList,result); } else if (sameString("trackHubs", namePt)) { cgiDecode(dataPt, dataPt, strlen(dataPt)); if (pTrackHubsVar) *pTrackHubsVar = cloneString(dataPt); } @@ -522,78 +568,88 @@ * looking for custom-tracks which are db-specific. */ { struct downloadResults *resultList = processCtsForDownloadInternals(contents, NULL); printf("<H2>Custom Tracks</H2>"); printf("You can backup the custom tracks which you previously uploaded to UCSC Genome Browser servers.<br>" "It will create a single .tar.gz file with custom track data,<br>" "which you can then download to your own machine for use as a backup.<br>" "\n"); struct downloadResults *result = NULL; -int nonBigDataUrlCount = 0; int ctCount = 0; long totalDataToDownload = 0; for (result=resultList; result; result=result->next) { if (!result->ctPath) // ct.bed was missing for the db continue; printf("<h3>Database %s</h3>\n", result->db); + //printf("result->ctPath %s <br>\n", result->ctPath); // DEBUG REMOVE + if (startsWith("hub_", result->db)) { unsigned hubId = isLiveHub(result->db, NULL, TRUE); if (hubId == 0) { printf("Skipping HUB Database %s. " "This hub db is not currently available and will not get backed up.<BR><BR>\n", result->db); continue; // skip db if not active. } } else { if (!isActiveDb(result->db)) { printf("Skipping Database %s. " "The db is not currently active and will not get backed up.<BR><BR>\n", result->db); continue; // skip db if not active. } } struct customTrack *cts = result->cts, *track = NULL; struct ctExtra *extras = result->ctExtras, *extra = NULL; printf("<table>\n"); for (track=cts,extra=extras; track; track=track->next,extra=extra->next) { + //printf("track name %s <br>\n", extra->name); // DEBUG REMOVE + //printf("track->dbTrack %d <br>\n", track->dbTrack); // DEBUG REMOVE + //printf("track->dbTableName %s <br>\n", track->dbTableName); // DEBUG REMOVE + //printf("track->dbTrackType %s <br>\n", track->dbTrackType); // DEBUG REMOVE + //printf("track->dbDataLoad %d <br>\n", track->dbDataLoad); // DEBUG REMOVE + //printf("extra->bigDataUrl %s <br>\n", extra->bigDataUrl); // DEBUG REMOVE + + boolean wibMissing = track->wibFile && !fileExists(track->wibFile); + + if (extra->bigDataUrl || (track->dbTrack && track->dbDataLoad && track->dbTableName && !wibMissing)) + { long trackDataToDownload = 0; ++ctCount; - // handle user-friendly sizes GB MB KB and B - if (track->dbTrack && track->dbDataLoad && track->dbTableName) + if (!extra->bigDataUrl) { char greek[32]; - ++nonBigDataUrlCount; sprintWithGreekByte(greek, sizeof(greek), extra->tableDataLength); trackDataToDownload += extra->tableDataLength; // handle wiggle cts which have an additional wig binary // wibFile=../trash/ct/hgtct_genome_542_dc1750.wib // wibFile=../trash/ct/ct_hgwdev_galt_e83f_3892d0.maf // wibFIle=../trash/ct/ct_hgwdev_galt_4ba3_415cc0.vcf if (track->wibFile && fileExists(track->wibFile)) { long wibFileSize = fileSize(track->wibFile); char greek[32]; sprintWithGreekByte(greek, sizeof(greek), wibFileSize); trackDataToDownload += wibFileSize; } } @@ -603,43 +659,47 @@ { long htmlFileSize = fileSize(track->htmlFile); char greek[32]; sprintWithGreekByte(greek, sizeof(greek), htmlFileSize); trackDataToDownload += htmlFileSize; } char greek[32]; sprintWithGreekByte(greek, sizeof(greek), trackDataToDownload); if (trackDataToDownload == 0) // suppress 0.0 B greek[0] = 0; // empty string printf("<tr><td>%s</td><td>%s</td></tr>\n", extra->name, greek); // track name totalDataToDownload += trackDataToDownload; } + } printf("</table>\n"); } printf("<br>\n"); printf("%d custom tracks found.<br>\n", ctCount); char greek[32]; sprintWithGreekByte(greek, sizeof(greek), totalDataToDownload); printf("Total custom track data to backup: %s ", greek); +if (ctCount > 0) + { cgiMakeButton(hgsMakeDownloadPrefix, "create custom tracks backup archive"); + } printf("<br>\n"); printf("<br>\n"); } void showDownloadSessionCtData(struct hashEl *downloadList) /* Show download page for the given session */ { char query[512]; char **row = NULL; struct sqlResult *sr = NULL; puts("Content-Type:text/html\n"); @@ -879,30 +939,31 @@ while(TRUE) { randPath=makeRandomKey(128+33); // at least 128 bits of protection, 33 for the world population size. // Avoid the possibility of somehow doing more than one instance of the session // backup at the same time. // On the extremely rare chance that the directory already exists, // just call the function again. safef(tempOutRand, sizeof tempOutRand, "%s/%s", tempOut, randPath); if (makeDir(tempOutRand)) break; ++randCount; if (randCount > 100) // should never happen errAbort("unable to create random output dir."); } +int ctCount = 0; int foundCount = 0; if ((row = sqlNextRow(sr)) != NULL) { ++foundCount; char *contents = row[0]; char *trackHubsVar = NULL; struct downloadResults *resultList = processCtsForDownloadInternals(contents, &trackHubsVar); struct downloadResults *result = NULL; for (result=resultList; result; result=result->next) @@ -935,71 +996,89 @@ else { if (!isActiveDb(result->db)) { continue; // skip db if not active. } } printf("<h3>%s</h3>\n", result->db); dyStringPrintf(dyProg,"<h3>%s</h3>\n", result->db); updateProgessFile(backgroundProgress, dyProg); lazarusLives(20 * 60); + //printf("result->ctPath %s <br>\n", result->ctPath); // DEBUG REMOVE + + if (startsWith("hub_", result->db)) { // Save the hubUrl now in case it gets restored on another machine // that does NOT have that hub loaded on it. // save output as hubUrl text file char hubUrlPath[1024]; safef(hubUrlPath, sizeof hubUrlPath, "%s/hubUrl", outDbDir); FILE *f = mustOpen(hubUrlPath, "w"); fprintf(f, "%s", hubUrl); // NO NEWLINE carefulClose(&f); } char cwd[PATH_LEN]; if (!getcwd(cwd, sizeof(cwd))) errnoAbort("unable to get current dir."); for (track=cts,extra=extras; track; track=track->next,extra=extra->next) { + //printf("track name %s <br>\n", extra->name); // DEBUG REMOVE + //printf("track->dbTrack %d <br>\n", track->dbTrack); // DEBUG REMOVE + //printf("track->dbTableName %s <br>\n", track->dbTableName); // DEBUG REMOVE + //printf("track->dbTrackType %s <br>\n", track->dbTrackType); // DEBUG REMOVE + //printf("track->dbDataLoad %d <br>\n", track->dbDataLoad); // DEBUG REMOVE + //printf("extra->bigDataUrl %s <br>\n", extra->bigDataUrl); // DEBUG REMOVE + + boolean wibMissing = track->wibFile && !fileExists(track->wibFile); + + if (extra->bigDataUrl || (track->dbTrack && track->dbDataLoad && track->dbTableName && !wibMissing)) + { + + ++ctCount; printf("%s <br>\n", extra->name); dyStringPrintf(dyProg, "%s <br>\n", extra->name); updateProgessFile(backgroundProgress, dyProg); lazarusLives(20 * 60); + char outNameCt[2014]; safef(outNameCt, sizeof outNameCt, "%s/%s.ct", outDbDir, extra->name); FILE *fct = mustOpen(outNameCt, "w"); // write the track header if (extra->browserLines) fprintf(fct, "%s", extra->browserLines); // should have ; converted \n already - fprintf(fct, "%s\n", extra->trackLine); - boolean wibMissing = track->wibFile && !fileExists(track->wibFile); + if (!extra->trackLine) + errAbort("origTrackLine is NULL!"); + fprintf(fct, "%s\n", extra->trackLine); - if (track->dbTrack && track->dbDataLoad && track->dbTableName && !wibMissing) - { //printf("%s <br>\n", track->wibFile); // DEBUG REMOVE + if (!extra->bigDataUrl) + { // handle wiggle cts which have an additional wig binary // wibFile='../trash/ct/hgtct_genome_542_dc1750.wib' // wibFile='../trash/ct/ct_hgwdev_galt_e83f_3892d0.maf' // wibFile='../trash/ct/ct_hgwdev_galt_4ba3_415cc0.vcf' // symlink for speed, file should not change if (track->wibFile) { if (endsWith(track->wibFile, ".wib")) { // dump wig as ascii char outNameWig[2014]; safef(outNameWig, sizeof outNameWig, "%s/%s.wig", outDbDir, extra->name); doOutWigData(track->dbTableName, CUSTOM_TRASH, outNameWig); // no easy way to append it to .ct directly // append text to ct @@ -1010,49 +1089,54 @@ { // append text to ct appendTrashFileToCt(fct, track->wibFile); } if (endsWith(track->wibFile, ".vcf")) { // append text to ct appendTrashFileToCt(fct, track->wibFile); } } else // simple BED-like { saveSqlDataForTable(track->dbTableName, fct); } + } // handle extra htmlFile track doc // htmlFile=../trash/ct/ct_hgwdev_galt_d011_383c50.html // symlink for speed, file should not change if (track->htmlFile && fileExists(track->htmlFile)) { makeTrashFileLink(track->htmlFile, cwd, outDbDir, extra->name); } - } carefulClose(&fct); } + + } } } sqlFreeResult(&sr); if (foundCount == 0) errAbort("No session found for hgsid=%u", cartSessionRawId(cart)); +if (ctCount == 0) + errAbort("No custom tracks found for hgsid=%u", cartSessionRawId(cart)); + char archiveName[1024]; safef(archiveName, sizeof archiveName, "savedSessionCtRaw.tar.gz"); dyStringPrintf(dyProg, "<br>\n"); int saveDySize = dyProg->stringSize; dyStringPrintf(dyProg, "creating and compressing archive %s <br>\n", archiveName); updateProgessFile(backgroundProgress, dyProg); lazarusLives(20 * 60); // create the archive char cmd[2048]; safef(cmd, sizeof cmd, "cd %s; tar -cpzhf %s *", tempOutRand, archiveName); mustSystem(cmd); dyProg->stringSize = saveDySize; // restore prev size, popping.