ff3dcac9269104df22c3b06fe44556f71bcae0c3 gperez2 Thu Jun 20 17:30:01 2024 -0700 Switching the Advanced track search page, View chromosomes page, hgConvert, hgCustom, hgGateway, hgLiftOver, hgPcr, hgSession, hgTracks, and hgCustom labels/buttons to sentence case, refs #25787 diff --git src/hg/hgSession/backup.c src/hg/hgSession/backup.c index dac8813..9f6f1ea 100644 --- src/hg/hgSession/backup.c +++ src/hg/hgSession/backup.c @@ -1,1264 +1,1264 @@ /* backupRestore - Routines related to backing up and restoring the current cart and its custom tracks. */ /* Copyright (C) 2014 The Regents of the University of California * See kent/LICENSE or http://genome.ucsc.edu/license/ for licensing information. */ #include "common.h" #include "cart.h" #include "net.h" #include "textOut.h" #include "base64.h" #include "md5.h" #include "obscure.h" #include "net.h" #include "hgConfig.h" #include "htmshell.h" #include "cheapcgi.h" #include "hgSession.h" #include <sys/wait.h> #include "hCommon.h" #include "hdb.h" #include "trackHub.h" #include "hubConnect.h" #include "customFactory.h" #include "jsHelper.h" #include "trashDir.h" #include "filePath.h" #include "cgiApoptosis.h" #include "wiggle.h" #include "pipeline.h" #define UCSC_GB_BACKUP_VERSION_FILENAME "UCSC_GB_BACKUP_VERSION" static void errAbortHandler(char *format, va_list args) { // provide more explicit message when we run out of memory (#5147). if(strstr(format, "needLargeMem:") || strstr(format, "carefulAlloc:")) htmlVaWarn("Region selected is too large for calculation. Please specify a smaller region or try limiting to fewer data points.", args); else { // call previous handler popWarnHandler(); vaWarn(format, args); } if(isErrAbortInProgress()) noWarnAbort(); } static void vaHtmlOpen(char *format, va_list args) /* Start up a page that will be in html format. */ { puts("Content-Type:text/html\n"); cartVaWebStart(cart, database, format, args); pushWarnHandler(errAbortHandler); } void htmlOpen(char *format, ...) /* Start up a page that will be in html format. */ { va_list args; va_start(args, format); vaHtmlOpen(format, args); va_end(args); } void htmlClose() /* Close down html format page. */ { popWarnHandler(); cartWebEnd(); } // ------------------------------------------ boolean waitForBackgroundFile(char *path) /* Wait 30 seconds for the file at path to appear. Return TRUE if timeout exceeded. */ { int waited = 0; int waitLimit = 30; // max wait in seconds // sometimes the background process is a little slow, // we can wait up to 30 seconds for it. while (!(fileExists(path) && fileSize(path) > 0) && (waited < waitLimit)) { sleep(1); ++waited; } if (waited >= waitLimit) { htmlOpen("Background Status"); errAbort("Background file %s not found or empty.", path); htmlClose(); return TRUE; } return FALSE; } void getBackgroundStatus(char *url) /* fetch status as the latest complete html block available. * fetch progress info instead if background proc still running. */ { char *html = NULL; // get the pid and see if it is still running char pidName[1024]; safef(pidName, sizeof pidName, "%s.pid", url); if (waitForBackgroundFile(pidName)) return; FILE *pidF = mustOpen(pidName, "r"); long pid = 0; fscanf(pidF, "%ld", &pid); carefulClose(&pidF); // If the background process is still running, grab the progress file instead if (getpgid(pid) >= 0) { htmlOpen("Background Status"); char progressName[1024]; safef(progressName, sizeof progressName, "%s.progress", url); if (waitForBackgroundFile(progressName)) return; if (fileExists(progressName)) { readInGulp(progressName, &html, NULL); printf("%s",html); } else { printf("progress file missing<br>\n"); } printf("<BR>\n<FORM ACTION=\"/cgi-bin/hgTables\" METHOD=GET>\n" //"<INPUT TYPE=SUBMIT NAME=\"%s\" VALUE=\"Back\" >" "<INPUT TYPE=SUBMIT NAME=\"Refresh\" id='Refresh' VALUE=\"Refresh\">" "</FORM>\n" ); jsOnEventById("click", "Refresh", "window.location=window.location;return false;"); jsInline("setTimeout(function(){location = location;},5000);\n"); jsInline("window.scrollTo(0,document.body.scrollHeight);"); htmlClose(); return; } // otherwise read the main html file // sometimes the background process is a little slow, if (waitForBackgroundFile(url)) return; readInGulp(url, &html, NULL); int numLines = chopString(html, "\n", NULL, 1000000); char **lines = NULL; AllocArray(lines, numLines); chopString(html, "\n", lines, numLines); int end; for (end=numLines-1; end >= 0 && ! (endsWith(lines[end], "</html>") || endsWith(lines[end], "</HTML>")) ; --end) /* do nothing */ ; if (end < 0) { htmlOpen("Background Status"); errAbort("No complete html found"); htmlClose(); return; } int start; for (start=end; start >= 0 && ! (startsWith("<html>", lines[start]) || startsWith("<HTML>", lines[start])) ; --start) /* do nothing */ ; if (start < 0) { htmlOpen("Background Status"); errAbort("No html start tag found"); htmlClose(); return; } puts("Content-Type: text/html\n"); int line; boolean autoRefreshFound = FALSE; boolean successfullyUploaded = FALSE; for (line=start; line <= end; line++) { puts(lines[line]); if (startsWith("setTimeout(function(){location = location;}", lines[line])) autoRefreshFound = TRUE; if (startsWith("Output has been successfully uploaded", lines[line])) successfullyUploaded = TRUE; } // if it looks like the background is no longer running, // include the .err stdout output for more informative problem message char urlErr[512]; char *textErr = NULL; safef(urlErr, sizeof urlErr, "%s.err", url); if (!autoRefreshFound && !successfullyUploaded && (fileSize(urlErr) > 0)) { readInGulp(urlErr, &textErr, NULL); //printf("%s", textErr); /* more annoying than helpful in some places. */ } } void startBackgroundWork(char *exec, char **pWorkUrl) /* deal with forking off child for background work * and setting up the trash file for communicating * from the child to the browser */ { char *workUrl = NULL; char hgsid[64]; struct tempName tn; safef(hgsid, sizeof(hgsid), "%s", cartSessionId(cart)); trashDirFile(&tn, "backGround", hgsid, ".tmp"); workUrl = cloneString(tn.forCgi); fflush(stdout); fflush(stderr); // seems that we need to use the double-fork trick // to create enough separation between the non-waiting parent // and the grand-child process. otherwise the OS and Apache are waiting on the child. int pid = fork(); if (pid == -1) { errAbort("can't fork, error %d", errno); } if (pid == 0) // child { int pid2 = fork(); if (pid2 == -1) { errAbort("can't fork, error %d", errno); } if (pid2 == 0) // grand child { // we need to close or redup to open stdout, stderr, stdin // in order for apache to break ties with it. // Will the grandchild cgi still be able to function? // redirect stdout of child to the trash file for easier use of // library functions that output html to stdout. int out = mustOpenFd(tn.forCgi, O_WRONLY | O_CREAT); fflush(stdout); dup2(out,STDOUT_FILENO); /* closes STDOUT before setting it back to saved descriptor */ close(out); // Unfortunately we must create our own stderr log file char errName[1024]; safef(errName, sizeof errName, "%s.err", tn.forCgi); int err = mustOpenFd(errName, O_CREAT | O_WRONLY | O_APPEND); dup2(err, STDERR_FILENO); close(err); // stdin input is just empty int in = mustOpenFd("/dev/null", O_RDONLY); dup2(in, STDIN_FILENO); close(in); // save own pid in .pid file long selfPid = (long) getpid(); char pidName[1024]; safef(pidName, sizeof pidName, "%s.pid", tn.forCgi); char pidBuf[64]; safef(pidBuf, sizeof pidBuf, "%ld", selfPid); int pidFd = mustOpenFd(pidName, O_CREAT | O_WRONLY | O_APPEND); mustWriteFd(pidFd, pidBuf, strlen(pidBuf)); close(pidFd); // progress is a separate channel for progress update char progressName[1024]; safef(progressName, sizeof progressName, "%s.progress", tn.forCgi); // execute so that we will be able to use database and other operations normally. char execPath[4096]; safef(execPath, sizeof execPath, "%s hgsid=%s backgroundProgress=%s", exec, hgsid, progressName); char *args[10]; int numArgs = chopString(execPath, " ", args, 10); args[numArgs] = NULL; // by creating a minimal environment and not inheriting from the parent, // it cause cgiSpoof to run, picking up command-line params as cgi vars. // SERVER_SOFTWARE triggers default cgi trash temp location char *newenviron[] = { "HGDB_CONF=hg.conf", "SERVER_SOFTWARE=Apache", NULL }; int sleepSeconds = 1; // was 5 sleep(sleepSeconds); // Give the foreground process time to write the cart. execve(args[0], args+1, newenviron); // SHOULD NOT GET HERE UNLESS EXEC FAILED. verbose(1,"execve failed for %s\n", exec); _exit(0); // exit without the usual cleanup which messes up parent's db connections etc. } else // child { _exit(0); // exit without the usual cleanup which messes up parent's db connections etc. } } else // parent { *pWorkUrl = workUrl; // wait for the exiting child (not grandchild) int w, status; do { w = waitpid(pid, &status, WUNTRACED | WCONTINUED); if (w == -1) { perror("waitpid"); exit(EXIT_FAILURE); } if (WIFEXITED(status)) { if (WEXITSTATUS(status) != 0) verbose(1, "exited, status=%d\n", WEXITSTATUS(status)); } else if (WIFSIGNALED(status)) { verbose(1, "killed by signal %d\n", WTERMSIG(status)); } else if (WIFSTOPPED(status)) { verbose(1, "stopped by signal %d\n", WSTOPSIG(status)); } else if (WIFCONTINUED(status)) { verbose(1, "continued\n"); } } while (!WIFEXITED(status) && !WIFSIGNALED(status)); // done waiting for child. } } boolean isActiveDb(char *db) /* check if the db is in dbDb and active */ { struct sqlConnection *conn = hConnectCentral(); char query[256]; sqlSafef(query, sizeof query, "select count(*) from dbDb where name='%s' and active=1", db); int count = sqlQuickNum(conn, query); hDisconnectCentral(&conn); return (count >= 1); } unsigned isLiveHub(char *db, char **pUrl, boolean talkative) { unsigned hubId = hubIdFromTrackName(db); struct sqlConnection *conn = hConnectCentral(); struct hubConnectStatus *hubStat = hubConnectStatusForId(conn, hubId); hDisconnectCentral(&conn); unsigned result = 0; if (hubStat) { if (hubStat->errorMessage) { if (talkative) printf("HUB errMessage=%s<BR>\n", hubStat->errorMessage); } else { if (hubStat->trackHub) { if (trackHubHasDatabase(hubStat->trackHub, db)) { if (pUrl) *pUrl = cloneString(hubStat->hubUrl); result = hubId; // indicate db found in hub. } if (talkative) printf("[%s]<br>\n", hubStat->trackHub->shortLabel); } } } return result; } char *getHubUrlFromId(unsigned hubId) { struct sqlConnection *conn = hConnectCentral(); struct hubConnectStatus *hubStat = hubConnectStatusForId(conn, hubId); hDisconnectCentral(&conn); char *result = NULL; if (hubStat) { if (!hubStat->errorMessage) { if (hubStat->trackHub) { result = cloneString(hubStat->hubUrl); } } } return result; } /* hold extra data that does not fit in the track struct itself, * but which is track-specific */ struct ctExtra { struct ctExtra *next; char *name; // important to identity of the track in hgCustom char *browserLines; char *trackLine; int tableRows; unsigned long tableDataLength; char *bigDataUrl; }; struct downloadResults { struct downloadResults *next; char *db; char *ctPath; // path to ct file in trash struct customTrack *cts; // tracks struct ctExtra *ctExtras; // parallel additional info to the tracks }; void addFieldToTrackLine(struct dyString *dy, struct customTrack *track, char *field, boolean isRequired) /* add optional or required field and its value to a track line */ { char *val = hashFindVal(track->tdb->settingsHash, field); if (val) { dyStringPrintf(dy, " %s='%s'", field, val); } else { if (isRequired) errAbort("required field %s is missing from custom track settings", field); } } char *fabricateWigTrackline(struct customTrack *track) /* make up a missing trackline for a ct exported from Table Browser as a wig */ { struct dyString *dy = dyStringNew(1024); dyStringPrintf(dy, "track "); addFieldToTrackLine(dy, track,"name", TRUE); // Required dyStringPrintf(dy, " type='wiggle_0'"); addFieldToTrackLine(dy, track,"description", FALSE); addFieldToTrackLine(dy, track,"visibility", FALSE); addFieldToTrackLine(dy, track,"priority", FALSE); addFieldToTrackLine(dy, track,"altColor", FALSE); return dyStringCannibalize(&dy); } struct downloadResults *processCtsForDownloadInternals(char *contents, char **pTrackHubsVar) /* Process the saved session cart contents, * looking for custom-tracks which are db-specific. */ { struct downloadResults *resultList = NULL; if (!contents) return resultList; char *contentsToChop = cloneString(contents); char *namePt = contentsToChop; struct sqlConnection *ctConn = hAllocConn(CUSTOM_TRASH); while (isNotEmpty(namePt)) { char *dataPt = strchr(namePt, '='); char *nextNamePt; if (dataPt == NULL) errAbort("ERROR: Mangled session content string %s", namePt); *dataPt++ = 0; nextNamePt = strchr(dataPt, '&'); if (nextNamePt != NULL) *nextNamePt++ = 0; if (startsWith(CT_FILE_VAR_PREFIX, namePt)) { cgiDecode(dataPt, dataPt, strlen(dataPt)); struct downloadResults *result = NULL; AllocVar(result); char *db = namePt + strlen(CT_FILE_VAR_PREFIX); result->db = cloneString(db); if (fileExists(dataPt)) { result->ctPath = cloneString(dataPt); struct customTrack *cts = NULL, *track = NULL; boolean thisGotLiveCT = FALSE, thisGotExpiredCT = FALSE; customFactoryTestExistence(db, dataPt, &thisGotLiveCT, &thisGotExpiredCT, &cts); result->cts = cts; // add it to a structure for tracking/printing later. for (track=cts; track; track = track->next) { struct ctExtra *extra = NULL; AllocVar(extra); // what to save in ct output // name is important because hgCustom identifies by this string char *name = hashFindVal(track->tdb->settingsHash, "name"); extra->name = cloneString(name); char *browserLines = hashFindVal(track->tdb->settingsHash, "browserLines"); // to handle multiple-lines replace semi-colon ";" with newline. if (browserLines) replaceChar(browserLines,';','\n'); extra->browserLines = cloneString(browserLines); char *origTrackLine = hashFindVal(track->tdb->settingsHash, "origTrackLine"); extra->trackLine = cloneString(origTrackLine); if (!extra->trackLine) // Table Browser creates some wiggles without an origTrackLine { char *tdbType = hashFindVal(track->tdb->settingsHash, "tdbType"); if (tdbType && startsWith("wig ", tdbType)) { extra->trackLine = fabricateWigTrackline(track); } } // is it weird that the loader customFactoryTestExistence() did not do this for me? char *wibFilePath = hashFindVal(track->tdb->settingsHash, "wibFile"); if (wibFilePath && fileExists(wibFilePath)) { track->wibFile = wibFilePath; } char *mafFilePath = hashFindVal(track->tdb->settingsHash, "mafFile"); if (mafFilePath && fileExists(mafFilePath)) { track->wibFile = mafFilePath; } // old vcf type (not vcfTabix) char *vcfFilePath = hashFindVal(track->tdb->settingsHash, "vcfFile"); if (vcfFilePath && fileExists(vcfFilePath)) { track->wibFile = vcfFilePath; } if (track->dbTrack && track->dbDataLoad && track->dbTableName) { extra->tableRows = sqlTableSizeIfExists(ctConn, track->dbTableName); extra->tableDataLength = sqlTableDataSizeFromSchema(ctConn, CUSTOM_TRASH, track->dbTableName); } char *bigDataUrl = hashFindVal(track->tdb->settingsHash, "bigDataUrl"); if (bigDataUrl) extra->bigDataUrl = bigDataUrl; slAddHead(&result->ctExtras,extra); } slReverse(&result->ctExtras); } slAddHead(&resultList,result); } else if (sameString("trackHubs", namePt)) { cgiDecode(dataPt, dataPt, strlen(dataPt)); if (pTrackHubsVar) *pTrackHubsVar = cloneString(dataPt); } namePt = nextNamePt; } hFreeConn(&ctConn); freeMem(contentsToChop); slReverse(&resultList); return resultList; } void processCtsForDownload(char *contents) /* Process the saved session cart contents, * looking for custom-tracks which are db-specific. */ { struct downloadResults *resultList = processCtsForDownloadInternals(contents, NULL); printf("<H2>Custom Tracks</H2>"); printf("You can backup the custom tracks which you previously uploaded to UCSC Genome Browser servers.<br>" "It will create a single .tar.gz file with custom track data,<br>" "which you can then download to your own machine for use as a backup.<br>" "\n"); struct downloadResults *result = NULL; int ctCount = 0; long totalDataToDownload = 0; for (result=resultList; result; result=result->next) { if (!result->ctPath) // ct.bed was missing for the db continue; printf("<h3>Database %s</h3>\n", result->db); if (startsWith("hub_", result->db)) { unsigned hubId = isLiveHub(result->db, NULL, TRUE); if (hubId == 0) { printf("Skipping HUB Database %s. " "This hub db is not currently available and will not get backed up.<BR><BR>\n", result->db); continue; // skip db if not active. } } else { if (!isActiveDb(result->db)) { printf("Skipping Database %s. " "The db is not currently active and will not get backed up.<BR><BR>\n", result->db); continue; // skip db if not active. } } struct customTrack *cts = result->cts, *track = NULL; struct ctExtra *extras = result->ctExtras, *extra = NULL; printf("<table>\n"); for (track=cts,extra=extras; track; track=track->next,extra=extra->next) { boolean wibMissing = track->wibFile && !fileExists(track->wibFile); if (extra->bigDataUrl || (track->dbTrack && track->dbDataLoad && track->dbTableName && !wibMissing)) { long trackDataToDownload = 0; ++ctCount; if (!extra->bigDataUrl) { char greek[32]; sprintWithGreekByte(greek, sizeof(greek), extra->tableDataLength); trackDataToDownload += extra->tableDataLength; // handle wiggle cts which have an additional wig binary // wibFile=../trash/ct/hgtct_genome_542_dc1750.wib // wibFile=../trash/ct/ct_hgwdev_galt_e83f_3892d0.maf // wibFIle=../trash/ct/ct_hgwdev_galt_4ba3_415cc0.vcf if (track->wibFile && fileExists(track->wibFile)) { long wibFileSize = fileSize(track->wibFile); char greek[32]; sprintWithGreekByte(greek, sizeof(greek), wibFileSize); trackDataToDownload += wibFileSize; } } // htmlFile=../trash/ct/ct_hgwdev_galt_d011_383c50.html if (track->htmlFile && fileExists(track->htmlFile)) { long htmlFileSize = fileSize(track->htmlFile); char greek[32]; sprintWithGreekByte(greek, sizeof(greek), htmlFileSize); trackDataToDownload += htmlFileSize; } char greek[32]; sprintWithGreekByte(greek, sizeof(greek), trackDataToDownload); if (trackDataToDownload == 0) // suppress 0.0 B greek[0] = 0; // empty string printf("<tr><td>%s</td><td>%s</td></tr>\n", extra->name, greek); // track name totalDataToDownload += trackDataToDownload; } } printf("</table>\n"); } printf("<br>\n"); printf("%d custom tracks found.<br>\n", ctCount); char greek[32]; sprintWithGreekByte(greek, sizeof(greek), totalDataToDownload); printf("Total custom track data to backup: %s ", greek); if (ctCount > 0) { - cgiMakeButton(hgsMakeDownloadPrefix, "create custom tracks backup archive"); + cgiMakeButton(hgsMakeDownloadPrefix, "Create custom tracks backup archive"); } printf("<br>\n"); printf("<br>\n"); } void showDownloadSessionCtData(struct hashEl *downloadList) /* Show download page for the given session */ { char query[512]; char **row = NULL; struct sqlResult *sr = NULL; puts("Content-Type:text/html\n"); cartWebStart(cart, NULL, "Backup Custom Tracks"); jsInit(); struct sqlConnection *conn = hConnectCentral(); printf("<FORM ACTION=\"%s\" NAME=\"mainForm\" METHOD=POST " "ENCTYPE=\"multipart/form-data\">\n", hgSessionName()); cartSaveSession(cart); sqlSafef(query, sizeof(query), "SELECT firstUse, contents from %s " "WHERE id=%u", "sessionDb", cartSessionRawId(cart)); sr = sqlGetResult(conn, query); if ((row = sqlNextRow(sr)) != NULL) { char *db = NULL; char *firstUse = row[0]; printf("<div style=\"max-width:1024px\">"); printf("<table id=\"sessionTable\" class=\"sessionTable stripe hover row-border compact\" borderwidth=0>\n"); printf("<thead><tr>" "<TH><B>created on</B></TH>" "<TH><B>assembly</B></TH>" "</tr></thead>"); printf("<tbody>\n"); printf("<TR>"); printf("<TD><nobr>%s</nobr> </TD>", firstUse); char *contents = row[1]; char *dbIdx = NULL; if (startsWith("db=", contents)) // the first variable in the cart (rare case) dbIdx = contents+3; else dbIdx = strstr(contents, "&db=") + 4; if (dbIdx) { char *dbEnd = strchr(dbIdx, '&'); if (dbEnd != NULL) db = cloneStringZ(dbIdx, dbEnd-dbIdx); else db = cloneString(dbIdx); } else db = cloneString("n/a"); printf("<TD align=center>%s</td>", db); printf("</tr>"); printf("</tbody>\n"); printf("</TABLE>\n"); printf("</div>\n"); printf("<P></P>\n"); processCtsForDownload(contents); } sqlFreeResult(&sr); hDisconnectCentral(&conn); printf("</FORM>\n"); cartWebEnd(); } void makeTrashFileLink(char *trashFile, char *cwd, char *outDbDir, char *name) /* Touch file and make a symlink to it as db/name.ext */ { // touch to refresh access time so it will not be cleaned out // this may be redundant in the future but also harmless readAndIgnore(trashFile); // make a symlink with name.ext using track name and wibFile extension // that points to the original wibFile char newPath[1024]; char toDir[PATH_LEN], toFile[FILENAME_LEN], toExt[FILEEXT_LEN]; splitPath(trashFile, toDir, toFile, toExt); // record the original path in a file. safef(newPath, sizeof newPath, "%s/%s%s", outDbDir, name, toExt); char *oldPath = expandRelativePath(cwd, trashFile); if (symlink(oldPath, newPath)) errAbort("failed to symlink from %s to %s", oldPath, newPath); } void saveSqlDataForTable(char *tableName, FILE *f) /* append table data as tab-separated text */ { struct sqlConnection *conn = sqlConnect(CUSTOM_TRASH); char query[256]; struct sqlResult *sr; char **row = NULL; sqlSafef(query, sizeof(query), "select * from %s", tableName); sr = sqlGetResult(conn, query); int numFields = sqlCountColumns(sr); boolean hasBin = (sqlFieldColumn(sr, "bin") != -1); struct dyString *dy = dyStringNew(1024); while ((row=sqlNextRow(sr))) { int start = 0; if (hasBin) // skip bin column ++start; int i; for (i=start; i<numFields; ++i) { if (i > start) dyStringAppendC(dy, '\t'); dyStringAppend(dy, row[i]); } dyStringAppendC(dy, '\n'); mustWrite(f, dy->string, dy->stringSize); dyStringClear(dy); } sqlFreeResult(&sr); sqlDisconnect(&conn); } void updateProgessFile(char *backgroundProgress, struct dyString *dy) /* update the progress file as a separate channel */ { // write to new file char newName[1024]; safef(newName, sizeof newName, "%s.new", backgroundProgress); FILE *f = mustOpen(newName, "w"); mustWrite(f, dy->string, dy->stringSize); carefulClose(&f); // replace original if (rename(newName, backgroundProgress)) errnoAbort("failed to rename progress file"); } void appendTrashFileToCt(FILE *fct, char *fileName) /* append wibFile to open fct file */ { FILE *f = mustOpen(fileName, "r"); long remaining = fileSize(fileName); int bufSize = 65536; char *buf = needMem(bufSize); while (remaining) { int bufRemain = bufSize; if (bufRemain > remaining) bufRemain = remaining; mustRead(f, buf, bufRemain); // mustRead OK since we are reading from disk mustWrite(fct, buf, bufRemain); remaining -= bufRemain; } carefulClose(&f); } void doOutWigData(char *table, char *database, char *outFile) // should be CUSTOM_TRASH /* Return wiggle data in variableStep format. */ { /* Write out wig data in region. */ struct wiggleDataStream *wds = NULL; int operations = wigFetchAscii; operations = wigFetchAscii; wds = wiggleDataStreamNew(); wds->getData(wds, database, table, operations); wds->asciiOut(wds, database, outFile, TRUE, FALSE); wiggleDataStreamFree(&wds); } void makeDownloadSessionCtData(char *param1, char *backgroundProgress) /* Download tables and data to save save in compressed archive. */ { char query[512]; char **row = NULL; struct sqlResult *sr = NULL; // Initialize .progress channel file struct dyString *dyProg = dyStringNew(256); dyStringPrintf(dyProg, "please wait, dumping data to archive ...<br>\n"); updateProgessFile(backgroundProgress, dyProg); lazarusLives(20 * 60); htmlOpen("Preparing Custom Tracks Backup Archive for Download"); jsInit(); struct sqlConnection *conn = hConnectCentral(); printf("<FORM ACTION=\"%s\" NAME=\"mainForm\" METHOD=POST " "ENCTYPE=\"multipart/form-data\">\n", hgSessionName()); cartSaveSession(cart); sqlSafef(query, sizeof(query), "SELECT contents from %s " "WHERE id=%u", "sessionDb", cartSessionRawId(cart)); sr = sqlGetResult(conn, query); // setup temp output dir: // ../trash/ // ctBackup/ // random@#$%/ # random value so it is unique /* make a directory, including parent directories */ char tempOut[1024]; safef(tempOut, sizeof tempOut, "../trash/ctBackup"); if (!fileExists(tempOut)) { makeDir(tempOut); } char tempOutRand[1024]; // make randomDir char *randPath=NULL; int randCount = 0; while(TRUE) { randPath=makeRandomKey(128+33); // at least 128 bits of protection, 33 for the world population size. // Avoid the possibility of somehow doing more than one instance of the session // backup at the same time. // On the extremely rare chance that the directory already exists, // just call the function again. safef(tempOutRand, sizeof tempOutRand, "%s/%s", tempOut, randPath); if (makeDir(tempOutRand)) break; ++randCount; if (randCount > 100) // should never happen errAbort("unable to create random output dir."); } int ctCount = 0; int foundCount = 0; if ((row = sqlNextRow(sr)) != NULL) { ++foundCount; char *contents = row[0]; char *trackHubsVar = NULL; struct downloadResults *resultList = processCtsForDownloadInternals(contents, &trackHubsVar); struct downloadResults *result = NULL; for (result=resultList; result; result=result->next) { struct customTrack *cts = result->cts, *track = NULL; struct ctExtra *extras = result->ctExtras, *extra = NULL; // empty db dir can trigger cleaning out of cart ct var for db // if it has no ct.bed char outDbDir[1024]; safef(outDbDir, sizeof outDbDir, "%s/%s", tempOutRand, result->db); if (!fileExists(outDbDir)) { makeDir(outDbDir); } if (!result->ctPath) // ct.bed was missing for the db continue; char *hubUrl = NULL; if (startsWith("hub_", result->db)) { unsigned hubId = isLiveHub(result->db, &hubUrl, FALSE); if (hubId == 0) { continue; // skip hub if not active. } } else { if (!isActiveDb(result->db)) { continue; // skip db if not active. } } printf("<h3>%s</h3>\n", result->db); dyStringPrintf(dyProg,"<h3>%s</h3>\n", result->db); updateProgessFile(backgroundProgress, dyProg); lazarusLives(20 * 60); if (startsWith("hub_", result->db)) { // Save the hubUrl now in case it gets restored on another machine // that does NOT have that hub loaded on it. // save output as hubUrl text file char hubUrlPath[1024]; safef(hubUrlPath, sizeof hubUrlPath, "%s/hubUrl", outDbDir); FILE *f = mustOpen(hubUrlPath, "w"); fprintf(f, "%s", hubUrl); // NO NEWLINE carefulClose(&f); } char cwd[PATH_LEN]; if (!getcwd(cwd, sizeof(cwd))) errnoAbort("unable to get current dir."); for (track=cts,extra=extras; track; track=track->next,extra=extra->next) { boolean wibMissing = track->wibFile && !fileExists(track->wibFile); if (extra->bigDataUrl || (track->dbTrack && track->dbDataLoad && track->dbTableName && !wibMissing)) { ++ctCount; printf("%s <br>\n", extra->name); dyStringPrintf(dyProg, "%s <br>\n", extra->name); updateProgessFile(backgroundProgress, dyProg); lazarusLives(20 * 60); char outNameCt[2014]; safef(outNameCt, sizeof outNameCt, "%s/%s.ct", outDbDir, extra->name); FILE *fct = mustOpen(outNameCt, "w"); // write the track header if (extra->browserLines) fprintf(fct, "%s", extra->browserLines); // should have ; converted \n already if (!extra->trackLine) errAbort("origTrackLine is NULL!"); fprintf(fct, "%s\n", extra->trackLine); if (!extra->bigDataUrl) { // handle wiggle cts which have an additional wig binary // wibFile='../trash/ct/hgtct_genome_542_dc1750.wib' // wibFile='../trash/ct/ct_hgwdev_galt_e83f_3892d0.maf' // wibFile='../trash/ct/ct_hgwdev_galt_4ba3_415cc0.vcf' // symlink for speed, file should not change if (track->wibFile) { if (endsWith(track->wibFile, ".wib")) { // dump wig as ascii char outNameWig[2014]; safef(outNameWig, sizeof outNameWig, "%s/%s.wig", outDbDir, extra->name); doOutWigData(track->dbTableName, CUSTOM_TRASH, outNameWig); // no easy way to append it to .ct directly // append text to ct appendTrashFileToCt(fct, outNameWig); remove(outNameWig); } if (endsWith(track->wibFile, ".maf")) { // append text to ct appendTrashFileToCt(fct, track->wibFile); } if (endsWith(track->wibFile, ".vcf")) { // append text to ct appendTrashFileToCt(fct, track->wibFile); } } else // simple BED-like { saveSqlDataForTable(track->dbTableName, fct); } } // handle extra htmlFile track doc // htmlFile=../trash/ct/ct_hgwdev_galt_d011_383c50.html // symlink for speed, file should not change if (track->htmlFile && fileExists(track->htmlFile)) { makeTrashFileLink(track->htmlFile, cwd, outDbDir, extra->name); } carefulClose(&fct); } } } } sqlFreeResult(&sr); if (foundCount == 0) errAbort("No session found for hgsid=%u", cartSessionRawId(cart)); if (ctCount == 0) errAbort("No custom tracks found for hgsid=%u", cartSessionRawId(cart)); char archiveName[1024]; safef(archiveName, sizeof archiveName, "savedSessionCtRaw.tar.gz"); dyStringPrintf(dyProg, "<br>\n"); int saveDySize = dyProg->stringSize; dyStringPrintf(dyProg, "creating and compressing archive %s <br>\n", archiveName); updateProgessFile(backgroundProgress, dyProg); lazarusLives(20 * 60); // create the archive char *cwd = cloneString(getCurrentDir()); setCurrentDir(tempOutRand); char excludeBuf[4096]; safef(excludeBuf, sizeof excludeBuf, "--exclude=%s", archiveName); char *pipeCmd1[] = { "tar", "-zcphf", archiveName, ".", excludeBuf, NULL}; struct pipeline *pl = pipelineOpen1(pipeCmd1, pipelineWrite | pipelineNoAbort, "/dev/null", NULL, 0); int sysVal = pipelineWait(pl); setCurrentDir(cwd); if (!((sysVal == 0) || (sysVal == 1))) // we tolerate 1 because tar doesn't like us creating the archive in the directory we're backing up errAbort("System call returned %d for:\n %s", sysVal, pipelineDesc(pl)); dyProg->stringSize = saveDySize; // restore prev size, popping. dyProg->string[dyProg->stringSize] = 0; dyStringPrintf(dyProg, "archive %s created <br>\n", archiveName); updateProgessFile(backgroundProgress, dyProg); lazarusLives(20 * 60); char downPath[1024]; safef(downPath, sizeof downPath, "%s/%s", tempOutRand, archiveName); printf("<br>\n"); char *encDownPath = cgiEncodeFull(downPath); char downButName[1024]; safef(downButName, sizeof(downButName), "%s%s", hgsDoDownloadPrefix, encDownPath); // using getElementById('%s') here is a work-around for special charachters in id. // I tried escaping, but it did not seem to work with our jQuery version. // mainly the special characters it sees are periods and slashes. char downButNameHack[1024]; safef(downButNameHack, sizeof(downButNameHack), "document.getElementById('%s')", downButName); printf("Download archive as local file:\n"); cgiMakeOnKeypressTextVar(hgsSaveLocalBackupFileName, cartUsualString(cart, hgsSaveLocalBackupFileName, ""), 20, jsPressOnEnter(downButNameHack)); printf(".tar.gz"); char greek[32]; sprintWithGreekByte(greek, sizeof(greek), fileSize(downPath)); printf(" %s ", greek); printf("<input type='submit' name='%s' id='%s' value='download backup archive'>", downButName, downButName); // check the file name, make sure it is not blank char js[1024]; safef(js, sizeof js, "var textVar = document.getElementById('%s');" "if (textVar.value != '')" " return true;" "alert('Please enter a filename for the backup archive');" "return false;" , hgsSaveLocalBackupFileName); jsOnEventById("click", downButName, js); // Add a button that just returns to the saved sessions list. // because after pressing download and it will go into the users downloads folder, // the browser remains sitting on this page and needs a way to return // to the saved sessions list. Also clear the download name field. printf(" "); printf("<input type='submit' id='hgsReturn' value='return'>"); safef(js, sizeof js, "var textVar = document.getElementById('%s');" "textVar.value = '';" "return true;" , hgsSaveLocalBackupFileName); jsOnEventById("click", "hgsReturn", js); printf("<br>\n"); printf("<br>\n"); hDisconnectCentral(&conn); printf("</FORM>\n"); htmlClose(); fflush(stdout); } void doDownloadSessionCtData(struct hashEl *downloadPathList) /* Download given table to browser to save. */ { struct hashEl *hel = NULL; // I think it should only get one at a time, not a list. hel = downloadPathList; char *encDownPath = hel->name + strlen(hgsDoDownloadPrefix); char *downPath = cgiDecodeClone(encDownPath); char *fileName = cartString(cart, hgsSaveLocalBackupFileName); char outFile[1024]; safef(outFile, sizeof outFile, "%s.tar.gz", fileName); long fSize = fileSize(downPath); printf("Content-Type: application/octet-stream\n"); printf("Content-Disposition: attachment; filename=\"%s\"\n", outFile); printf("Content-Length: %ld\n", fSize); printf("\n"); FILE *f = mustOpen(downPath, "r"); long remaining = fSize; int bufSize = 65536; char *buf = needMem(bufSize); while (remaining) { int bufRemain = bufSize; if (bufRemain > remaining) bufRemain = remaining; mustRead(f, buf, bufRemain); // mustRead OK since we are reading from disk mustWrite(stdout, buf, bufRemain); remaining -= bufRemain; lazarusLives(20 * 60); // extend keep-alive time. for big downloads on slow connections. } carefulClose(&f); cartRemove(cart, hgsSaveLocalBackupFileName); }