44ccfacbe3a3d4b300f80d48651c77837a4b571e galt Tue Apr 26 11:12:02 2022 -0700 SQL INJECTION Prevention Version 2 - this improves our methods by making subclauses of SQL that get passed around be both easy and correct to use. The way that was achieved was by getting rid of the obscure and not well used functions sqlSafefFrag and sqlDyStringPrintfFrag and replacing them with the plain versions of those functions, since these are not needed anymore. The new version checks for NOSQLINJ in unquoted %-s which is used to include SQL clauses, and will give an error the NOSQLINJ clause is not present, and this will automatically require the correct behavior by developers. sqlDyStringPrint is a very useful function, however because it was not enforced, users could use various other dyString functions and they operated without any awareness or checking for SQL correct use. Now those dyString functions are prohibited and it will produce an error if you try to use a dyString function on a SQL string, which is simply detected by the presence of the NOSQLINJ prefix. diff --git src/hg/hgSession/backup.c src/hg/hgSession/backup.c index e536a76..dac8813 100644 --- src/hg/hgSession/backup.c +++ src/hg/hgSession/backup.c @@ -1,1264 +1,1264 @@ /* backupRestore - Routines related to backing up and restoring the current cart and its custom tracks. */ /* Copyright (C) 2014 The Regents of the University of California * See kent/LICENSE or http://genome.ucsc.edu/license/ for licensing information. */ #include "common.h" #include "cart.h" #include "net.h" #include "textOut.h" #include "base64.h" #include "md5.h" #include "obscure.h" #include "net.h" #include "hgConfig.h" #include "htmshell.h" #include "cheapcgi.h" #include "hgSession.h" #include <sys/wait.h> #include "hCommon.h" #include "hdb.h" #include "trackHub.h" #include "hubConnect.h" #include "customFactory.h" #include "jsHelper.h" #include "trashDir.h" #include "filePath.h" #include "cgiApoptosis.h" #include "wiggle.h" #include "pipeline.h" #define UCSC_GB_BACKUP_VERSION_FILENAME "UCSC_GB_BACKUP_VERSION" static void errAbortHandler(char *format, va_list args) { // provide more explicit message when we run out of memory (#5147). if(strstr(format, "needLargeMem:") || strstr(format, "carefulAlloc:")) htmlVaWarn("Region selected is too large for calculation. Please specify a smaller region or try limiting to fewer data points.", args); else { // call previous handler popWarnHandler(); vaWarn(format, args); } if(isErrAbortInProgress()) noWarnAbort(); } static void vaHtmlOpen(char *format, va_list args) /* Start up a page that will be in html format. */ { puts("Content-Type:text/html\n"); cartVaWebStart(cart, database, format, args); pushWarnHandler(errAbortHandler); } void htmlOpen(char *format, ...) /* Start up a page that will be in html format. */ { va_list args; va_start(args, format); vaHtmlOpen(format, args); va_end(args); } void htmlClose() /* Close down html format page. */ { popWarnHandler(); cartWebEnd(); } // ------------------------------------------ boolean waitForBackgroundFile(char *path) /* Wait 30 seconds for the file at path to appear. Return TRUE if timeout exceeded. */ { int waited = 0; int waitLimit = 30; // max wait in seconds // sometimes the background process is a little slow, // we can wait up to 30 seconds for it. while (!(fileExists(path) && fileSize(path) > 0) && (waited < waitLimit)) { sleep(1); ++waited; } if (waited >= waitLimit) { htmlOpen("Background Status"); errAbort("Background file %s not found or empty.", path); htmlClose(); return TRUE; } return FALSE; } void getBackgroundStatus(char *url) /* fetch status as the latest complete html block available. * fetch progress info instead if background proc still running. */ { char *html = NULL; // get the pid and see if it is still running char pidName[1024]; safef(pidName, sizeof pidName, "%s.pid", url); if (waitForBackgroundFile(pidName)) return; FILE *pidF = mustOpen(pidName, "r"); long pid = 0; fscanf(pidF, "%ld", &pid); carefulClose(&pidF); // If the background process is still running, grab the progress file instead if (getpgid(pid) >= 0) { htmlOpen("Background Status"); char progressName[1024]; safef(progressName, sizeof progressName, "%s.progress", url); if (waitForBackgroundFile(progressName)) return; if (fileExists(progressName)) { readInGulp(progressName, &html, NULL); printf("%s",html); } else { printf("progress file missing<br>\n"); } printf("<BR>\n<FORM ACTION=\"/cgi-bin/hgTables\" METHOD=GET>\n" //"<INPUT TYPE=SUBMIT NAME=\"%s\" VALUE=\"Back\" >" "<INPUT TYPE=SUBMIT NAME=\"Refresh\" id='Refresh' VALUE=\"Refresh\">" "</FORM>\n" ); jsOnEventById("click", "Refresh", "window.location=window.location;return false;"); jsInline("setTimeout(function(){location = location;},5000);\n"); jsInline("window.scrollTo(0,document.body.scrollHeight);"); htmlClose(); return; } // otherwise read the main html file // sometimes the background process is a little slow, if (waitForBackgroundFile(url)) return; readInGulp(url, &html, NULL); int numLines = chopString(html, "\n", NULL, 1000000); char **lines = NULL; AllocArray(lines, numLines); chopString(html, "\n", lines, numLines); int end; for (end=numLines-1; end >= 0 && ! (endsWith(lines[end], "</html>") || endsWith(lines[end], "</HTML>")) ; --end) /* do nothing */ ; if (end < 0) { htmlOpen("Background Status"); errAbort("No complete html found"); htmlClose(); return; } int start; for (start=end; start >= 0 && ! (startsWith("<html>", lines[start]) || startsWith("<HTML>", lines[start])) ; --start) /* do nothing */ ; if (start < 0) { htmlOpen("Background Status"); errAbort("No html start tag found"); htmlClose(); return; } puts("Content-Type: text/html\n"); int line; boolean autoRefreshFound = FALSE; boolean successfullyUploaded = FALSE; for (line=start; line <= end; line++) { puts(lines[line]); if (startsWith("setTimeout(function(){location = location;}", lines[line])) autoRefreshFound = TRUE; if (startsWith("Output has been successfully uploaded", lines[line])) successfullyUploaded = TRUE; } // if it looks like the background is no longer running, // include the .err stdout output for more informative problem message char urlErr[512]; char *textErr = NULL; safef(urlErr, sizeof urlErr, "%s.err", url); if (!autoRefreshFound && !successfullyUploaded && (fileSize(urlErr) > 0)) { readInGulp(urlErr, &textErr, NULL); //printf("%s", textErr); /* more annoying than helpful in some places. */ } } void startBackgroundWork(char *exec, char **pWorkUrl) /* deal with forking off child for background work * and setting up the trash file for communicating * from the child to the browser */ { char *workUrl = NULL; char hgsid[64]; struct tempName tn; safef(hgsid, sizeof(hgsid), "%s", cartSessionId(cart)); trashDirFile(&tn, "backGround", hgsid, ".tmp"); workUrl = cloneString(tn.forCgi); fflush(stdout); fflush(stderr); // seems that we need to use the double-fork trick // to create enough separation between the non-waiting parent // and the grand-child process. otherwise the OS and Apache are waiting on the child. int pid = fork(); if (pid == -1) { errAbort("can't fork, error %d", errno); } if (pid == 0) // child { int pid2 = fork(); if (pid2 == -1) { errAbort("can't fork, error %d", errno); } if (pid2 == 0) // grand child { // we need to close or redup to open stdout, stderr, stdin // in order for apache to break ties with it. // Will the grandchild cgi still be able to function? // redirect stdout of child to the trash file for easier use of // library functions that output html to stdout. int out = mustOpenFd(tn.forCgi, O_WRONLY | O_CREAT); fflush(stdout); dup2(out,STDOUT_FILENO); /* closes STDOUT before setting it back to saved descriptor */ close(out); // Unfortunately we must create our own stderr log file char errName[1024]; safef(errName, sizeof errName, "%s.err", tn.forCgi); int err = mustOpenFd(errName, O_CREAT | O_WRONLY | O_APPEND); dup2(err, STDERR_FILENO); close(err); // stdin input is just empty int in = mustOpenFd("/dev/null", O_RDONLY); dup2(in, STDIN_FILENO); close(in); // save own pid in .pid file long selfPid = (long) getpid(); char pidName[1024]; safef(pidName, sizeof pidName, "%s.pid", tn.forCgi); char pidBuf[64]; safef(pidBuf, sizeof pidBuf, "%ld", selfPid); int pidFd = mustOpenFd(pidName, O_CREAT | O_WRONLY | O_APPEND); mustWriteFd(pidFd, pidBuf, strlen(pidBuf)); close(pidFd); // progress is a separate channel for progress update char progressName[1024]; safef(progressName, sizeof progressName, "%s.progress", tn.forCgi); // execute so that we will be able to use database and other operations normally. char execPath[4096]; safef(execPath, sizeof execPath, "%s hgsid=%s backgroundProgress=%s", exec, hgsid, progressName); char *args[10]; int numArgs = chopString(execPath, " ", args, 10); args[numArgs] = NULL; // by creating a minimal environment and not inheriting from the parent, // it cause cgiSpoof to run, picking up command-line params as cgi vars. // SERVER_SOFTWARE triggers default cgi trash temp location char *newenviron[] = { "HGDB_CONF=hg.conf", "SERVER_SOFTWARE=Apache", NULL }; int sleepSeconds = 1; // was 5 sleep(sleepSeconds); // Give the foreground process time to write the cart. execve(args[0], args+1, newenviron); // SHOULD NOT GET HERE UNLESS EXEC FAILED. verbose(1,"execve failed for %s\n", exec); _exit(0); // exit without the usual cleanup which messes up parent's db connections etc. } else // child { _exit(0); // exit without the usual cleanup which messes up parent's db connections etc. } } else // parent { *pWorkUrl = workUrl; // wait for the exiting child (not grandchild) int w, status; do { w = waitpid(pid, &status, WUNTRACED | WCONTINUED); if (w == -1) { perror("waitpid"); exit(EXIT_FAILURE); } if (WIFEXITED(status)) { if (WEXITSTATUS(status) != 0) verbose(1, "exited, status=%d\n", WEXITSTATUS(status)); } else if (WIFSIGNALED(status)) { verbose(1, "killed by signal %d\n", WTERMSIG(status)); } else if (WIFSTOPPED(status)) { verbose(1, "stopped by signal %d\n", WSTOPSIG(status)); } else if (WIFCONTINUED(status)) { verbose(1, "continued\n"); } } while (!WIFEXITED(status) && !WIFSIGNALED(status)); // done waiting for child. } } boolean isActiveDb(char *db) /* check if the db is in dbDb and active */ { struct sqlConnection *conn = hConnectCentral(); char query[256]; sqlSafef(query, sizeof query, "select count(*) from dbDb where name='%s' and active=1", db); int count = sqlQuickNum(conn, query); hDisconnectCentral(&conn); return (count >= 1); } unsigned isLiveHub(char *db, char **pUrl, boolean talkative) { unsigned hubId = hubIdFromTrackName(db); struct sqlConnection *conn = hConnectCentral(); struct hubConnectStatus *hubStat = hubConnectStatusForId(conn, hubId); hDisconnectCentral(&conn); unsigned result = 0; if (hubStat) { if (hubStat->errorMessage) { if (talkative) printf("HUB errMessage=%s<BR>\n", hubStat->errorMessage); } else { if (hubStat->trackHub) { if (trackHubHasDatabase(hubStat->trackHub, db)) { if (pUrl) *pUrl = cloneString(hubStat->hubUrl); result = hubId; // indicate db found in hub. } if (talkative) printf("[%s]<br>\n", hubStat->trackHub->shortLabel); } } } return result; } char *getHubUrlFromId(unsigned hubId) { struct sqlConnection *conn = hConnectCentral(); struct hubConnectStatus *hubStat = hubConnectStatusForId(conn, hubId); hDisconnectCentral(&conn); char *result = NULL; if (hubStat) { if (!hubStat->errorMessage) { if (hubStat->trackHub) { result = cloneString(hubStat->hubUrl); } } } return result; } /* hold extra data that does not fit in the track struct itself, * but which is track-specific */ struct ctExtra { struct ctExtra *next; char *name; // important to identity of the track in hgCustom char *browserLines; char *trackLine; int tableRows; unsigned long tableDataLength; char *bigDataUrl; }; struct downloadResults { struct downloadResults *next; char *db; char *ctPath; // path to ct file in trash struct customTrack *cts; // tracks struct ctExtra *ctExtras; // parallel additional info to the tracks }; void addFieldToTrackLine(struct dyString *dy, struct customTrack *track, char *field, boolean isRequired) /* add optional or required field and its value to a track line */ { char *val = hashFindVal(track->tdb->settingsHash, field); if (val) { dyStringPrintf(dy, " %s='%s'", field, val); } else { if (isRequired) errAbort("required field %s is missing from custom track settings", field); } } char *fabricateWigTrackline(struct customTrack *track) /* make up a missing trackline for a ct exported from Table Browser as a wig */ { struct dyString *dy = dyStringNew(1024); dyStringPrintf(dy, "track "); addFieldToTrackLine(dy, track,"name", TRUE); // Required dyStringPrintf(dy, " type='wiggle_0'"); addFieldToTrackLine(dy, track,"description", FALSE); addFieldToTrackLine(dy, track,"visibility", FALSE); addFieldToTrackLine(dy, track,"priority", FALSE); addFieldToTrackLine(dy, track,"altColor", FALSE); return dyStringCannibalize(&dy); } struct downloadResults *processCtsForDownloadInternals(char *contents, char **pTrackHubsVar) /* Process the saved session cart contents, * looking for custom-tracks which are db-specific. */ { struct downloadResults *resultList = NULL; if (!contents) return resultList; char *contentsToChop = cloneString(contents); char *namePt = contentsToChop; struct sqlConnection *ctConn = hAllocConn(CUSTOM_TRASH); while (isNotEmpty(namePt)) { char *dataPt = strchr(namePt, '='); char *nextNamePt; if (dataPt == NULL) errAbort("ERROR: Mangled session content string %s", namePt); *dataPt++ = 0; nextNamePt = strchr(dataPt, '&'); if (nextNamePt != NULL) *nextNamePt++ = 0; if (startsWith(CT_FILE_VAR_PREFIX, namePt)) { cgiDecode(dataPt, dataPt, strlen(dataPt)); struct downloadResults *result = NULL; AllocVar(result); char *db = namePt + strlen(CT_FILE_VAR_PREFIX); result->db = cloneString(db); if (fileExists(dataPt)) { result->ctPath = cloneString(dataPt); struct customTrack *cts = NULL, *track = NULL; boolean thisGotLiveCT = FALSE, thisGotExpiredCT = FALSE; customFactoryTestExistence(db, dataPt, &thisGotLiveCT, &thisGotExpiredCT, &cts); result->cts = cts; // add it to a structure for tracking/printing later. for (track=cts; track; track = track->next) { struct ctExtra *extra = NULL; AllocVar(extra); // what to save in ct output // name is important because hgCustom identifies by this string char *name = hashFindVal(track->tdb->settingsHash, "name"); extra->name = cloneString(name); char *browserLines = hashFindVal(track->tdb->settingsHash, "browserLines"); // to handle multiple-lines replace semi-colon ";" with newline. if (browserLines) replaceChar(browserLines,';','\n'); extra->browserLines = cloneString(browserLines); char *origTrackLine = hashFindVal(track->tdb->settingsHash, "origTrackLine"); extra->trackLine = cloneString(origTrackLine); if (!extra->trackLine) // Table Browser creates some wiggles without an origTrackLine { char *tdbType = hashFindVal(track->tdb->settingsHash, "tdbType"); if (tdbType && startsWith("wig ", tdbType)) { extra->trackLine = fabricateWigTrackline(track); } } // is it weird that the loader customFactoryTestExistence() did not do this for me? char *wibFilePath = hashFindVal(track->tdb->settingsHash, "wibFile"); if (wibFilePath && fileExists(wibFilePath)) { track->wibFile = wibFilePath; } char *mafFilePath = hashFindVal(track->tdb->settingsHash, "mafFile"); if (mafFilePath && fileExists(mafFilePath)) { track->wibFile = mafFilePath; } // old vcf type (not vcfTabix) char *vcfFilePath = hashFindVal(track->tdb->settingsHash, "vcfFile"); if (vcfFilePath && fileExists(vcfFilePath)) { track->wibFile = vcfFilePath; } if (track->dbTrack && track->dbDataLoad && track->dbTableName) { extra->tableRows = sqlTableSizeIfExists(ctConn, track->dbTableName); extra->tableDataLength = sqlTableDataSizeFromSchema(ctConn, CUSTOM_TRASH, track->dbTableName); } char *bigDataUrl = hashFindVal(track->tdb->settingsHash, "bigDataUrl"); if (bigDataUrl) extra->bigDataUrl = bigDataUrl; slAddHead(&result->ctExtras,extra); } slReverse(&result->ctExtras); } slAddHead(&resultList,result); } else if (sameString("trackHubs", namePt)) { cgiDecode(dataPt, dataPt, strlen(dataPt)); if (pTrackHubsVar) *pTrackHubsVar = cloneString(dataPt); } namePt = nextNamePt; } hFreeConn(&ctConn); freeMem(contentsToChop); slReverse(&resultList); return resultList; } void processCtsForDownload(char *contents) /* Process the saved session cart contents, * looking for custom-tracks which are db-specific. */ { struct downloadResults *resultList = processCtsForDownloadInternals(contents, NULL); printf("<H2>Custom Tracks</H2>"); printf("You can backup the custom tracks which you previously uploaded to UCSC Genome Browser servers.<br>" "It will create a single .tar.gz file with custom track data,<br>" "which you can then download to your own machine for use as a backup.<br>" "\n"); struct downloadResults *result = NULL; int ctCount = 0; long totalDataToDownload = 0; for (result=resultList; result; result=result->next) { if (!result->ctPath) // ct.bed was missing for the db continue; printf("<h3>Database %s</h3>\n", result->db); if (startsWith("hub_", result->db)) { unsigned hubId = isLiveHub(result->db, NULL, TRUE); if (hubId == 0) { printf("Skipping HUB Database %s. " "This hub db is not currently available and will not get backed up.<BR><BR>\n", result->db); continue; // skip db if not active. } } else { if (!isActiveDb(result->db)) { printf("Skipping Database %s. " "The db is not currently active and will not get backed up.<BR><BR>\n", result->db); continue; // skip db if not active. } } struct customTrack *cts = result->cts, *track = NULL; struct ctExtra *extras = result->ctExtras, *extra = NULL; printf("<table>\n"); for (track=cts,extra=extras; track; track=track->next,extra=extra->next) { boolean wibMissing = track->wibFile && !fileExists(track->wibFile); if (extra->bigDataUrl || (track->dbTrack && track->dbDataLoad && track->dbTableName && !wibMissing)) { long trackDataToDownload = 0; ++ctCount; if (!extra->bigDataUrl) { char greek[32]; sprintWithGreekByte(greek, sizeof(greek), extra->tableDataLength); trackDataToDownload += extra->tableDataLength; // handle wiggle cts which have an additional wig binary // wibFile=../trash/ct/hgtct_genome_542_dc1750.wib // wibFile=../trash/ct/ct_hgwdev_galt_e83f_3892d0.maf // wibFIle=../trash/ct/ct_hgwdev_galt_4ba3_415cc0.vcf if (track->wibFile && fileExists(track->wibFile)) { long wibFileSize = fileSize(track->wibFile); char greek[32]; sprintWithGreekByte(greek, sizeof(greek), wibFileSize); trackDataToDownload += wibFileSize; } } // htmlFile=../trash/ct/ct_hgwdev_galt_d011_383c50.html if (track->htmlFile && fileExists(track->htmlFile)) { long htmlFileSize = fileSize(track->htmlFile); char greek[32]; sprintWithGreekByte(greek, sizeof(greek), htmlFileSize); trackDataToDownload += htmlFileSize; } char greek[32]; sprintWithGreekByte(greek, sizeof(greek), trackDataToDownload); if (trackDataToDownload == 0) // suppress 0.0 B greek[0] = 0; // empty string printf("<tr><td>%s</td><td>%s</td></tr>\n", extra->name, greek); // track name totalDataToDownload += trackDataToDownload; } } printf("</table>\n"); } printf("<br>\n"); printf("%d custom tracks found.<br>\n", ctCount); char greek[32]; sprintWithGreekByte(greek, sizeof(greek), totalDataToDownload); printf("Total custom track data to backup: %s ", greek); if (ctCount > 0) { cgiMakeButton(hgsMakeDownloadPrefix, "create custom tracks backup archive"); } printf("<br>\n"); printf("<br>\n"); } void showDownloadSessionCtData(struct hashEl *downloadList) /* Show download page for the given session */ { char query[512]; char **row = NULL; struct sqlResult *sr = NULL; puts("Content-Type:text/html\n"); cartWebStart(cart, NULL, "Backup Custom Tracks"); jsInit(); struct sqlConnection *conn = hConnectCentral(); printf("<FORM ACTION=\"%s\" NAME=\"mainForm\" METHOD=POST " "ENCTYPE=\"multipart/form-data\">\n", hgSessionName()); cartSaveSession(cart); sqlSafef(query, sizeof(query), "SELECT firstUse, contents from %s " "WHERE id=%u", "sessionDb", cartSessionRawId(cart)); sr = sqlGetResult(conn, query); if ((row = sqlNextRow(sr)) != NULL) { char *db = NULL; char *firstUse = row[0]; printf("<div style=\"max-width:1024px\">"); printf("<table id=\"sessionTable\" class=\"sessionTable stripe hover row-border compact\" borderwidth=0>\n"); printf("<thead><tr>" "<TH><B>created on</B></TH>" "<TH><B>assembly</B></TH>" "</tr></thead>"); printf("<tbody>\n"); printf("<TR>"); printf("<TD><nobr>%s</nobr> </TD>", firstUse); char *contents = row[1]; char *dbIdx = NULL; if (startsWith("db=", contents)) // the first variable in the cart (rare case) dbIdx = contents+3; else dbIdx = strstr(contents, "&db=") + 4; if (dbIdx) { char *dbEnd = strchr(dbIdx, '&'); if (dbEnd != NULL) db = cloneStringZ(dbIdx, dbEnd-dbIdx); else db = cloneString(dbIdx); } else db = cloneString("n/a"); printf("<TD align=center>%s</td>", db); printf("</tr>"); printf("</tbody>\n"); printf("</TABLE>\n"); printf("</div>\n"); printf("<P></P>\n"); processCtsForDownload(contents); } sqlFreeResult(&sr); hDisconnectCentral(&conn); printf("</FORM>\n"); cartWebEnd(); } void makeTrashFileLink(char *trashFile, char *cwd, char *outDbDir, char *name) /* Touch file and make a symlink to it as db/name.ext */ { // touch to refresh access time so it will not be cleaned out // this may be redundant in the future but also harmless readAndIgnore(trashFile); // make a symlink with name.ext using track name and wibFile extension // that points to the original wibFile char newPath[1024]; char toDir[PATH_LEN], toFile[FILENAME_LEN], toExt[FILEEXT_LEN]; splitPath(trashFile, toDir, toFile, toExt); // record the original path in a file. safef(newPath, sizeof newPath, "%s/%s%s", outDbDir, name, toExt); char *oldPath = expandRelativePath(cwd, trashFile); if (symlink(oldPath, newPath)) errAbort("failed to symlink from %s to %s", oldPath, newPath); } void saveSqlDataForTable(char *tableName, FILE *f) /* append table data as tab-separated text */ { struct sqlConnection *conn = sqlConnect(CUSTOM_TRASH); char query[256]; struct sqlResult *sr; char **row = NULL; sqlSafef(query, sizeof(query), "select * from %s", tableName); sr = sqlGetResult(conn, query); int numFields = sqlCountColumns(sr); boolean hasBin = (sqlFieldColumn(sr, "bin") != -1); struct dyString *dy = dyStringNew(1024); while ((row=sqlNextRow(sr))) { int start = 0; if (hasBin) // skip bin column ++start; int i; for (i=start; i<numFields; ++i) { if (i > start) dyStringAppendC(dy, '\t'); dyStringAppend(dy, row[i]); } dyStringAppendC(dy, '\n'); mustWrite(f, dy->string, dy->stringSize); dyStringClear(dy); } sqlFreeResult(&sr); sqlDisconnect(&conn); } void updateProgessFile(char *backgroundProgress, struct dyString *dy) /* update the progress file as a separate channel */ { // write to new file char newName[1024]; safef(newName, sizeof newName, "%s.new", backgroundProgress); FILE *f = mustOpen(newName, "w"); mustWrite(f, dy->string, dy->stringSize); carefulClose(&f); // replace original if (rename(newName, backgroundProgress)) errnoAbort("failed to rename progress file"); } void appendTrashFileToCt(FILE *fct, char *fileName) /* append wibFile to open fct file */ { FILE *f = mustOpen(fileName, "r"); long remaining = fileSize(fileName); int bufSize = 65536; char *buf = needMem(bufSize); while (remaining) { int bufRemain = bufSize; if (bufRemain > remaining) bufRemain = remaining; mustRead(f, buf, bufRemain); // mustRead OK since we are reading from disk mustWrite(fct, buf, bufRemain); remaining -= bufRemain; } carefulClose(&f); } void doOutWigData(char *table, char *database, char *outFile) // should be CUSTOM_TRASH /* Return wiggle data in variableStep format. */ { /* Write out wig data in region. */ struct wiggleDataStream *wds = NULL; int operations = wigFetchAscii; operations = wigFetchAscii; wds = wiggleDataStreamNew(); wds->getData(wds, database, table, operations); wds->asciiOut(wds, database, outFile, TRUE, FALSE); wiggleDataStreamFree(&wds); } void makeDownloadSessionCtData(char *param1, char *backgroundProgress) /* Download tables and data to save save in compressed archive. */ { char query[512]; char **row = NULL; struct sqlResult *sr = NULL; // Initialize .progress channel file -struct dyString *dyProg = newDyString(256); +struct dyString *dyProg = dyStringNew(256); dyStringPrintf(dyProg, "please wait, dumping data to archive ...<br>\n"); updateProgessFile(backgroundProgress, dyProg); lazarusLives(20 * 60); htmlOpen("Preparing Custom Tracks Backup Archive for Download"); jsInit(); struct sqlConnection *conn = hConnectCentral(); printf("<FORM ACTION=\"%s\" NAME=\"mainForm\" METHOD=POST " "ENCTYPE=\"multipart/form-data\">\n", hgSessionName()); cartSaveSession(cart); sqlSafef(query, sizeof(query), "SELECT contents from %s " "WHERE id=%u", "sessionDb", cartSessionRawId(cart)); sr = sqlGetResult(conn, query); // setup temp output dir: // ../trash/ // ctBackup/ // random@#$%/ # random value so it is unique /* make a directory, including parent directories */ char tempOut[1024]; safef(tempOut, sizeof tempOut, "../trash/ctBackup"); if (!fileExists(tempOut)) { makeDir(tempOut); } char tempOutRand[1024]; // make randomDir char *randPath=NULL; int randCount = 0; while(TRUE) { randPath=makeRandomKey(128+33); // at least 128 bits of protection, 33 for the world population size. // Avoid the possibility of somehow doing more than one instance of the session // backup at the same time. // On the extremely rare chance that the directory already exists, // just call the function again. safef(tempOutRand, sizeof tempOutRand, "%s/%s", tempOut, randPath); if (makeDir(tempOutRand)) break; ++randCount; if (randCount > 100) // should never happen errAbort("unable to create random output dir."); } int ctCount = 0; int foundCount = 0; if ((row = sqlNextRow(sr)) != NULL) { ++foundCount; char *contents = row[0]; char *trackHubsVar = NULL; struct downloadResults *resultList = processCtsForDownloadInternals(contents, &trackHubsVar); struct downloadResults *result = NULL; for (result=resultList; result; result=result->next) { struct customTrack *cts = result->cts, *track = NULL; struct ctExtra *extras = result->ctExtras, *extra = NULL; // empty db dir can trigger cleaning out of cart ct var for db // if it has no ct.bed char outDbDir[1024]; safef(outDbDir, sizeof outDbDir, "%s/%s", tempOutRand, result->db); if (!fileExists(outDbDir)) { makeDir(outDbDir); } if (!result->ctPath) // ct.bed was missing for the db continue; char *hubUrl = NULL; if (startsWith("hub_", result->db)) { unsigned hubId = isLiveHub(result->db, &hubUrl, FALSE); if (hubId == 0) { continue; // skip hub if not active. } } else { if (!isActiveDb(result->db)) { continue; // skip db if not active. } } printf("<h3>%s</h3>\n", result->db); dyStringPrintf(dyProg,"<h3>%s</h3>\n", result->db); updateProgessFile(backgroundProgress, dyProg); lazarusLives(20 * 60); if (startsWith("hub_", result->db)) { // Save the hubUrl now in case it gets restored on another machine // that does NOT have that hub loaded on it. // save output as hubUrl text file char hubUrlPath[1024]; safef(hubUrlPath, sizeof hubUrlPath, "%s/hubUrl", outDbDir); FILE *f = mustOpen(hubUrlPath, "w"); fprintf(f, "%s", hubUrl); // NO NEWLINE carefulClose(&f); } char cwd[PATH_LEN]; if (!getcwd(cwd, sizeof(cwd))) errnoAbort("unable to get current dir."); for (track=cts,extra=extras; track; track=track->next,extra=extra->next) { boolean wibMissing = track->wibFile && !fileExists(track->wibFile); if (extra->bigDataUrl || (track->dbTrack && track->dbDataLoad && track->dbTableName && !wibMissing)) { ++ctCount; printf("%s <br>\n", extra->name); dyStringPrintf(dyProg, "%s <br>\n", extra->name); updateProgessFile(backgroundProgress, dyProg); lazarusLives(20 * 60); char outNameCt[2014]; safef(outNameCt, sizeof outNameCt, "%s/%s.ct", outDbDir, extra->name); FILE *fct = mustOpen(outNameCt, "w"); // write the track header if (extra->browserLines) fprintf(fct, "%s", extra->browserLines); // should have ; converted \n already if (!extra->trackLine) errAbort("origTrackLine is NULL!"); fprintf(fct, "%s\n", extra->trackLine); if (!extra->bigDataUrl) { // handle wiggle cts which have an additional wig binary // wibFile='../trash/ct/hgtct_genome_542_dc1750.wib' // wibFile='../trash/ct/ct_hgwdev_galt_e83f_3892d0.maf' // wibFile='../trash/ct/ct_hgwdev_galt_4ba3_415cc0.vcf' // symlink for speed, file should not change if (track->wibFile) { if (endsWith(track->wibFile, ".wib")) { // dump wig as ascii char outNameWig[2014]; safef(outNameWig, sizeof outNameWig, "%s/%s.wig", outDbDir, extra->name); doOutWigData(track->dbTableName, CUSTOM_TRASH, outNameWig); // no easy way to append it to .ct directly // append text to ct appendTrashFileToCt(fct, outNameWig); remove(outNameWig); } if (endsWith(track->wibFile, ".maf")) { // append text to ct appendTrashFileToCt(fct, track->wibFile); } if (endsWith(track->wibFile, ".vcf")) { // append text to ct appendTrashFileToCt(fct, track->wibFile); } } else // simple BED-like { saveSqlDataForTable(track->dbTableName, fct); } } // handle extra htmlFile track doc // htmlFile=../trash/ct/ct_hgwdev_galt_d011_383c50.html // symlink for speed, file should not change if (track->htmlFile && fileExists(track->htmlFile)) { makeTrashFileLink(track->htmlFile, cwd, outDbDir, extra->name); } carefulClose(&fct); } } } } sqlFreeResult(&sr); if (foundCount == 0) errAbort("No session found for hgsid=%u", cartSessionRawId(cart)); if (ctCount == 0) errAbort("No custom tracks found for hgsid=%u", cartSessionRawId(cart)); char archiveName[1024]; safef(archiveName, sizeof archiveName, "savedSessionCtRaw.tar.gz"); dyStringPrintf(dyProg, "<br>\n"); int saveDySize = dyProg->stringSize; dyStringPrintf(dyProg, "creating and compressing archive %s <br>\n", archiveName); updateProgessFile(backgroundProgress, dyProg); lazarusLives(20 * 60); // create the archive char *cwd = cloneString(getCurrentDir()); setCurrentDir(tempOutRand); char excludeBuf[4096]; safef(excludeBuf, sizeof excludeBuf, "--exclude=%s", archiveName); char *pipeCmd1[] = { "tar", "-zcphf", archiveName, ".", excludeBuf, NULL}; struct pipeline *pl = pipelineOpen1(pipeCmd1, pipelineWrite | pipelineNoAbort, "/dev/null", NULL, 0); int sysVal = pipelineWait(pl); setCurrentDir(cwd); if (!((sysVal == 0) || (sysVal == 1))) // we tolerate 1 because tar doesn't like us creating the archive in the directory we're backing up errAbort("System call returned %d for:\n %s", sysVal, pipelineDesc(pl)); dyProg->stringSize = saveDySize; // restore prev size, popping. dyProg->string[dyProg->stringSize] = 0; dyStringPrintf(dyProg, "archive %s created <br>\n", archiveName); updateProgessFile(backgroundProgress, dyProg); lazarusLives(20 * 60); char downPath[1024]; safef(downPath, sizeof downPath, "%s/%s", tempOutRand, archiveName); printf("<br>\n"); char *encDownPath = cgiEncodeFull(downPath); char downButName[1024]; safef(downButName, sizeof(downButName), "%s%s", hgsDoDownloadPrefix, encDownPath); // using getElementById('%s') here is a work-around for special charachters in id. // I tried escaping, but it did not seem to work with our jQuery version. // mainly the special characters it sees are periods and slashes. char downButNameHack[1024]; safef(downButNameHack, sizeof(downButNameHack), "document.getElementById('%s')", downButName); printf("Download archive as local file:\n"); cgiMakeOnKeypressTextVar(hgsSaveLocalBackupFileName, cartUsualString(cart, hgsSaveLocalBackupFileName, ""), 20, jsPressOnEnter(downButNameHack)); printf(".tar.gz"); char greek[32]; sprintWithGreekByte(greek, sizeof(greek), fileSize(downPath)); printf(" %s ", greek); printf("<input type='submit' name='%s' id='%s' value='download backup archive'>", downButName, downButName); // check the file name, make sure it is not blank char js[1024]; safef(js, sizeof js, "var textVar = document.getElementById('%s');" "if (textVar.value != '')" " return true;" "alert('Please enter a filename for the backup archive');" "return false;" , hgsSaveLocalBackupFileName); jsOnEventById("click", downButName, js); // Add a button that just returns to the saved sessions list. // because after pressing download and it will go into the users downloads folder, // the browser remains sitting on this page and needs a way to return // to the saved sessions list. Also clear the download name field. printf(" "); printf("<input type='submit' id='hgsReturn' value='return'>"); safef(js, sizeof js, "var textVar = document.getElementById('%s');" "textVar.value = '';" "return true;" , hgsSaveLocalBackupFileName); jsOnEventById("click", "hgsReturn", js); printf("<br>\n"); printf("<br>\n"); hDisconnectCentral(&conn); printf("</FORM>\n"); htmlClose(); fflush(stdout); } void doDownloadSessionCtData(struct hashEl *downloadPathList) /* Download given table to browser to save. */ { struct hashEl *hel = NULL; // I think it should only get one at a time, not a list. hel = downloadPathList; char *encDownPath = hel->name + strlen(hgsDoDownloadPrefix); char *downPath = cgiDecodeClone(encDownPath); char *fileName = cartString(cart, hgsSaveLocalBackupFileName); char outFile[1024]; safef(outFile, sizeof outFile, "%s.tar.gz", fileName); long fSize = fileSize(downPath); printf("Content-Type: application/octet-stream\n"); printf("Content-Disposition: attachment; filename=\"%s\"\n", outFile); printf("Content-Length: %ld\n", fSize); printf("\n"); FILE *f = mustOpen(downPath, "r"); long remaining = fSize; int bufSize = 65536; char *buf = needMem(bufSize); while (remaining) { int bufRemain = bufSize; if (bufRemain > remaining) bufRemain = remaining; mustRead(f, buf, bufRemain); // mustRead OK since we are reading from disk mustWrite(stdout, buf, bufRemain); remaining -= bufRemain; lazarusLives(20 * 60); // extend keep-alive time. for big downloads on slow connections. } carefulClose(&f); cartRemove(cart, hgsSaveLocalBackupFileName); }