a53b9958fa734f73aeffb9ddfe2fbad1ca65f90c galt Mon Jan 30 16:18:41 2017 -0800 Check-in of CSP2 Content-Security-Policy work. All C-language CGIs should now support CSP2 in browser to stop major forms of XSS javascript injection. Javascript on pages is gathered together, and then emitted in a single script block at the end with a nonce that tells the browser, this is js that we generated instead of being injected by a hacker. Both inline script from script blocks and inline js event handlers had to be pulled out and separated. You will not see js sprinkled through-out the page now. Older browsers that support CSP1 or that do not understand CSP at all will still work, just without protection. External js libraries loaded at runtime need to be added to the CSP policy header in src/lib/htmshell.c. diff --git src/hg/lib/fileUi.c src/hg/lib/fileUi.c index 2481dc0..a730971 100644 --- src/hg/lib/fileUi.c +++ src/hg/lib/fileUi.c @@ -1,1198 +1,1203 @@ /* fileUi.c - human genome file downloads common controls. */ /* Copyright (C) 2014 The Regents of the University of California * See README in this or parent directory for licensing information. */ #include "common.h" #include "hash.h" #include "cheapcgi.h" #include "jsHelper.h" #include "cart.h" #include "hdb.h" #include "fileUi.h" #include "hui.h" #include "obscure.h" #include "mdb.h" #include "jsHelper.h" #include "web.h" #include "trashDir.h" static boolean timeIt = FALSE; // Can remove when satisfied with timing. void fileDbFree(struct fileDb **pFileList) // free one or more fileDb objects { while (pFileList && *pFileList) { struct fileDb *oneFile = slPopHead(pFileList); freeMem(oneFile->fileName); freeMem(oneFile->fileType); freeMem(oneFile->fileDate); freeMem(oneFile->sortFields); freeMem(oneFile->reverse); mdbObjsFree(&(oneFile->mdb)); freeMem(oneFile); } } static struct fileDb *fileDbReadFromBackup(char *db, char *dir, char *subDir) { struct tempName buFile; boolean exists = trashDirReusableFile(&buFile, dir, subDir, db); // encodeDCC/composite.db if (!exists) return NULL; struct fileDb *fileList = NULL; struct fileDb *oneFile = NULL; struct lineFile *lf = lineFileOpen(buFile.forCgi, TRUE); char *words[4]; while (lineFileChop(lf, words) >= 3) { AllocVar(oneFile); oneFile->fileName = cloneString(words[0]); oneFile->fileSize = sqlUnsignedLong(words[1]); oneFile->fileDate = cloneString(words[2]); slAddHead(&fileList,oneFile); } lineFileClose(&lf); if (fileList == NULL) unlink(buFile.forCgi); // remove empty file return fileList; } // Cache is not faster, so just use it as a backup #define CACHE_IS_FASTER_THAN_RSYNC #ifdef CACHE_IS_FASTER_THAN_RSYNC static boolean fileDbBackupAvailable(char *db, char *dir, char *subDir) // Checks if there is a recent enough cache file { if (cgiVarExists("clearCache")) // Trick to invalidate cache at will. return FALSE; struct tempName buFile; boolean exists = trashDirReusableFile(&buFile, dir, subDir, db); // encodeDCC/composite.db if (exists) { struct stat mystat; ZeroVar(&mystat); if (stat(buFile.forCgi,&mystat)==0) { // how old is old? int secs = (clock1() - mystat.st_ctime); // seconds since created if (secs < (24 * 60 * 60)) // one date return TRUE; } } return FALSE; } #endif///def CACHE_IS_FASTER_THAN_RSYNC static void fileDbWriteToBackup(char *db, char *dir, char *subDir,struct fileDb *fileList) { struct tempName buFile; (void)trashDirReusableFile(&buFile, dir, subDir, db); // encodeDCC/composite.db FILE *fd = NULL; if ((fd = fopen(buFile.forCgi, "w")) != NULL) { struct fileDb *oneFile = fileList; for (;oneFile != NULL;oneFile=oneFile->next) { char buf[1024]; safef(buf,sizeof buf,"%s %ld %s\n",oneFile->fileName,oneFile->fileSize,oneFile->fileDate); fwrite(buf, strlen(buf), 1, fd); } fclose(fd); } } struct fileDb *fileDbGet(char *db, char *dir, char *subDir, char *fileName) // Returns NULL or if found a fileDb struct with name, size and date filled in. { static char *savedDb = NULL; static char *savedDir = NULL; static char *savedSubDir = NULL; static struct fileDb *foundFiles = NULL;// Use static list to save excess IO struct fileDb *oneFile = NULL; if (foundFiles == NULL || savedDb == NULL || differentString(savedDb, db) || savedDir == NULL || differentString(savedDir,dir) || savedSubDir == NULL || differentString(savedSubDir,subDir)) { // free up any static mem freeMem(savedDb); freeMem(savedDir); freeMem(savedSubDir); fileDbFree(&foundFiles); #ifdef CACHE_IS_FASTER_THAN_RSYNC if (fileDbBackupAvailable(db, dir, subDir)) // check backup first foundFiles = fileDbReadFromBackup(db, dir, subDir); else #endif///def CACHE_IS_FASTER_THAN_RSYNC { FILE *scriptOutput = NULL; char buf[1024]; char cmd[512]; char *words[10]; char *server = hDownloadsServer(); // Always uses rsync, even when same server, to ensure testability if (hIsBetaHost()) { // NOTE: Force this case because beta may think it's downloads server is // "hgdownload.cse.ucsc.edu" safef(cmd,sizeof(cmd),"rsync -n rsync://hgdownload-test.cse.ucsc.edu/goldenPath/" "%s/%s/%s/beta/", db, dir, subDir); } else safef(cmd,sizeof(cmd),"rsync -n rsync://%s/goldenPath/%s/%s/%s/", server, db, dir, subDir); scriptOutput = popen(cmd, "r"); while (fgets(buf, sizeof(buf), scriptOutput)) { eraseTrailingSpaces(buf); if (!endsWith(buf,".md5sum")) // Just ignore these { int count = chopLine(buf, words); if (count == 5) { //-rw-rw-r-- 26420982 2009/09/29 14:53:30 wgEncodeBroadChipSeq/wgEncode... // rsync 3.1 adds commas: //-rw-rw-r-- 26,420,982 2009/09/29 14:53:30 wgEncodeBroadChipSeq/wgEncode... AllocVar(oneFile); stripChar(words[1], ','); oneFile->fileSize = sqlUnsignedLong(words[1]); oneFile->fileDate = cloneString(words[2]); strSwapChar(oneFile->fileDate,'/','-');// Standardize YYYY-MM-DD, no time oneFile->fileName = cloneString(words[4]); slAddHead(&foundFiles,oneFile); } } } pclose(scriptOutput); if (foundFiles == NULL) { foundFiles = fileDbReadFromBackup(db, dir, subDir); if (foundFiles == NULL) { AllocVar(oneFile); oneFile->fileName = cloneString("No files found!"); oneFile->fileDate = cloneString(cmd); slAddHead(&foundFiles,oneFile); warn("No files found for command:\n%s",cmd); } } else { fileDbWriteToBackup(db, dir, subDir,foundFiles); if (timeIt) uglyTime("Successful rsync found %d files",slCount(foundFiles)); } } // mark this as done to avoid excessive io savedDb = cloneString(db); savedDir = cloneString(dir); savedSubDir = cloneString(subDir); if (foundFiles == NULL) return NULL; } // special code that only gets called in debug mode if (sameString(fileName,"listAll")) { for (oneFile=foundFiles;oneFile;oneFile=oneFile->next) warn("%s",oneFile->fileName); return NULL; } // Look up the file and return it struct fileDb *newList = NULL; while (foundFiles) { oneFile = slPopHead(&foundFiles); if (sameString(fileName,oneFile->fileName)) break; // Found means removed from list: shorter list for next file. slAddHead(&newList,oneFile); oneFile = NULL; } if (newList) foundFiles = slCat(foundFiles,newList); // Order does not remain the same if (oneFile != NULL && oneFile->fileType == NULL) { char *suffix = strchr(oneFile->fileName, '.'); if (suffix != NULL && strlen(suffix) > 2) { oneFile->fileType = cloneString(suffix + 1); if (endsWith(oneFile->fileType,".gz")) chopSuffix(oneFile->fileType); } } return oneFile; } static sortOrder_t *fileSortOrderGet(struct cart *cart,struct trackDb *parentTdb, struct mdbObj *mdbObjs) // Parses 'fileSortOrder' trackDb/cart instructions and returns a sort order struct or NULL. // Some trickiness here. sortOrder->sortOrder is from cart (changed by user action), // as is sortOrder->order, but columns are in original tdb order (unchanging)! // However, if cart is null, all is from trackDb.ra. { int ix; sortOrder_t *sortOrder = NULL; char *carveSetting = NULL,*setting = NULL; if (parentTdb) setting = trackDbSetting(parentTdb, FILE_SORT_ORDER); if (setting) { sortOrder = needMem(sizeof(sortOrder_t)); sortOrder->setting = cloneString(setting); carveSetting = sortOrder->setting; } else { if (mdbObjs != NULL) { struct dyString *dySortFields = dyStringNew(512); struct mdbObj *commonVars = mdbObjsCommonVars(mdbObjs); // common vars are already in cv defined order, searchable is also sortable struct mdbVar *var = commonVars->vars; for (;var != NULL; var = var->next) { if (differentWord(var->var,MDB_VAR_LAB_VERSION) // Exclude certain vars && differentWord(var->var,MDB_VAR_SOFTWARE_VERSION) && cvSearchMethod(var->var) != cvNotSearchable) // searchable is also sortable { if (mdbObjsHasCommonVar(mdbObjs, var->var,TRUE)) // Don't bother if all the vals continue; // are the same (missing okay) dyStringPrintf(dySortFields,"%s=%s ",var->var, strSwapChar(cloneString(cvLabel(NULL,var->var)),' ','_')); } } if (dyStringLen(dySortFields)) { dyStringAppend(dySortFields,"fileSize=Size fileType=File_Type"); setting = dyStringCannibalize(&dySortFields); } else dyStringFree(&dySortFields); mdbObjsFree(&commonVars); } if (setting == NULL) // Must be in trackDb or not a sortable list of files { #define FILE_SORT_ORDER_DEFAULT "cell=Cell_Line lab=Lab view=View replicate=Rep " \ "fileSize=Size fileType=File_Type dateSubmitted=Submitted " \ "dateUnrestricted=RESTRICTED<BR>Until" setting = FILE_SORT_ORDER_DEFAULT; } sortOrder = needMem(sizeof(sortOrder_t)); carveSetting = cloneString(setting); sortOrder->setting = NULL; } if (parentTdb) { sortOrder->htmlId = needMem(strlen(parentTdb->track)+20); safef(sortOrder->htmlId, (strlen(parentTdb->track)+20), "%s.%s", parentTdb->track,FILE_SORT_ORDER); if (cart != NULL) sortOrder->sortOrder = cloneString(cartOptionalString(cart, sortOrder->htmlId)); } sortOrder->count = chopByWhite(carveSetting,NULL,0); // Get size sortOrder->column = needMem(sortOrder->count*sizeof(char*)); sortOrder->count = chopByWhite(carveSetting,sortOrder->column,sortOrder->count); sortOrder->title = needMem(sortOrder->count*sizeof(char*)); sortOrder->forward = needMem(sortOrder->count*sizeof(boolean)); sortOrder->order = needMem(sortOrder->count*sizeof(int)); for (ix = 0; ix<sortOrder->count; ix++) { // separate out mtaDb var in sortColumn from title sortOrder->title[ix] = strchr(sortOrder->column[ix],'='); // Could be 'cell=Cell_Line' if (sortOrder->title[ix] != NULL) { sortOrder->title[ix][0] = '\0'; // +1 jumps to next char after '=' sortOrder->title[ix] = strSwapChar(sortOrder->title[ix]+1,'_',' '); } else sortOrder->title[ix] = sortOrder->column[ix]; // or could be just 'cell' // Sort order defaults to forward but may be found in a cart var sortOrder->order[ix] = ix+1; sortOrder->forward[ix] = TRUE; if (sortOrder->sortOrder != NULL) { // find tdb substr in cart current order string char *pos = stringIn(sortOrder->column[ix], sortOrder->sortOrder); if (pos != NULL && pos[strlen(sortOrder->column[ix])] == '=') { int ord=1; char* pos2 = sortOrder->sortOrder; for (;*pos2 && pos2 < pos;pos2++) { if (*pos2 == '=') // Discovering sort order in cart ord++; } sortOrder->forward[ix] = (pos[strlen(sortOrder->column[ix]) + 1] == '+'); sortOrder->order[ix] = ord; } } } if (sortOrder->sortOrder == NULL) sortOrder->sortOrder = cloneString(setting); // no order in cart, all power to trackDb return sortOrder; // NOTE cloneString:words[0]==*sortOrder->column[0] } // will be freed when sortOrder is freed static int fileDbSortCmp(const void *va, const void *vb) // Compare two sortable tdb items based upon sort columns. { const struct fileDb *a = *((struct fileDb **)va); const struct fileDb *b = *((struct fileDb **)vb); char **fieldsA = a->sortFields; char **fieldsB = b->sortFields; int ix=0; int compared = 0; while (fieldsA[ix] != NULL && fieldsB[ix] != NULL) { compared = strcmp(fieldsA[ix], fieldsB[ix]); if (compared != 0) { if (a->reverse[ix]) compared *= -1; break; } ix++; } return compared; } static void fileDbSortList(struct fileDb **fileList, sortOrder_t *sortOrder) // If sortOrder struct provided, will update sortFields in fileList, then sort it { if (sortOrder && fileList) { struct fileDb *oneFile = NULL; for (oneFile = *fileList;oneFile != NULL;oneFile=oneFile->next) { // + 1 Null terminated oneFile->sortFields = needMem(sizeof(char *) * (sortOrder->count + 1)); oneFile->reverse = needMem(sizeof(boolean *) * sortOrder->count); int ix; for (ix=0;ix<sortOrder->count;ix++) { char *field = NULL; if (sameString("fileSize",sortOrder->column[ix])) { char niceNumber[32]; sprintf(niceNumber, "%.15lu", oneFile->fileSize); field = cloneString(niceNumber); } else if (sameString("fileType",sortOrder->column[ix])) field = oneFile->fileType; else field = mdbObjFindValue(oneFile->mdb,sortOrder->column[ix]); if (field) { oneFile->sortFields[sortOrder->order[ix] - 1] = field; oneFile->reverse[ sortOrder->order[ix] - 1] = (sortOrder->forward[ix] == FALSE); } else { oneFile->sortFields[sortOrder->order[ix] - 1] = NULL; oneFile->reverse[ sortOrder->order[ix] - 1] = FALSE; } } oneFile->sortFields[sortOrder->count] = NULL; } slSort(fileList,fileDbSortCmp); } } static int removeCommonMdbVarsNotInSortOrder(struct mdbObj *mdbObjs,sortOrder_t *sortOrder) // Removes varaibles common to all mdbObjs and not found in sortOrder. // Returns count of vars removed { if (sortOrder != NULL) { // Remove common vars from mdbs grant=Bernstein; lab=Broad; dataType=ChipSeq; setType=exp; // However, keep the term if it is in the sortOrder int count = 0; struct dyString *dyCommon = dyStringNew(256); char *commonTerms[] = { "grant", "lab", "dataType", "control", "setType" }; int tIx=0,sIx = 0; for (;tIx<ArraySize(commonTerms);tIx++) { for (sIx = 0; sIx<sortOrder->count && differentString(commonTerms[tIx],sortOrder->column[sIx]); sIx++) ; if (sIx<sortOrder->count) // Found in sort Order so leave it in mdbObjs continue; if (mdbObjsHasCommonVar(mdbObjs, commonTerms[tIx], TRUE)) // val the same or missing { count++; dyStringPrintf(dyCommon,"%s ",commonTerms[tIx]); } } if (count > 0) mdbObjRemoveVars(mdbObjs,dyStringContents(dyCommon)); // removes from full list of mdbs dyStringFree(&dyCommon); return count; } return 0; } static char *labelWithVocabLink(char *var,char *title,struct slPair *valsAndLabels, boolean tagsNotVals) // If the parentTdb has a controlledVocabulary setting and the vocabType is found, // then label will be wrapped with the link to all relevent terms. Return string is cloned. { // Determine if the var is cvDefined. If not, simple link boolean cvDefined = FALSE; struct hash *cvTypesOfTerms = (struct hash *)cvTermTypeHash(); if (cvTypesOfTerms != NULL) { struct hash *cvTermDef = hashFindVal(cvTypesOfTerms,var); if (cvTermDef) cvDefined = SETTING_IS_ON(hashFindVal(cvTermDef,"cvDefined")); } struct dyString *dyLink = dyStringNew(256); if (!cvDefined) dyStringPrintf(dyLink,"<A HREF='hgEncodeVocab?type=%s' title='Click for details of \"%s\"'" " TARGET=ucscVocab>%s</A>",var,title,title); else { dyStringPrintf(dyLink,"<A HREF='hgEncodeVocab?%s=",tagsNotVals?"tag":"term"); struct slPair *oneVal = valsAndLabels; for (;oneVal!=NULL;oneVal=oneVal->next) { if (oneVal != valsAndLabels) dyStringAppendC(dyLink,','); dyStringAppend(dyLink,mdbPairVal(oneVal)); } dyStringPrintf(dyLink,"' title='Click for details of each \"%s\"' TARGET=ucscVocab>%s</A>", title,title); } return dyStringCannibalize(&dyLink); } static int filterBoxesForFilesList(char *db,struct mdbObj *mdbObjs,sortOrder_t *sortOrder) // Will create filterBoxes for each sortOrder field. Returns bitmask of sortOrder colums included { int count = 0; int filterableBits = 0; if (sortOrder != NULL) { struct dyString *dyFilters = dyStringNew(256); int sIx=0; for (sIx = 0;sIx<sortOrder->count;sIx++) { char *var = sortOrder->column[sIx]; enum cvSearchable searchBy = cvSearchMethod(var); if (searchBy == cvNotSearchable || searchBy == cvSearchByFreeText) continue; // Free text is not good candidate for filters. Best is single word/date/int. // get all vals for var, then convert to tag/label pairs for filterBys struct slName *vals = mdbObjsFindAllVals(mdbObjs, var, CV_LABEL_EMPTY_IS_NONE); if (searchBy != cvSearchByMultiSelect && searchBy != cvSearchBySingleSelect) { // We can't be too ambitious about creating filterboxes on the fly so some limitations: // If there are more than 80 options, the filterBy is way too large and of limited use // If there is a distinct val for each file in the table, then the filterBy is the // same size as the table and of no help. Really the number of options should be // half the number of rows but we are being lenient and cutting off at 0.8 not 0.5 // If there is any non-alphanum char in a value then the filterBy will fail in js code. // Those filterBy's are abandoned a but further down. int valCount = slCount(vals); if (valCount > 80 || valCount > (slCount(mdbObjs) * 0.8)) { slNameFreeList(&vals); continue; } } struct slPair *tagLabelPairs = NULL; while (vals != NULL) { char buf[256]; struct slName *term = slPopHead(&vals); char *tag = (char *)cvTag(var,term->name); if (tag == NULL) // Does not require cv defined! { safecpy(buf,sizeof buf,term->name); tag = buf; // Bad news if tag has special chars, eraseNonAlphaNum(tag); // unfortunately this does not pretect us from dups // filtering by terms not in cv or regularized should be // abandanded at the first sign of trouble! if (searchBy != cvSearchByMultiSelect && searchBy != cvSearchBySingleSelect && searchBy != cvSearchByDateRange) { if (strlen(term->name) > strlen(tag)) { slNameFreeList(&vals); slNameFree(&term); break; } } } slPairAdd(&tagLabelPairs,tag,cloneString((char *)cvLabel(var,term->name))); slNameFree(&term); } // If there is more than one val for this var then create filterBy box for it if (slCount(tagLabelPairs) > 1) { // should have a list sorted on the label enum cvDataType eCvDataType = cvDataType(var); if (eCvDataType == cvInteger) slPairValAtoiSort(&tagLabelPairs); else slPairValSortCase(&tagLabelPairs); char extraClasses[256]; safef(extraClasses,sizeof extraClasses,"filterTable %s",var); char *dropDownHtml = cgiMakeMultiSelectDropList(var,tagLabelPairs,NULL,"All", - extraClasses,"onchange='filterTable.filter(this);' style='font-size:.9em;'"); + extraClasses, "change", "filterTable.filter(this);", "font-size:.9em;", NULL); // Note filterBox has classes: filterBy & {var} if (dropDownHtml) { dyStringPrintf(dyFilters, "<td align='left'>\n<B>%s</B>:<BR>\n%s</td><td width=10> </td>\n", labelWithVocabLink(var,sortOrder->title[sIx],tagLabelPairs,TRUE), dropDownHtml); // TRUE were sending tags, not values freeMem(dropDownHtml); count++; if (sIx < 32) // avoid bit overflow but 32 filterBoxes? I don't think so filterableBits |= (0x1<<(sIx)); } } if (tagLabelPairs != NULL) slPairFreeValsAndList(&tagLabelPairs); } // Finally ready to print the filterBys out if (count) { webIncludeResourceFile("ui.dropdownchecklist.css"); jsIncludeFile("ui.dropdownchecklist.js",NULL); #define FILTERBY_HELP_LINK "<A HREF=\"../goldenPath/help/multiView.html\" " \ "TARGET=ucscHelp>help</A>" cgiDown(0.9); printf("<B>Filter files by:</B> (select multiple %sitems - %s)\n" "<table><tr valign='bottom'>\n", (count >= 1 ? "categories and ":""),FILTERBY_HELP_LINK); printf("%s\n",dyStringContents(dyFilters)); printf("</tr></table>\n"); jsIncludeFile("ddcl.js",NULL); } dyStringFree(&dyFilters); } return filterableBits; } static void filesDownloadsPreamble(char *db, struct trackDb *tdb, boolean isUnrestricted) // Replacement for preamble.html which should expose parent dir, files.txt and supplemental, but // not have any specialized notes per composite. Specialized notes belong in track description. { char *server = hDownloadsServer(); char *subDir = ""; if (hIsBetaHost()) { server = "hgdownload-test.cse.ucsc.edu"; // NOTE: Force this case because beta may think subDir = "/beta"; // it's downloads server is "hgdownload.cse.ucsc.edu" } if (!isUnrestricted) { cgiDown(0.9); puts("<B>Data is <A HREF='../ENCODE/terms.html' TARGET='_BLANK'>RESTRICTED FROM USE</a>"); puts("in publication until the restriction date noted for the given data file.</B>"); } cgiDown(0.7); puts("Additional resources:"); printf("<BR>• <B><A HREF='http://%s/goldenPath/%s/%s/%s%s/files.txt' " "TARGET=ucscDownloads>files.txt</A></B> - lists the name and metadata for each download.\n", server,db,ENCODE_DCC_DOWNLOADS, tdb->track, subDir); printf("<BR>• <B><A HREF='http://%s/goldenPath/%s/%s/%s%s/md5sum.txt' " "TARGET=ucscDownloads>md5sum.txt</A></B> - lists the md5sum output for each download.\n", server,db,ENCODE_DCC_DOWNLOADS, tdb->track, subDir); printf("<BR>• <B><A HREF='http://%s/goldenPath/%s/%s/%s%s'>downloads server</A></B> - " "alternative access to downloadable files (may include obsolete data).\n", server,db,ENCODE_DCC_DOWNLOADS, tdb->track, subDir); struct fileDb *oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, tdb->track, "supplemental"); if (oneFile != NULL) { printf("<BR>• <B><A HREF='http://%s/goldenPath/%s/%s/%s%s/supplemental/' " "TARGET=ucscDownloads>supplemental materials</A></B> - " "any related files provided by the laboratory.\n", server,db,ENCODE_DCC_DOWNLOADS, tdb->track, subDir); } } static int filesPrintTable(char *db, struct trackDb *parentTdb, struct fileDb *fileList, sortOrder_t *sortOrder,int filterable) // Prints filesList as a sortable table. Returns count { if (timeIt) uglyTime("Start table"); // Table class=sortable int columnCount = 0; +int butCount = 0; int restrictedColumn = 0; char *nowrap = (sortOrder->setting != NULL ? " nowrap":""); // Sort order trackDb setting found so rely on <BR> in titles for wrapping printf("<TABLE class='sortable' style='border: 2px outset #006600;'>\n"); printf("<THEAD class='sortable'>\n"); printf("<TR class='sortable' valign='bottom'>\n"); printf("<TD align='center' valign='center'> "); int filesCount = slCount(fileList); if (filesCount > 5) printf("<em><span class='filesCount'></span>%d files</em>",filesCount); // NOTE: This could be done to preserve sort order in cart. // However hgFileUi would need form OR changes would need to be ajaxed over // AND hgsid would be needed. //if (sortOrder) // printf("<INPUT TYPE=HIDDEN NAME='%s' class='sortOrder' VALUE=\"%s\">", // sortOrder->htmlId, sortOrder->sortOrder); printf("</TD>\n"); columnCount++; // Now the columns int curOrder = 0,ix=0; if (sortOrder) { curOrder = sortOrder->count; for (ix=0;ix<sortOrder->count;ix++) { char *align = (sameString("labVersion",sortOrder->column[ix]) || sameString("softwareVersion",sortOrder->column[ix]) ? " align='left'":""); printf("<TH class='sortable sort%d%s' %s%s%s>%s</TH>\n", sortOrder->order[ix],(sortOrder->forward[ix]?"":" sortRev"), (sameString("fileSize",sortOrder->column[ix])?"abbr='use' ":""), nowrap,align,sortOrder->title[ix]); // keeing track of sortOrder columnCount++; if (sameWord(sortOrder->column[ix],"dateUnrestricted")) restrictedColumn = columnCount; } } //#define INCLUDE_FILENAMES #ifndef INCLUDE_FILENAMES else #endif///defn INCLUDE_FILENAMES { printf("<TH class='sortable sort%d' nowrap>File Name</TH>\n",++curOrder); columnCount++; } printf("<TH class='sortable sort%d' align='left' nowrap>Additional Details</TH>\n",++curOrder); columnCount++; printf("</TR></THEAD>\n"); // Now the files... char *server = hDownloadsServer(); char *subDir = ""; if (hIsBetaHost()) { server = "hgdownload-test.cse.ucsc.edu"; // NOTE: Force this case because beta may think subDir = "/beta"; // it's downloads server is "hgdownload.cse.ucsc.edu" } struct fileDb *oneFile = fileList; printf("<TBODY class='sortable sorting'>\n"); // 'sorting' is a fib but it conveniently greys if (timeIt) // the list till the table is initialized. uglyTime("Finished column headers"); for (;oneFile!= NULL;oneFile=oneFile->next) { oneFile->mdb->next = NULL; // mdbs were in list for generating sortOrder, char *field = NULL; // but list no longer needed printf("<TR valign='top'%s>",(filterable != 0) ?" class='filterable'":""); // Download button printf("<TD nowrap>"); if (parentTdb) field = parentTdb->track; else { field = cloneString(mdbObjFindValue(oneFile->mdb,MDB_VAR_COMPOSITE)); mdbObjRemoveOneVar(oneFile->mdb,MDB_VAR_COMPOSITE,NULL); } assert(field != NULL); - printf("<input type='button' value='Download' onclick=\"window.location=" - "'http://%s/goldenPath/%s/%s/%s%s/%s';\" title='Download %s ...'>", - server,db,ENCODE_DCC_DOWNLOADS, field, subDir, oneFile->fileName, oneFile->fileName); + char id[256]; + safef(id, sizeof id, "ftpBut_%d", butCount++); + printf("<input type='button' id='%s' value='Download' title='Download %s ...'>", id, oneFile->fileName); + char javascript[1024]; + safef(javascript, sizeof javascript, "window.location='http://%s/goldenPath/%s/%s/%s%s/%s';" + ,server,db,ENCODE_DCC_DOWNLOADS, field, subDir, oneFile->fileName); + jsOnEventById("click", id, javascript); #define SHOW_FOLDER_FRO_COMPOSITE_DOWNLOADS #ifdef SHOW_FOLDER_FRO_COMPOSITE_DOWNLOADS if (parentTdb == NULL) printf(" <A href='../cgi-bin/hgFileUi?db=%s&g=%s' title='Navigate to downloads " "page for %s set...'><IMG SRC='../images/folderC.png'></a> ", db,field,field); #endif///def SHOW_FOLDER_FRO_COMPOSITE_DOWNLOADS printf("</TD>\n"); // Each of the pulled out mdb vars if (sortOrder) { for (ix=0;ix<sortOrder->count;ix++) { if (sameString("fileSize",sortOrder->column[ix])) { char niceNumber[128]; sprintWithGreekByte(niceNumber, sizeof(niceNumber), oneFile->fileSize); field = oneFile->sortFields[sortOrder->order[ix] - 1]; printf("<TD abbr='%s' align='right' nowrap>%s</td>",field,niceNumber); } else { field = oneFile->sortFields[sortOrder->order[ix] - 1]; boolean isFieldEmpty = isEmpty(field); struct hash *termHash = NULL; if (!isFieldEmpty) { termHash = (struct hash *)cvOneTermHash(sortOrder->column[ix],field); if (termHash && sameString(field,MDB_VAL_ENCODE_EDV_NONE)) isFieldEmpty = cvTermIsEmpty(sortOrder->column[ix],field); } char class[128]; class[0] = '\0'; if (filterable & (0x1<<ix)) { char *cleanClass = NULL; char buf[256]; if (isFieldEmpty) cleanClass = CV_LABEL_EMPTY_IS_NONE; else { if (termHash) cleanClass = (char *)hashFindVal((struct hash *)termHash,CV_TAG); if (cleanClass == NULL) { // This may not be needed because the filterBy code // already eliminated these. safecpy(buf,sizeof buf,field); cleanClass = buf; eraseNonAlphaNum(cleanClass); } } safef(class,sizeof class," class='%s %s'",sortOrder->column[ix],cleanClass); } char *align = (sameString("labVersion",sortOrder->column[ix]) || sameString("softwareVersion",sortOrder->column[ix]) ? " align='left'":" align='center'"); if (sameString("dateUnrestricted",sortOrder->column[ix]) && field && dateIsOld(field, MDB_ENCODE_DATE_FORMAT)) printf("<TD%s nowrap style='color: #BBBBBB;'%s>%s</td>",align,class,field); else { // use label if (!isFieldEmpty && termHash) { char *label = hashFindVal(termHash,CV_LABEL); if (label != NULL) field = label; } printf("<TD%s nowrap%s>%s</td>",align,class,isFieldEmpty?" ":field); } if (!sameString("fileType",sortOrder->column[ix])) mdbObjRemoveOneVar(oneFile->mdb,sortOrder->column[ix],NULL); } } } #ifndef INCLUDE_FILENAMES else #endif///ndef INCLUDE_FILENAMES { // fileName printf("<TD nowrap>%s",oneFile->fileName); } // Extras grant=Bernstein; lab=Broad; dataType=ChipSeq; setType=exp; control=std; field = mdbObjVarValPairsAsLine(oneFile->mdb,TRUE,FALSE); printf("<TD nowrap>%s</td>",field?field:" "); printf("</TR>\n"); } if (timeIt) uglyTime("Finished files"); printf("</TBODY><TFOOT class='bgLevel1'>\n"); printf("<TR valign='top'>"); // Restriction policy link in first column? if (restrictedColumn == 1) printf("<TH colspan=%d><A HREF='%s' TARGET=BLANK style='font-size:.9em;'>" "Restriction Policy</A></TH>", (columnCount - restrictedColumn), ENCODE_DATA_RELEASE_POLICY); printf("<TD colspan=%d> ", (restrictedColumn > 1 ? (restrictedColumn - 1) : columnCount)); // Total if (filesCount > 5) printf("<em><span class='filesCount'></span>%d files</em>",filesCount); // Restriction policy link in later column? if (restrictedColumn > 1) printf("</TD><TH colspan=%d align='left'><A HREF='%s' TARGET=BLANK style='font-size:.9em;'>" "Restriction Policy</A>", columnCount,ENCODE_DATA_RELEASE_POLICY); printf("</TD></TR>\n"); printf("</TFOOT></TABLE>\n"); if (parentTdb == NULL) - printf("<script type='text/javascript'>{$(document).ready(function() {" - "sortTable.initialize($('table.sortable')[0],true,true);});}</script>\n"); + jsInline("{$(document).ready(function() {" + "sortTable.initialize($('table.sortable')[0],true,true);});}\n"); if (timeIt) uglyTime("Finished table"); return filesCount; } static int filesFindInDir(char *db, struct mdbObj **pmdbFiles, struct fileDb **pFileList, char *fileType, int limit, boolean *exceededLimit, boolean *isUnrestricted) // Prints list of files in downloads directories matching mdb search terms. Returns count { int fileCount = 0; if (isUnrestricted != NULL) *isUnrestricted = TRUE; // Verify file existance and make fileList of those found struct fileDb *fileList = NULL, *oneFile = NULL; // Will contain found files struct mdbObj *mdbFiles = NULL; // Will caontain a list of mdbs for the found files struct mdbObj *mdbList = *pmdbFiles; while (mdbList && (limit == 0 || fileCount < limit)) { boolean found = FALSE; struct mdbObj *mdbFile = slPopHead(&mdbList); char *composite = mdbObjFindValue(mdbFile,MDB_VAR_COMPOSITE); if (composite == NULL) { mdbObjsFree(&mdbFile); continue; } // First for FileName char *fileName = mdbObjFindValue(mdbFile,MDB_VAR_FILENAME); if (fileName == NULL) { mdbObjsFree(&mdbFile); continue; } // Are any files still restricted access under ENCODE data policy ? if (isUnrestricted != NULL && *isUnrestricted) *isUnrestricted = mdbObjEncodeIsUnrestricted(mdbFile); struct slName *fileSet = slNameListFromComma(fileName); struct slName *md5Set = NULL; char *md5sums = mdbObjFindValue(mdbFile,MDB_VAR_MD5SUM); if (md5sums != NULL) md5Set = slNameListFromComma(md5sums); // Could be that "bai" is implicit with "bam" if ((slCount(fileSet) == 1) && endsWith(fileSet->name,".bam")) { char buf[512]; safef(buf,sizeof(buf),"%s.bai",fileSet->name); slNameAddTail(&fileSet, buf); } while (fileSet != NULL) { struct slName *file = slPopHead(&fileSet); struct slName *md5 = NULL; if (md5Set) md5 = slPopHead(&md5Set); oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, composite, file->name); if (oneFile == NULL) { slNameFree(&file); if (md5) slNameFree(&md5); continue; } //warn("%s == %s",fileType,oneFile->fileType); if (isEmpty(fileType) || sameWord(fileType,"Any") || ( oneFile->fileType && startsWithWordByDelimiter(fileType,'.',oneFile->fileType))) // Starts with! This ensures both bam and bam.bai are found. { slAddHead(&fileList,oneFile); if (found) // if already found then need two mdbObjs (assertable but this is metadata) oneFile->mdb = mdbObjClone(mdbFile); // Yes clone this as differences will occur else oneFile->mdb = mdbFile; if (md5 != NULL) mdbObjSetVar(oneFile->mdb,MDB_VAR_MD5SUM,md5->name); else mdbObjRemoveOneVar(oneFile->mdb,MDB_VAR_MD5SUM,NULL); slAddHead(&mdbFiles,oneFile->mdb); found = TRUE; fileCount++; if (limit > 0 && fileCount >= limit) { slNameFreeList(&fileSet); if (md5Set) slNameFreeList(&md5Set); break; } } else fileDbFree(&oneFile); slNameFree(&file); if (md5) slNameFree(&md5); } // FIXME: This support of fileIndex should be removed when mdb is cleaned up. // Now for FileIndexes if (limit == 0 || fileCount < limit) { fileName = mdbObjFindValue(mdbFile,MDB_VAR_FILEINDEX);// This mdb var should be going away. if (fileName == NULL) continue; // Verify existance first oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, composite, fileName); if (oneFile == NULL) // NOTE: won't be found if already found in comma delimited fileName! continue; if (isEmpty(fileType) || sameWord(fileType,"Any") || (oneFile->fileType && sameWord(fileType,oneFile->fileType)) || (oneFile->fileType && sameWord(fileType,"bam") && sameWord("bam.bai",oneFile->fileType))) { // TODO: put fileType matching into search.c lib code to segregate index logic. slAddHead(&fileList,oneFile); if (found) // if already found then need two mdbObjs (assertable but this is metadata) oneFile->mdb = mdbObjClone(mdbFile); else oneFile->mdb = mdbFile; mdbObjRemoveOneVar(oneFile->mdb,MDB_VAR_MD5SUM,NULL); slAddHead(&mdbFiles,oneFile->mdb); fileCount++; found = TRUE; continue; } else fileDbFree(&oneFile); } // FIXME: This support of fileIndex should be removed when mdb is cleaned up. if (!found) mdbObjsFree(&mdbFile); } *pmdbFiles = mdbFiles; *pFileList = fileList; if (exceededLimit != NULL) *exceededLimit = FALSE; if (mdbList != NULL) { if (exceededLimit != NULL) *exceededLimit = TRUE; mdbObjsFree(&mdbList); } return fileCount; } void filesDownloadUi(char *db, struct cart *cart, struct trackDb *tdb) // UI for a "composite like" track: This will list downloadable files associated with // a single trackDb entry (composite or of type "downloadsOnly". The list of files // will have links to their download and have metadata information associated. // The list will be a sortable table and there may be filtering controls. { timeIt = cartUsualBoolean(cart, "measureTiming",FALSE); // static to file if (timeIt) uglyTime("Starting file search"); boolean debug = cartUsualBoolean(cart,"debug",FALSE); struct sqlConnection *conn = hAllocConn(db); char *mdbTable = mdbTableName(conn,TRUE); // Look for sandBox name first if (mdbTable == NULL) errAbort("TABLE NOT FOUND: '%s.%s'.\n",db,MDB_DEFAULT_NAME); // Get an mdbObj list of all that belong to this track and have a fileName char buf[256]; safef(buf,sizeof(buf),"%s=%s %s=?",MDB_VAR_COMPOSITE,tdb->track,MDB_VAR_FILENAME); struct mdbByVar *mdbVars = mdbByVarsLineParse(buf); struct mdbObj *mdbList = mdbObjsQueryByVars(conn,mdbTable,mdbVars); // Now get Indexes But be sure not to duplicate entries in the list!!! safef(buf,sizeof(buf),"%s=%s %s= %s!=",MDB_VAR_COMPOSITE,tdb->track, MDB_VAR_FILEINDEX,MDB_VAR_FILENAME); mdbVars = mdbByVarsLineParse(buf); mdbList = slCat(mdbList, mdbObjsQueryByVars(conn,mdbTable,mdbVars)); mdbObjRemoveHiddenVars(mdbList); hFreeConn(&conn); if (mdbList) (void)mdbObjsFilter(&mdbList,"objStatus","re*",TRUE); // revoked, replaced, renamed if (slCount(mdbList) == 0) { warn("No files specified in metadata for: %s\n%s",tdb->track,tdb->longLabel); return; } if (timeIt) uglyTime("Found %d mdb objects",slCount(mdbList)); // Verify file existance and make fileList of those found struct fileDb *fileList = NULL; // Will contain found files boolean isUnrestricted; int fileCount = filesFindInDir(db, &mdbList, &fileList, NULL, 0, NULL, &isUnrestricted); if (timeIt) uglyTime("Found %d files in dir",fileCount); assert(fileCount == slCount(fileList)); if (fileCount == 0) { warn("No downloadable files currently available for: %s\n%s",tdb->track,tdb->longLabel); return; // No files so nothing to do. } if (debug) { warn("The following files are in goldenPath/%s/%s/%s/ but NOT in the mdb:", db,ENCODE_DCC_DOWNLOADS, tdb->track); fileDbGet(db, ENCODE_DCC_DOWNLOADS, tdb->track, "listAll"); } jsIncludeFile("hui.js",NULL); jsIncludeFile("ajax.js",NULL); // standard preamble filesDownloadsPreamble(db,tdb, isUnrestricted); // remove these now to get them out of the way mdbObjRemoveVars(mdbList,MDB_VAR_FILENAME " " MDB_VAR_FILEINDEX " " MDB_VAR_COMPOSITE " " MDB_VAR_PROJECT); if (timeIt) uglyTime("<BR>Removed 4 unwanted vars"); // Now update all files with their sortable fields and sort the list mdbObjReorderByCv(mdbList,FALSE);// Start with cv defined order for visible vars. // NOTE: will not need to reorder during print! sortOrder_t *sortOrder = fileSortOrderGet(cart,tdb,mdbList); int filterable = 0; if (sortOrder != NULL) { int removed = removeCommonMdbVarsNotInSortOrder(mdbList,sortOrder); if (removed && debug) warn("%d terms are common and were removed",removed); if (timeIt) uglyTime("Removed %d common vars",removed); // Fill in and sort fileList fileDbSortList(&fileList,sortOrder); if (timeIt) uglyTime("Sorted %d files on %d columns",fileCount,sortOrder->count); // FilterBoxes filterable = filterBoxesForFilesList(db,mdbList,sortOrder); if (timeIt && filterable) uglyTime("Created filter boxes"); } // Print table filesPrintTable(db,tdb,fileList,sortOrder,filterable); //fileDbFree(&fileList); // Why bother on this very long running cgi? //mdbObjsFree(&mdbList); } int fileSearchResults(char *db, struct sqlConnection *conn, struct cart *cart, struct slPair *varValPairs, char *fileType) // Prints list of files in downloads directories matching mdb search terms. Returns count { timeIt = cartUsualBoolean(cart, "measureTiming",FALSE); // static to file if (timeIt) uglyTime("Starting file search"); struct sqlConnection *connLocal = conn; if (conn == NULL) connLocal = hAllocConn(db); struct mdbObj *mdbList = mdbObjRepeatedSearch(connLocal,varValPairs,FALSE,TRUE); if (conn == NULL) hFreeConn(&connLocal); mdbObjRemoveHiddenVars(mdbList); if (mdbList) (void)mdbObjsFilter(&mdbList,"objStatus","re*",TRUE); // revoked, replaced, renamed if (slCount(mdbList) == 0) { printf("<DIV id='filesFound'><BR>No files found.<BR></DIV><BR>\n"); return 0; } if (timeIt) uglyTime("Found %d mdb objects",slCount(mdbList)); // Now sort mdbObjs so that composites will stay together & lookup of files will be most efficient mdbObjsSortOnVars(&mdbList, MDB_VAR_COMPOSITE); #define FOUND_FILE_LIMIT 1000 struct fileDb *fileList = NULL; // Will contain found files int filesExpected = slCount(mdbList); boolean exceededLimit = FALSE; int fileCount = filesFindInDir(db, &mdbList, &fileList, fileType, FOUND_FILE_LIMIT, &exceededLimit, NULL); if (timeIt) uglyTime("Found %d files in dir",fileCount); assert(fileCount == slCount(fileList)); if (fileCount == 0) { printf("<DIV id='filesFound'><BR>No files found.<BR></DIV><BR>\n"); return 0; // No files so nothing to do. } // remove these now to get them out of the way mdbObjRemoveVars(mdbList,MDB_VAR_FILENAME " " MDB_VAR_FILEINDEX " " MDB_VAR_PROJECT " " MDB_VAR_TABLENAME); if (timeIt) uglyTime("Removed 4 unwanted vars"); // Now update all files with their sortable fields and sort the list mdbObjReorderByCv(mdbList,FALSE);// Start with cv defined order for visible vars. // NOTE: will not need to reorder during print! sortOrder_t *sortOrder = fileSortOrderGet(NULL,NULL,mdbList); // No cart, no tdb if (sortOrder != NULL) { // Fill in and sort fileList fileDbSortList(&fileList,sortOrder); if (timeIt) uglyTime("Sorted %d files on %d columns",fileCount,sortOrder->count); } // Print table printf("<DIV id='filesFound'>"); if (exceededLimit) { // What is the expected count? Difficult to say because of comma delimited list in fileName. if (filesExpected <= FOUND_FILE_LIMIT) filesExpected = FOUND_FILE_LIMIT + 1; printf("<DIV class='redBox' style='width: 380px;'>Too many files found. Displaying first " "%d of at least %d.<BR>Narrow search parameters and try again.</DIV><BR>\n", fileCount,filesExpected); } // 0=No columns 'filtered' on file search page fileCount = filesPrintTable(db,NULL,fileList,sortOrder,0); printf("</DIV><BR>\n"); //fileDbFree(&fileList); // Why bother on this very long running cgi? //mdbObjsFree(&mdbList); return fileCount; }