533112afe2a2005e80cdb1f82904ea65032d4302 braney Sat Oct 2 11:37:34 2021 -0700 split hg/lib into two separate libaries, one only used by the cgis diff --git src/hg/cgilib/fileUi.c src/hg/cgilib/fileUi.c new file mode 100644 index 0000000..21f0d03 --- /dev/null +++ src/hg/cgilib/fileUi.c @@ -0,0 +1,1201 @@ +/* fileUi.c - human genome file downloads common controls. */ + +/* Copyright (C) 2014 The Regents of the University of California + * See README in this or parent directory for licensing information. */ + +#include "common.h" +#include "hash.h" +#include "cheapcgi.h" +#include "jsHelper.h" +#include "cart.h" +#include "hdb.h" +#include "fileUi.h" +#include "hui.h" +#include "obscure.h" +#include "mdb.h" +#include "jsHelper.h" +#include "web.h" +#include "trashDir.h" + + +static boolean timeIt = FALSE; // Can remove when satisfied with timing. + +void fileDbFree(struct fileDb **pFileList) +// free one or more fileDb objects +{ +while (pFileList && *pFileList) + { + struct fileDb *oneFile = slPopHead(pFileList); + + freeMem(oneFile->fileName); + freeMem(oneFile->fileType); + freeMem(oneFile->fileDate); + freeMem(oneFile->sortFields); + freeMem(oneFile->reverse); + mdbObjsFree(&(oneFile->mdb)); + freeMem(oneFile); + } +} + +static struct fileDb *fileDbReadFromBackup(char *db, char *dir, char *subDir) +{ +struct tempName buFile; +boolean exists = trashDirReusableFile(&buFile, dir, subDir, db); // encodeDCC/composite.db +if (!exists) + return NULL; + +struct fileDb *fileList = NULL; +struct fileDb *oneFile = NULL; +struct lineFile *lf = lineFileOpen(buFile.forCgi, TRUE); +char *words[4]; +while (lineFileChop(lf, words) >= 3) + { + AllocVar(oneFile); + oneFile->fileName = cloneString(words[0]); + oneFile->fileSize = sqlUnsignedLong(words[1]); + oneFile->fileDate = cloneString(words[2]); + slAddHead(&fileList,oneFile); + } +lineFileClose(&lf); +if (fileList == NULL) + unlink(buFile.forCgi); // remove empty file + +return fileList; +} + +// Cache is not faster, so just use it as a backup +#define CACHE_IS_FASTER_THAN_RSYNC +#ifdef CACHE_IS_FASTER_THAN_RSYNC +static boolean fileDbBackupAvailable(char *db, char *dir, char *subDir) +// Checks if there is a recent enough cache file +{ +if (cgiVarExists("clearCache")) // Trick to invalidate cache at will. + return FALSE; + +struct tempName buFile; +boolean exists = trashDirReusableFile(&buFile, dir, subDir, db); // encodeDCC/composite.db +if (exists) + { + struct stat mystat; + ZeroVar(&mystat); + if (stat(buFile.forCgi,&mystat)==0) + { + // how old is old? + int secs = (clock1() - mystat.st_ctime); // seconds since created + if (secs < (24 * 60 * 60)) // one date + return TRUE; + } + } +return FALSE; +} +#endif///def CACHE_IS_FASTER_THAN_RSYNC + +static void fileDbWriteToBackup(char *db, char *dir, char *subDir,struct fileDb *fileList) +{ +struct tempName buFile; +(void)trashDirReusableFile(&buFile, dir, subDir, db); // encodeDCC/composite.db + +FILE *fd = NULL; +if ((fd = fopen(buFile.forCgi, "w")) != NULL) + { + struct fileDb *oneFile = fileList; + for (;oneFile != NULL;oneFile=oneFile->next) + { + char buf[1024]; + safef(buf,sizeof buf,"%s %ld %s\n",oneFile->fileName,oneFile->fileSize,oneFile->fileDate); + fwrite(buf, strlen(buf), 1, fd); + } + fclose(fd); + } +} + +struct fileDb *fileDbGet(char *db, char *dir, char *subDir, char *fileName) +// Returns NULL or if found a fileDb struct with name, size and date filled in. +{ +static char *savedDb = NULL; +static char *savedDir = NULL; +static char *savedSubDir = NULL; +static struct fileDb *foundFiles = NULL;// Use static list to save excess IO +struct fileDb *oneFile = NULL; +if (foundFiles == NULL +|| savedDb == NULL || differentString(savedDb, db) +|| savedDir == NULL || differentString(savedDir,dir) +|| savedSubDir == NULL || differentString(savedSubDir,subDir)) + { + // free up any static mem + freeMem(savedDb); + freeMem(savedDir); + freeMem(savedSubDir); + fileDbFree(&foundFiles); + +#ifdef CACHE_IS_FASTER_THAN_RSYNC + if (fileDbBackupAvailable(db, dir, subDir)) // check backup first + foundFiles = fileDbReadFromBackup(db, dir, subDir); + else +#endif///def CACHE_IS_FASTER_THAN_RSYNC + { + FILE *scriptOutput = NULL; + char buf[1024]; + char cmd[512]; + char *words[10]; + char *server = hDownloadsServer(); + + // Always uses rsync, even when same server, to ensure testability + if (hIsBetaHost()) + { + // NOTE: Force this case because beta may think it's downloads server is + // "hgdownload.soe.ucsc.edu" + safef(cmd,sizeof(cmd),"rsync -n rsync://hgdownload-test.soe.ucsc.edu/goldenPath/" + "%s/%s/%s/beta/", db, dir, subDir); + } + else + safef(cmd,sizeof(cmd),"rsync -n rsync://%s/goldenPath/%s/%s/%s/", + server, db, dir, subDir); + + scriptOutput = popen(cmd, "r"); + while (fgets(buf, sizeof(buf), scriptOutput)) + { + eraseTrailingSpaces(buf); + if (!endsWith(buf,".md5sum")) // Just ignore these + { + int count = chopLine(buf, words); + if (count == 5) + { + //-rw-rw-r-- 26420982 2009/09/29 14:53:30 wgEncodeBroadChipSeq/wgEncode... + // rsync 3.1 adds commas: + //-rw-rw-r-- 26,420,982 2009/09/29 14:53:30 wgEncodeBroadChipSeq/wgEncode... + AllocVar(oneFile); + stripChar(words[1], ','); + oneFile->fileSize = sqlUnsignedLong(words[1]); + oneFile->fileDate = cloneString(words[2]); + strSwapChar(oneFile->fileDate,'/','-');// Standardize YYYY-MM-DD, no time + oneFile->fileName = cloneString(words[4]); + slAddHead(&foundFiles,oneFile); + } + } + } + pclose(scriptOutput); + if (foundFiles == NULL) + { + foundFiles = fileDbReadFromBackup(db, dir, subDir); + if (foundFiles == NULL) + { + AllocVar(oneFile); + oneFile->fileName = cloneString("No files found!"); + oneFile->fileDate = cloneString(cmd); + slAddHead(&foundFiles,oneFile); + warn("No files found for command:\n%s",cmd); + } + } + else + { + fileDbWriteToBackup(db, dir, subDir,foundFiles); + if (timeIt) + uglyTime("Successful rsync found %d files",slCount(foundFiles)); + } + } + + // mark this as done to avoid excessive io + savedDb = cloneString(db); + savedDir = cloneString(dir); + savedSubDir = cloneString(subDir); + + if (foundFiles == NULL) + return NULL; + } + +// special code that only gets called in debug mode +if (sameString(fileName,"listAll")) + { + for (oneFile=foundFiles;oneFile;oneFile=oneFile->next) + warn("%s",oneFile->fileName); + return NULL; + } +// Look up the file and return it +struct fileDb *newList = NULL; +while (foundFiles) + { + oneFile = slPopHead(&foundFiles); + if (sameString(fileName,oneFile->fileName)) + break; // Found means removed from list: shorter list for next file. + + slAddHead(&newList,oneFile); + oneFile = NULL; + } +if (newList) + foundFiles = slCat(foundFiles,newList); // Order does not remain the same + +if (oneFile != NULL && oneFile->fileType == NULL) + { + char *suffix = strchr(oneFile->fileName, '.'); + if (suffix != NULL && strlen(suffix) > 2) + { + oneFile->fileType = cloneString(suffix + 1); + if (endsWith(oneFile->fileType,".gz")) + chopSuffix(oneFile->fileType); + } + } +return oneFile; +} + + +static sortOrder_t *fileSortOrderGet(struct cart *cart,struct trackDb *parentTdb, + struct mdbObj *mdbObjs) +// Parses 'fileSortOrder' trackDb/cart instructions and returns a sort order struct or NULL. +// Some trickiness here. sortOrder->sortOrder is from cart (changed by user action), +// as is sortOrder->order, but columns are in original tdb order (unchanging)! +// However, if cart is null, all is from trackDb.ra. +{ +int ix; +sortOrder_t *sortOrder = NULL; +char *carveSetting = NULL,*setting = NULL; +if (parentTdb) + setting = trackDbSetting(parentTdb, FILE_SORT_ORDER); +if (setting) + { + sortOrder = needMem(sizeof(sortOrder_t)); + sortOrder->setting = cloneString(setting); + carveSetting = sortOrder->setting; + } +else + { + if (mdbObjs != NULL) + { + struct dyString *dySortFields = dyStringNew(512); + struct mdbObj *commonVars = mdbObjsCommonVars(mdbObjs); + // common vars are already in cv defined order, searchable is also sortable + struct mdbVar *var = commonVars->vars; + for (;var != NULL; var = var->next) + { + if (differentWord(var->var,MDB_VAR_LAB_VERSION) // Exclude certain vars + && differentWord(var->var,MDB_VAR_SOFTWARE_VERSION) + && cvSearchMethod(var->var) != cvNotSearchable) // searchable is also sortable + { + if (mdbObjsHasCommonVar(mdbObjs, var->var,TRUE)) // Don't bother if all the vals + continue; // are the same (missing okay) + dyStringPrintf(dySortFields,"%s=%s ",var->var, + strSwapChar(cloneString(cvLabel(NULL,var->var)),' ','_')); + } + } + if (dyStringLen(dySortFields)) + { + dyStringAppend(dySortFields,"fileSize=Size fileType=File_Type"); + setting = dyStringCannibalize(&dySortFields); + } + else + dyStringFree(&dySortFields); + mdbObjsFree(&commonVars); + } + if (setting == NULL) // Must be in trackDb or not a sortable list of files + { + #define FILE_SORT_ORDER_DEFAULT "cell=Cell_Line lab=Lab view=View replicate=Rep " \ + "fileSize=Size fileType=File_Type dateSubmitted=Submitted " \ + "dateUnrestricted=RESTRICTED<BR>Until" + setting = FILE_SORT_ORDER_DEFAULT; + } + sortOrder = needMem(sizeof(sortOrder_t)); + carveSetting = cloneString(setting); + sortOrder->setting = NULL; + } +if (parentTdb) + { + sortOrder->htmlId = needMem(strlen(parentTdb->track)+20); + safef(sortOrder->htmlId, (strlen(parentTdb->track)+20), "%s.%s", + parentTdb->track,FILE_SORT_ORDER); + if (cart != NULL) + sortOrder->sortOrder = cloneString(cartOptionalString(cart, sortOrder->htmlId)); + } + +sortOrder->count = chopByWhite(carveSetting,NULL,0); // Get size +sortOrder->column = needMem(sortOrder->count*sizeof(char*)); +sortOrder->count = chopByWhite(carveSetting,sortOrder->column,sortOrder->count); +sortOrder->title = needMem(sortOrder->count*sizeof(char*)); +sortOrder->forward = needMem(sortOrder->count*sizeof(boolean)); +sortOrder->order = needMem(sortOrder->count*sizeof(int)); +for (ix = 0; ix<sortOrder->count; ix++) + { + // separate out mtaDb var in sortColumn from title + sortOrder->title[ix] = strchr(sortOrder->column[ix],'='); // Could be 'cell=Cell_Line' + if (sortOrder->title[ix] != NULL) + { + sortOrder->title[ix][0] = '\0'; // +1 jumps to next char after '=' + sortOrder->title[ix] = strSwapChar(sortOrder->title[ix]+1,'_',' '); + } + else + sortOrder->title[ix] = sortOrder->column[ix]; // or could be just 'cell' + + // Sort order defaults to forward but may be found in a cart var + sortOrder->order[ix] = ix+1; + sortOrder->forward[ix] = TRUE; + if (sortOrder->sortOrder != NULL) + { + // find tdb substr in cart current order string + char *pos = stringIn(sortOrder->column[ix], sortOrder->sortOrder); + if (pos != NULL && pos[strlen(sortOrder->column[ix])] == '=') + { + int ord=1; + char* pos2 = sortOrder->sortOrder; + for (;*pos2 && pos2 < pos;pos2++) + { + if (*pos2 == '=') // Discovering sort order in cart + ord++; + } + sortOrder->forward[ix] = (pos[strlen(sortOrder->column[ix]) + 1] == '+'); + sortOrder->order[ix] = ord; + } + } + } +if (sortOrder->sortOrder == NULL) + sortOrder->sortOrder = cloneString(setting); // no order in cart, all power to trackDb +return sortOrder; // NOTE cloneString:words[0]==*sortOrder->column[0] +} // will be freed when sortOrder is freed + +static int fileDbSortCmp(const void *va, const void *vb) +// Compare two sortable tdb items based upon sort columns. +{ +const struct fileDb *a = *((struct fileDb **)va); +const struct fileDb *b = *((struct fileDb **)vb); +char **fieldsA = a->sortFields; +char **fieldsB = b->sortFields; +int ix=0; +int compared = 0; +while (fieldsA[ix] != NULL && fieldsB[ix] != NULL) + { + compared = strcmp(fieldsA[ix], fieldsB[ix]); + if (compared != 0) + { + if (a->reverse[ix]) + compared *= -1; + break; + } + ix++; + } +return compared; +} + +static void fileDbSortList(struct fileDb **fileList, sortOrder_t *sortOrder) +// If sortOrder struct provided, will update sortFields in fileList, then sort it +{ +if (sortOrder && fileList) + { + struct fileDb *oneFile = NULL; + for (oneFile = *fileList;oneFile != NULL;oneFile=oneFile->next) + { // + 1 Null terminated + oneFile->sortFields = needMem(sizeof(char *) * (sortOrder->count + 1)); + oneFile->reverse = needMem(sizeof(boolean *) * sortOrder->count); + int ix; + for (ix=0;ix<sortOrder->count;ix++) + { + char *field = NULL; + if (sameString("fileSize",sortOrder->column[ix])) + { + char niceNumber[32]; + sprintf(niceNumber, "%.15lu", oneFile->fileSize); + field = cloneString(niceNumber); + } + else if (sameString("fileType",sortOrder->column[ix])) + field = oneFile->fileType; + else + field = mdbObjFindValue(oneFile->mdb,sortOrder->column[ix]); + + if (field) + { + oneFile->sortFields[sortOrder->order[ix] - 1] = field; + oneFile->reverse[ sortOrder->order[ix] - 1] = (sortOrder->forward[ix] == FALSE); + } + else + { + oneFile->sortFields[sortOrder->order[ix] - 1] = NULL; + oneFile->reverse[ sortOrder->order[ix] - 1] = FALSE; + } + } + oneFile->sortFields[sortOrder->count] = NULL; + } + slSort(fileList,fileDbSortCmp); + } +} + +static int removeCommonMdbVarsNotInSortOrder(struct mdbObj *mdbObjs,sortOrder_t *sortOrder) +// Removes varaibles common to all mdbObjs and not found in sortOrder. +// Returns count of vars removed +{ +if (sortOrder != NULL) + { + // Remove common vars from mdbs grant=Bernstein; lab=Broad; dataType=ChipSeq; setType=exp; + // However, keep the term if it is in the sortOrder + int count = 0; + struct dyString *dyCommon = dyStringNew(256); + char *commonTerms[] = { "grant", "lab", "dataType", "control", "setType" }; + int tIx=0,sIx = 0; + for (;tIx<ArraySize(commonTerms);tIx++) + { + for (sIx = 0; + sIx<sortOrder->count && differentString(commonTerms[tIx],sortOrder->column[sIx]); + sIx++) ; + if (sIx<sortOrder->count) // Found in sort Order so leave it in mdbObjs + continue; + + if (mdbObjsHasCommonVar(mdbObjs, commonTerms[tIx], TRUE)) // val the same or missing + { + count++; + dyStringPrintf(dyCommon,"%s ",commonTerms[tIx]); + } + } + if (count > 0) + mdbObjRemoveVars(mdbObjs,dyStringContents(dyCommon)); // removes from full list of mdbs + dyStringFree(&dyCommon); + return count; + } +return 0; +} + +static char *labelWithVocabLink(char *var,char *title,struct slPair *valsAndLabels, + boolean tagsNotVals) +// If the parentTdb has a controlledVocabulary setting and the vocabType is found, +// then label will be wrapped with the link to all relevent terms. Return string is cloned. +{ +// Determine if the var is cvDefined. If not, simple link +boolean cvDefined = FALSE; +struct hash *cvTypesOfTerms = (struct hash *)cvTermTypeHash(); +if (cvTypesOfTerms != NULL) + { + struct hash *cvTermDef = hashFindVal(cvTypesOfTerms,var); + if (cvTermDef) + cvDefined = SETTING_IS_ON(hashFindVal(cvTermDef,"cvDefined")); + } + +struct dyString *dyLink = dyStringNew(256); +if (!cvDefined) + dyStringPrintf(dyLink,"<A HREF='hgEncodeVocab?type=%s' title='Click for details of \"%s\"'" + " TARGET=ucscVocab>%s</A>",var,title,title); +else + { + dyStringPrintf(dyLink,"<A HREF='hgEncodeVocab?%s=",tagsNotVals?"tag":"term"); + struct slPair *oneVal = valsAndLabels; + for (;oneVal!=NULL;oneVal=oneVal->next) + { + if (oneVal != valsAndLabels) + dyStringAppendC(dyLink,','); + dyStringAppend(dyLink,mdbPairVal(oneVal)); + } + dyStringPrintf(dyLink,"' title='Click for details of each \"%s\"' TARGET=ucscVocab>%s</A>", + title,title); + } +return dyStringCannibalize(&dyLink); +} + +static int filterBoxesForFilesList(char *db,struct mdbObj *mdbObjs,sortOrder_t *sortOrder) +// Will create filterBoxes for each sortOrder field. Returns bitmask of sortOrder colums included +{ +int count = 0; +int filterableBits = 0; +if (sortOrder != NULL) + { + struct dyString *dyFilters = dyStringNew(256); + int sIx=0; + for (sIx = 0;sIx<sortOrder->count;sIx++) + { + char *var = sortOrder->column[sIx]; + enum cvSearchable searchBy = cvSearchMethod(var); + if (searchBy == cvNotSearchable || searchBy == cvSearchByFreeText) + continue; // Free text is not good candidate for filters. Best is single word/date/int. + + // get all vals for var, then convert to tag/label pairs for filterBys + struct slName *vals = mdbObjsFindAllVals(mdbObjs, var, CV_LABEL_EMPTY_IS_NONE); + if (searchBy != cvSearchByMultiSelect && searchBy != cvSearchBySingleSelect) + { + // We can't be too ambitious about creating filterboxes on the fly so some limitations: + // If there are more than 80 options, the filterBy is way too large and of limited use + // If there is a distinct val for each file in the table, then the filterBy is the + // same size as the table and of no help. Really the number of options should be + // half the number of rows but we are being lenient and cutting off at 0.8 not 0.5 + // If there is any non-alphanum char in a value then the filterBy will fail in js code. + // Those filterBy's are abandoned a but further down. + int valCount = slCount(vals); + if (valCount > 80 || valCount > (slCount(mdbObjs) * 0.8)) + { + slNameFreeList(&vals); + continue; + } + } + struct slPair *tagLabelPairs = NULL; + while (vals != NULL) + { + char buf[256]; + struct slName *term = slPopHead(&vals); + char *tag = (char *)cvTag(var,term->name); + if (tag == NULL) // Does not require cv defined! + { + safecpy(buf,sizeof buf,term->name); + tag = buf; // Bad news if tag has special chars, + eraseNonAlphaNum(tag); // unfortunately this does not pretect us from dups + // filtering by terms not in cv or regularized should be + // abandanded at the first sign of trouble! + if (searchBy != cvSearchByMultiSelect + && searchBy != cvSearchBySingleSelect + && searchBy != cvSearchByDateRange) + { + if (strlen(term->name) > strlen(tag)) + { + slNameFreeList(&vals); + slNameFree(&term); + break; + } + } + } + slPairAdd(&tagLabelPairs,tag,cloneString((char *)cvLabel(var,term->name))); + slNameFree(&term); + } + + // If there is more than one val for this var then create filterBy box for it + if (slCount(tagLabelPairs) > 1) + { + // should have a list sorted on the label + enum cvDataType eCvDataType = cvDataType(var); + if (eCvDataType == cvInteger) + slPairValAtoiSort(&tagLabelPairs); + else + slPairValSortCase(&tagLabelPairs); + char extraClasses[256]; + safef(extraClasses,sizeof extraClasses,"filterTable %s",var); + char *dropDownHtml = cgiMakeMultiSelectDropList(var,tagLabelPairs,NULL,"All", + extraClasses, "change", "filterTable.filter(this);", "font-size:.9em;", NULL); + // Note filterBox has classes: filterBy & {var} + if (dropDownHtml) + { + dyStringPrintf(dyFilters, + "<td align='left'>\n<B>%s</B>:<BR>\n%s</td><td width=10> </td>\n", + labelWithVocabLink(var,sortOrder->title[sIx],tagLabelPairs,TRUE), + dropDownHtml); // TRUE were sending tags, not values + freeMem(dropDownHtml); + count++; + if (sIx < 32) // avoid bit overflow but 32 filterBoxes? I don't think so + filterableBits |= (0x1<<(sIx)); + } + } + if (tagLabelPairs != NULL) + slPairFreeValsAndList(&tagLabelPairs); + } + + // Finally ready to print the filterBys out + if (count) + { + webIncludeResourceFile("ui.dropdownchecklist.css"); + jsIncludeFile("ui.dropdownchecklist.js",NULL); + #define FILTERBY_HELP_LINK "<A HREF=\"../goldenPath/help/multiView.html\" " \ + "TARGET=ucscHelp>help</A>" + cgiDown(0.9); + printf("<B>Filter files by:</B> (select multiple %sitems - %s)\n" + "<table><tr valign='bottom'>\n", + (count >= 1 ? "categories and ":""),FILTERBY_HELP_LINK); + printf("%s\n",dyStringContents(dyFilters)); + printf("</tr></table>\n"); + jsIncludeFile("ddcl.js",NULL); + } + dyStringFree(&dyFilters); + } +return filterableBits; +} + +static void filesDownloadsPreamble(char *db, struct trackDb *tdb, boolean isUnrestricted) +// Replacement for preamble.html which should expose parent dir, files.txt and supplemental, but +// not have any specialized notes per composite. Specialized notes belong in track description. +{ +char *server = hDownloadsServer(); +char *subDir = ""; +if (hIsBetaHost()) + { + server = "hgdownload-test.soe.ucsc.edu"; // NOTE: Force this case because beta may think + subDir = "/beta"; // it's downloads server is "hgdownload.soe.ucsc.edu" + } +if (!isUnrestricted) + { + cgiDown(0.9); + puts("<B>Data is <A HREF='../ENCODE/terms.html' TARGET='_BLANK'>RESTRICTED FROM USE</a>"); + puts("in publication until the restriction date noted for the given data file.</B>"); +} +cgiDown(0.7); +puts("Additional resources:"); +printf("<BR>• <B><A HREF='http://%s/goldenPath/%s/%s/%s%s/files.txt' " + "TARGET=ucscDownloads>files.txt</A></B> - lists the name and metadata for each download.\n", + server,db,ENCODE_DCC_DOWNLOADS, tdb->track, subDir); +printf("<BR>• <B><A HREF='http://%s/goldenPath/%s/%s/%s%s/md5sum.txt' " + "TARGET=ucscDownloads>md5sum.txt</A></B> - lists the md5sum output for each download.\n", + server,db,ENCODE_DCC_DOWNLOADS, tdb->track, subDir); +printf("<BR>• <B><A HREF='http://%s/goldenPath/%s/%s/%s%s'>downloads server</A></B> - " + "alternative access to downloadable files (may include obsolete data).\n", + server,db,ENCODE_DCC_DOWNLOADS, tdb->track, subDir); + +struct fileDb *oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, tdb->track, "supplemental"); +if (oneFile != NULL) + { + printf("<BR>• <B><A HREF='http://%s/goldenPath/%s/%s/%s%s/supplemental/' " + "TARGET=ucscDownloads>supplemental materials</A></B> - " + "any related files provided by the laboratory.\n", + server,db,ENCODE_DCC_DOWNLOADS, tdb->track, subDir); + } +} + +static int filesPrintTable(char *db, struct trackDb *parentTdb, struct fileDb *fileList, + sortOrder_t *sortOrder,int filterable) +// Prints filesList as a sortable table. Returns count +{ +if (timeIt) + uglyTime("Start table"); +// Table class=sortable +int columnCount = 0; +int butCount = 0; +int restrictedColumn = 0; +char *nowrap = (sortOrder->setting != NULL ? " nowrap":""); + // Sort order trackDb setting found so rely on <BR> in titles for wrapping +printf("<TABLE class='sortable' style='border: 2px outset #006600;'>\n"); +printf("<THEAD class='sortable'>\n"); +printf("<TR class='sortable' valign='bottom'>\n"); +printf("<TD align='center' valign='center'> "); +int filesCount = slCount(fileList); +if (filesCount > 5) + printf("<em><span class='filesCount'></span>%d files</em>",filesCount); + +// NOTE: This could be done to preserve sort order in cart. +// However hgFileUi would need form OR changes would need to be ajaxed over +// AND hgsid would be needed. +//if (sortOrder) +// printf("<INPUT TYPE=HIDDEN NAME='%s' class='sortOrder' VALUE=\"%s\">", +// sortOrder->htmlId, sortOrder->sortOrder); +printf("</TD>\n"); +columnCount++; + +// Now the columns +int curOrder = 0,ix=0; +if (sortOrder) + { + curOrder = sortOrder->count; + for (ix=0;ix<sortOrder->count;ix++) + { + char *align = (sameString("labVersion",sortOrder->column[ix]) + || sameString("softwareVersion",sortOrder->column[ix]) ? " align='left'":""); + printf("<TH class='sortable sort%d%s' %s%s%s>%s</TH>\n", + sortOrder->order[ix],(sortOrder->forward[ix]?"":" sortRev"), + (sameString("fileSize",sortOrder->column[ix])?"abbr='use' ":""), + nowrap,align,sortOrder->title[ix]); // keeing track of sortOrder + columnCount++; + if (sameWord(sortOrder->column[ix],"dateUnrestricted")) + restrictedColumn = columnCount; + } + } +//#define INCLUDE_FILENAMES +#ifndef INCLUDE_FILENAMES +else +#endif///defn INCLUDE_FILENAMES + { + printf("<TH class='sortable sort%d' nowrap>File Name</TH>\n",++curOrder); + columnCount++; + } +printf("<TH class='sortable sort%d' align='left' nowrap>Additional Details</TH>\n",++curOrder); +columnCount++; +printf("</TR></THEAD>\n"); + +// Now the files... +char *server = hDownloadsServer(); +char *subDir = ""; +if (hIsBetaHost()) + { + server = "hgdownload-test.soe.ucsc.edu"; // NOTE: Force this case because beta may think + subDir = "/beta"; // it's downloads server is "hgdownload.soe.ucsc.edu" + } +struct fileDb *oneFile = fileList; +printf("<TBODY class='sortable sorting'>\n"); // 'sorting' is a fib but it conveniently greys +if (timeIt) // the list till the table is initialized. + uglyTime("Finished column headers"); +for (;oneFile!= NULL;oneFile=oneFile->next) + { + oneFile->mdb->next = NULL; // mdbs were in list for generating sortOrder, + char *field = NULL; // but list no longer needed + + printf("<TR valign='top'%s>",(filterable != 0) ?" class='filterable'":""); + // Download button + printf("<TD nowrap>"); + if (parentTdb) + field = parentTdb->track; + else + { + field = cloneString(mdbObjFindValue(oneFile->mdb,MDB_VAR_COMPOSITE)); + mdbObjRemoveOneVar(oneFile->mdb,MDB_VAR_COMPOSITE,NULL); + } + assert(field != NULL); + + char id[256]; + safef(id, sizeof id, "ftpBut_%d", butCount++); + printf("<input type='button' id='%s' value='Download' title='Download %s ...'>", id, oneFile->fileName); + jsOnEventByIdF("click", id, "window.location='http://%s/goldenPath/%s/%s/%s%s/%s';" + ,server,db,ENCODE_DCC_DOWNLOADS, field, subDir, oneFile->fileName); + +#define SHOW_FOLDER_FRO_COMPOSITE_DOWNLOADS +#ifdef SHOW_FOLDER_FRO_COMPOSITE_DOWNLOADS + if (parentTdb == NULL) + printf(" <A href='../cgi-bin/hgFileUi?db=%s&g=%s' title='Navigate to downloads " + "page for %s set...'><IMG SRC='../images/folderC.png'></a> ", db,field,field); +#endif///def SHOW_FOLDER_FRO_COMPOSITE_DOWNLOADS + printf("</TD>\n"); + + // Each of the pulled out mdb vars + if (sortOrder) + { + for (ix=0;ix<sortOrder->count;ix++) + { + if (sameString("fileSize",sortOrder->column[ix])) + { + char niceNumber[128]; + sprintWithGreekByte(niceNumber, sizeof(niceNumber), oneFile->fileSize); + field = oneFile->sortFields[sortOrder->order[ix] - 1]; + printf("<TD abbr='%s' align='right' nowrap>%s</td>",field,niceNumber); + } + else + { + field = oneFile->sortFields[sortOrder->order[ix] - 1]; + + boolean isFieldEmpty = isEmpty(field); + struct hash *termHash = NULL; + if (!isFieldEmpty) + { + termHash = (struct hash *)cvOneTermHash(sortOrder->column[ix],field); + if (termHash && sameString(field,MDB_VAL_ENCODE_EDV_NONE)) + isFieldEmpty = cvTermIsEmpty(sortOrder->column[ix],field); + } + char class[128]; + class[0] = '\0'; + + if (filterable & (0x1<<ix)) + { + char *cleanClass = NULL; + char buf[256]; + if (isFieldEmpty) + cleanClass = CV_LABEL_EMPTY_IS_NONE; + else + { + if (termHash) + cleanClass = (char *)hashFindVal((struct hash *)termHash,CV_TAG); + if (cleanClass == NULL) + { + // This may not be needed because the filterBy code + // already eliminated these. + safecpy(buf,sizeof buf,field); + cleanClass = buf; + eraseNonAlphaNum(cleanClass); + } + } + safef(class,sizeof class," class='%s %s'",sortOrder->column[ix],cleanClass); + } + + char *align = (sameString("labVersion",sortOrder->column[ix]) + || sameString("softwareVersion",sortOrder->column[ix]) ? + " align='left'":" align='center'"); + if (sameString("dateUnrestricted",sortOrder->column[ix]) + && field + && dateIsOld(field, MDB_ENCODE_DATE_FORMAT)) + printf("<TD%s nowrap style='color: #BBBBBB;'%s>%s</td>",align,class,field); + else + { + // use label + if (!isFieldEmpty && termHash) + { + char *label = hashFindVal(termHash,CV_LABEL); + if (label != NULL) + field = label; + } + printf("<TD%s nowrap%s>%s</td>",align,class,isFieldEmpty?" ":field); + } + if (!sameString("fileType",sortOrder->column[ix])) + mdbObjRemoveOneVar(oneFile->mdb,sortOrder->column[ix],NULL); + } + } + } +#ifndef INCLUDE_FILENAMES + else +#endif///ndef INCLUDE_FILENAMES + { // fileName + printf("<TD nowrap>%s",oneFile->fileName); + } + + // Extras grant=Bernstein; lab=Broad; dataType=ChipSeq; setType=exp; control=std; + field = mdbObjVarValPairsAsLine(oneFile->mdb,TRUE,FALSE); + printf("<TD nowrap>%s</td>",field?field:" "); + + printf("</TR>\n"); + } +if (timeIt) + uglyTime("Finished files"); + +printf("</TBODY><TFOOT class='bgLevel1'>\n"); +printf("<TR valign='top'>"); + +// Restriction policy link in first column? +if (restrictedColumn == 1) + printf("<TH colspan=%d><A HREF='%s' TARGET=BLANK style='font-size:.9em;'>" + "Restriction Policy</A></TH>", (columnCount - restrictedColumn), + ENCODE_DATA_RELEASE_POLICY); + +printf("<TD colspan=%d> ", + (restrictedColumn > 1 ? (restrictedColumn - 1) : columnCount)); + +// Total +if (filesCount > 5) + printf("<em><span class='filesCount'></span>%d files</em>",filesCount); + +// Restriction policy link in later column? +if (restrictedColumn > 1) + printf("</TD><TH colspan=%d align='left'><A HREF='%s' TARGET=BLANK style='font-size:.9em;'>" + "Restriction Policy</A>", columnCount,ENCODE_DATA_RELEASE_POLICY); + +printf("</TD></TR>\n"); +printf("</TFOOT></TABLE>\n"); + +if (parentTdb == NULL) + jsInline("$(document).ready(function() {" + "sortTable.initialize($('table.sortable')[0],true,true);});\n"); + +if (timeIt) + uglyTime("Finished table"); +return filesCount; +} + + +static int filesFindInDir(char *db, struct mdbObj **pmdbFiles, struct fileDb **pFileList, + char *fileType, int limit, boolean *exceededLimit, boolean *isUnrestricted) +// Prints list of files in downloads directories matching mdb search terms. Returns count +{ +int fileCount = 0; +if (isUnrestricted != NULL) + *isUnrestricted = TRUE; + +// Verify file existance and make fileList of those found +struct fileDb *fileList = NULL, *oneFile = NULL; // Will contain found files +struct mdbObj *mdbFiles = NULL; // Will caontain a list of mdbs for the found files +struct mdbObj *mdbList = *pmdbFiles; +while (mdbList && (limit == 0 || fileCount < limit)) + { + boolean found = FALSE; + struct mdbObj *mdbFile = slPopHead(&mdbList); + char *composite = mdbObjFindValue(mdbFile,MDB_VAR_COMPOSITE); + if (composite == NULL) + { + mdbObjsFree(&mdbFile); + continue; + } + + // First for FileName + char *fileName = mdbObjFindValue(mdbFile,MDB_VAR_FILENAME); + if (fileName == NULL) + { + mdbObjsFree(&mdbFile); + continue; + } + + // Are any files still restricted access under ENCODE data policy ? + if (isUnrestricted != NULL && *isUnrestricted) + *isUnrestricted = mdbObjEncodeIsUnrestricted(mdbFile); + + struct slName *fileSet = slNameListFromComma(fileName); + struct slName *md5Set = NULL; + char *md5sums = mdbObjFindValue(mdbFile,MDB_VAR_MD5SUM); + if (md5sums != NULL) + md5Set = slNameListFromComma(md5sums); + + // Could be that "bai" is implicit with "bam" + if ((slCount(fileSet) == 1) && endsWith(fileSet->name,".bam")) + { + char buf[512]; + safef(buf,sizeof(buf),"%s.bai",fileSet->name); + slNameAddTail(&fileSet, buf); + } + while (fileSet != NULL) + { + struct slName *file = slPopHead(&fileSet); + struct slName *md5 = NULL; + if (md5Set) + md5 = slPopHead(&md5Set); + oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, composite, file->name); + if (oneFile == NULL) + { + slNameFree(&file); + if (md5) + slNameFree(&md5); + continue; + } + + //warn("%s == %s",fileType,oneFile->fileType); + if (isEmpty(fileType) + || sameWord(fileType,"Any") + || ( oneFile->fileType + && startsWithWordByDelimiter(fileType,'.',oneFile->fileType))) + // Starts with! This ensures both bam and bam.bai are found. + { + slAddHead(&fileList,oneFile); + if (found) // if already found then need two mdbObjs (assertable but this is metadata) + oneFile->mdb = mdbObjClone(mdbFile); // Yes clone this as differences will occur + else + oneFile->mdb = mdbFile; + if (md5 != NULL) + mdbObjSetVar(oneFile->mdb,MDB_VAR_MD5SUM,md5->name); + else + mdbObjRemoveOneVar(oneFile->mdb,MDB_VAR_MD5SUM,NULL); + slAddHead(&mdbFiles,oneFile->mdb); + found = TRUE; + fileCount++; + if (limit > 0 && fileCount >= limit) + { + slNameFreeList(&fileSet); + if (md5Set) + slNameFreeList(&md5Set); + break; + } + } + else + fileDbFree(&oneFile); + + slNameFree(&file); + if (md5) + slNameFree(&md5); + } + + // FIXME: This support of fileIndex should be removed when mdb is cleaned up. + // Now for FileIndexes + if (limit == 0 || fileCount < limit) + { + fileName = mdbObjFindValue(mdbFile,MDB_VAR_FILEINDEX);// This mdb var should be going away. + if (fileName == NULL) + continue; + + // Verify existance first + oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, composite, fileName); + if (oneFile == NULL) // NOTE: won't be found if already found in comma delimited fileName! + continue; + + if (isEmpty(fileType) || sameWord(fileType,"Any") + || (oneFile->fileType && sameWord(fileType,oneFile->fileType)) + || (oneFile->fileType && sameWord(fileType,"bam") && sameWord("bam.bai",oneFile->fileType))) + { // TODO: put fileType matching into search.c lib code to segregate index logic. + slAddHead(&fileList,oneFile); + if (found) // if already found then need two mdbObjs (assertable but this is metadata) + oneFile->mdb = mdbObjClone(mdbFile); + else + oneFile->mdb = mdbFile; + mdbObjRemoveOneVar(oneFile->mdb,MDB_VAR_MD5SUM,NULL); + slAddHead(&mdbFiles,oneFile->mdb); + fileCount++; + found = TRUE; + continue; + } + else + fileDbFree(&oneFile); + } + // FIXME: This support of fileIndex should be removed when mdb is cleaned up. + + if (!found) + mdbObjsFree(&mdbFile); + } +*pmdbFiles = mdbFiles; +*pFileList = fileList; +if (exceededLimit != NULL) + *exceededLimit = FALSE; +if (mdbList != NULL) + { + if (exceededLimit != NULL) + *exceededLimit = TRUE; + mdbObjsFree(&mdbList); + } +return fileCount; +} + +void filesDownloadUi(char *db, struct cart *cart, struct trackDb *tdb) +// UI for a "composite like" track: This will list downloadable files associated with +// a single trackDb entry (composite or of type "downloadsOnly". The list of files +// will have links to their download and have metadata information associated. +// The list will be a sortable table and there may be filtering controls. +{ +timeIt = cartUsualBoolean(cart, "measureTiming",FALSE); // static to file +if (timeIt) + uglyTime("Starting file search"); + +boolean debug = cartUsualBoolean(cart,"debug",FALSE); + +struct sqlConnection *conn = hAllocConn(db); +char *mdbTable = mdbTableName(conn,TRUE); // Look for sandBox name first +if (mdbTable == NULL) + errAbort("TABLE NOT FOUND: '%s.%s'.\n",db,MDB_DEFAULT_NAME); + +// Get an mdbObj list of all that belong to this track and have a fileName +char buf[256]; +safef(buf,sizeof(buf),"%s=%s %s=?",MDB_VAR_COMPOSITE,tdb->track,MDB_VAR_FILENAME); +struct mdbByVar *mdbVars = mdbByVarsLineParse(buf); +struct mdbObj *mdbList = mdbObjsQueryByVars(conn,mdbTable,mdbVars); + +// Now get Indexes But be sure not to duplicate entries in the list!!! +safef(buf,sizeof(buf),"%s=%s %s= %s!=",MDB_VAR_COMPOSITE,tdb->track, + MDB_VAR_FILEINDEX,MDB_VAR_FILENAME); +mdbVars = mdbByVarsLineParse(buf); +mdbList = slCat(mdbList, mdbObjsQueryByVars(conn,mdbTable,mdbVars)); +mdbObjRemoveHiddenVars(mdbList); +hFreeConn(&conn); + +if (mdbList) + (void)mdbObjsFilter(&mdbList,"objStatus","re*",TRUE); // revoked, replaced, renamed + +if (slCount(mdbList) == 0) + { + warn("No files specified in metadata for: %s\n%s",tdb->track,tdb->longLabel); + return; + } +if (timeIt) + uglyTime("Found %d mdb objects",slCount(mdbList)); + +// Verify file existance and make fileList of those found +struct fileDb *fileList = NULL; // Will contain found files + +boolean isUnrestricted; +int fileCount = filesFindInDir(db, &mdbList, &fileList, NULL, 0, NULL, &isUnrestricted); +if (timeIt) + uglyTime("Found %d files in dir",fileCount); +assert(fileCount == slCount(fileList)); + +if (fileCount == 0) + { + warn("No downloadable files currently available for: %s\n%s",tdb->track,tdb->longLabel); + return; // No files so nothing to do. + } +if (debug) + { + warn("The following files are in goldenPath/%s/%s/%s/ but NOT in the mdb:", + db,ENCODE_DCC_DOWNLOADS, tdb->track); + fileDbGet(db, ENCODE_DCC_DOWNLOADS, tdb->track, "listAll"); + } + +jsIncludeFile("hui.js",NULL); +jsIncludeFile("ajax.js",NULL); + +// standard preamble +filesDownloadsPreamble(db,tdb, isUnrestricted); + +// remove these now to get them out of the way +mdbObjRemoveVars(mdbList,MDB_VAR_FILENAME " " MDB_VAR_FILEINDEX " " + MDB_VAR_COMPOSITE " " MDB_VAR_PROJECT); +if (timeIt) + uglyTime("<BR>Removed 4 unwanted vars"); + +// Now update all files with their sortable fields and sort the list +mdbObjReorderByCv(mdbList,FALSE);// Start with cv defined order for visible vars. + // NOTE: will not need to reorder during print! +sortOrder_t *sortOrder = fileSortOrderGet(cart,tdb,mdbList); +int filterable = 0; +if (sortOrder != NULL) + { + int removed = removeCommonMdbVarsNotInSortOrder(mdbList,sortOrder); + if (removed && debug) + warn("%d terms are common and were removed",removed); + if (timeIt) + uglyTime("Removed %d common vars",removed); + + // Fill in and sort fileList + fileDbSortList(&fileList,sortOrder); + if (timeIt) + uglyTime("Sorted %d files on %d columns",fileCount,sortOrder->count); + + // FilterBoxes + filterable = filterBoxesForFilesList(db,mdbList,sortOrder); + if (timeIt && filterable) + uglyTime("Created filter boxes"); + } + +// Print table +filesPrintTable(db,tdb,fileList,sortOrder,filterable); + +//fileDbFree(&fileList); // Why bother on this very long running cgi? +//mdbObjsFree(&mdbList); +} + +int fileSearchResults(char *db, struct sqlConnection *conn, struct cart *cart, + struct slPair *varValPairs, char *fileType) +// Prints list of files in downloads directories matching mdb search terms. Returns count +{ +timeIt = cartUsualBoolean(cart, "measureTiming",FALSE); // static to file +if (timeIt) + uglyTime("Starting file search"); + +struct sqlConnection *connLocal = conn; +if (conn == NULL) + connLocal = hAllocConn(db); +struct mdbObj *mdbList = mdbObjRepeatedSearch(connLocal,varValPairs,FALSE,TRUE); +if (conn == NULL) + hFreeConn(&connLocal); + +mdbObjRemoveHiddenVars(mdbList); +if (mdbList) + (void)mdbObjsFilter(&mdbList,"objStatus","re*",TRUE); // revoked, replaced, renamed + +if (slCount(mdbList) == 0) + { + printf("<DIV id='filesFound'><BR>No files found.<BR></DIV><BR>\n"); + return 0; + } +if (timeIt) + uglyTime("Found %d mdb objects",slCount(mdbList)); + +// Now sort mdbObjs so that composites will stay together & lookup of files will be most efficient +mdbObjsSortOnVars(&mdbList, MDB_VAR_COMPOSITE); + +#define FOUND_FILE_LIMIT 1000 +struct fileDb *fileList = NULL; // Will contain found files +int filesExpected = slCount(mdbList); +boolean exceededLimit = FALSE; +int fileCount = filesFindInDir(db, &mdbList, &fileList, fileType, FOUND_FILE_LIMIT, + &exceededLimit, NULL); +if (timeIt) + uglyTime("Found %d files in dir",fileCount); +assert(fileCount == slCount(fileList)); + +if (fileCount == 0) + { + printf("<DIV id='filesFound'><BR>No files found.<BR></DIV><BR>\n"); + return 0; // No files so nothing to do. + } + +// remove these now to get them out of the way +mdbObjRemoveVars(mdbList,MDB_VAR_FILENAME " " MDB_VAR_FILEINDEX " " + MDB_VAR_PROJECT " " MDB_VAR_TABLENAME); +if (timeIt) + uglyTime("Removed 4 unwanted vars"); + +// Now update all files with their sortable fields and sort the list +mdbObjReorderByCv(mdbList,FALSE);// Start with cv defined order for visible vars. + // NOTE: will not need to reorder during print! +sortOrder_t *sortOrder = fileSortOrderGet(NULL,NULL,mdbList); // No cart, no tdb +if (sortOrder != NULL) + { + // Fill in and sort fileList + fileDbSortList(&fileList,sortOrder); + if (timeIt) + uglyTime("Sorted %d files on %d columns",fileCount,sortOrder->count); + } + +// Print table +printf("<DIV id='filesFound'>"); +if (exceededLimit) + { + // What is the expected count? Difficult to say because of comma delimited list in fileName. + if (filesExpected <= FOUND_FILE_LIMIT) + filesExpected = FOUND_FILE_LIMIT + 1; + + printf("<DIV class='redBox' style='width: 380px;'>Too many files found. Displaying first " + "%d of at least %d.<BR>Narrow search parameters and try again.</DIV><BR>\n", + fileCount,filesExpected); + } + // 0=No columns 'filtered' on file search page +fileCount = filesPrintTable(db,NULL,fileList,sortOrder,0); +printf("</DIV><BR>\n"); + +//fileDbFree(&fileList); // Why bother on this very long running cgi? +//mdbObjsFree(&mdbList); + +return fileCount; +} +