2eb4c6d91873ad7c8f68f8e2e419e3382d46a16f tdreszer Thu Oct 20 17:37:48 2011 -0700 Added file list caching to hgFileUi and extended the filterBox logic to include non-multiSelect options. Redmine 5527 for the fileBoxes. diff --git src/hg/lib/fileUi.c src/hg/lib/fileUi.c index 56d68c3..72972a1 100644 --- src/hg/lib/fileUi.c +++ src/hg/lib/fileUi.c @@ -1,70 +1,154 @@ /* fileUi.c - human genome file downloads common controls. */ #include "common.h" #include "hash.h" #include "cheapcgi.h" #include "jsHelper.h" #include "cart.h" #include "hdb.h" #include "fileUi.h" #include "hui.h" #include "obscure.h" #include "mdb.h" #include "jsHelper.h" #include "web.h" - -// FIXME: Move to hui.h since hui.c also needs this -#define ENCODE_DCC_DOWNLOADS "encodeDCC" +#include "trashDir.h" void fileDbFree(struct fileDb **pFileList) // free one or more fileDb objects { while (pFileList && *pFileList) { struct fileDb *oneFile = slPopHead(pFileList); freeMem(oneFile->fileName); freeMem(oneFile->fileType); freeMem(oneFile->fileDate); freeMem(oneFile->sortFields); freeMem(oneFile->reverse); mdbObjsFree(&(oneFile->mdb)); freeMem(oneFile); } } +static struct fileDb *fileDbReadFromCache(char *db, char *dir, char *subDir) +{ +struct tempName cacheFile; +boolean exists = trashDirReusableFile(&cacheFile, dir, subDir, db); // encodeDCC/composite.db +if (!exists) + return NULL; +//warn("Reading: %s",cacheFile.forCgi); + +struct fileDb *fileList = NULL; +struct fileDb *oneFile = NULL; +struct lineFile *lf = lineFileOpen(cacheFile.forCgi, TRUE); +char *words[4]; +while (lineFileChop(lf, words) >= 3) + { + AllocVar(oneFile); + oneFile->fileName = cloneString(words[0]); + oneFile->fileSize = sqlUnsignedLong(words[1]); + oneFile->fileDate = cloneString(words[2]); + slAddHead(&fileList,oneFile); + } +lineFileClose(&lf); +if (fileList == NULL) + unlink(cacheFile.forCgi); // remove empty file + +return fileList; +} + +// Cache is not faster, so just use it as a backup +//#define CACHE_IS_FASTER_THAN_RSYNC +#ifdef CACHE_IS_FASTER_THAN_RSYNC +static boolean fileDbCacheAvailable(char *db, char *dir, char *subDir) +{ // Checks if there is a recent enough cache file + // TODO: Add some other trick to invalidate cache at will. +struct tempName cacheFile; +boolean exists = trashDirReusableFile(&cacheFile, dir, subDir, db); // encodeDCC/composite.db +if (exists) + { + //char *clearCache = cartOptionalString(cart,"clearCache"); // where is cart coming from? + //if (clearCache && sameWord(clearCache,subDir)) + // { + // cartRemove(cart,"clearCache"); + // unlink(cacheFile.forCgi); // remove empty file + // return FALSE; + // } + struct stat mystat; + ZeroVar(&mystat); + if (stat(cacheFile.forCgi,&mystat)==0) + { + // how old is old? + int secs = (clock1() - mystat.st_ctime); // seconds since created + if (secs < (24 * 60 * 60)) // one date + return TRUE; + } + } +return FALSE; +} +#endif///def CACHE_IS_FASTER_THAN_RSYNC + +static void fileDbWriteToCache(char *db, char *dir, char *subDir,struct fileDb *fileList) +{ +struct tempName cacheFile; +(void)trashDirReusableFile(&cacheFile, dir, subDir, db); // encodeDCC/composite.db +//warn("Writing: %s",cacheFile.forCgi); + +FILE *fd = NULL; +if ((fd = fopen(cacheFile.forCgi, "w")) != NULL) + { + struct fileDb *oneFile = fileList; + for(;oneFile != NULL;oneFile=oneFile->next) + { + char buf[1024]; + safef(buf,sizeof buf,"%s %ld %s\n",oneFile->fileName,oneFile->fileSize,oneFile->fileDate); + fwrite(buf, strlen(buf), 1, fd); + } + fclose(fd); + } +} + struct fileDb *fileDbGet(char *db, char *dir, char *subDir, char *fileName) // Returns NULL or if found a fileDb struct with name, size and date filled in. { static char *savedDb = NULL; static char *savedDir = NULL; static char *savedSubDir = NULL; static struct fileDb *foundFiles = NULL;// Use static list to save excess IO struct fileDb *oneFile = NULL; if (foundFiles == NULL || savedDb == NULL || differentString(savedDb, db) || savedDir == NULL || differentString(savedDir,dir) || savedSubDir == NULL || differentString(savedSubDir,subDir)) { // free up any static mem freeMem(savedDb); freeMem(savedDir); freeMem(savedSubDir); fileDbFree(&foundFiles); +#ifdef CACHE_IS_FASTER_THAN_RSYNC + if (fileDbCacheAvailable(db, dir, subDir)) // check cache first + { + foundFiles = fileDbReadFromCache(db, dir, subDir); + } + else +#endif///def CACHE_IS_FASTER_THAN_RSYNC + { FILE *scriptOutput = NULL; char buf[1024]; char cmd[512]; char *words[10]; char *server = hDownloadsServer(); boolean useRsync = TRUE; // Works: rsync -avn rsync://hgdownload.cse.ucsc.edu/goldenPath/hg18/encodeDCC/wgEncodeBroadChipSeq/ if (hIsBetaHost()) safef(cmd,sizeof(cmd),"rsync -n rsync://hgdownload-test.cse.ucsc.edu/goldenPath/%s/%s/%s/beta/", db, dir, subDir); // NOTE: Force this case because beta may think it's downloads server is "hgdownload.cse.ucsc.edu" else safef(cmd,sizeof(cmd),"rsync -n rsync://%s/goldenPath/%s/%s/%s/", server, db, dir, subDir); scriptOutput = popen(cmd, "r"); while(fgets(buf, sizeof(buf), scriptOutput)) @@ -88,46 +172,54 @@ } else if (count == 5 && useRsync == TRUE)// genome and hgwbeta can use rsync because files are on different machine { //-rw-rw-r-- 26420982 2009/09/29 14:53:30 wgEncodeBroadChipSeq/wgEncodeBroadChipSeqSignalNhlfH4k20me1.wig.gz AllocVar(oneFile); oneFile->fileSize = sqlUnsignedLong(words[1]); oneFile->fileDate = cloneString(words[2]); strSwapChar(oneFile->fileDate,'/','-');// Standardize YYYY-MM-DD, no time oneFile->fileName = cloneString(words[4]); slAddHead(&foundFiles,oneFile); } //warn("File:%s size:%ld",foundFiles->fileName,foundFiles->fileSize); } } pclose(scriptOutput); - - // mark this as done to avoid excessive io - savedDb = cloneString(db); - savedDir = cloneString(dir); - savedSubDir = cloneString(subDir); - + if (foundFiles == NULL) + { + foundFiles = fileDbReadFromCache(db, dir, subDir); if (foundFiles == NULL) { AllocVar(oneFile); oneFile->fileName = cloneString("No files found!"); oneFile->fileDate = cloneString(cmd); slAddHead(&foundFiles,oneFile); warn("No files found for command:\n%s",cmd); - return NULL; } } + else + fileDbWriteToCache(db, dir, subDir,foundFiles); + } + + // mark this as done to avoid excessive io + savedDb = cloneString(db); + savedDir = cloneString(dir); + savedSubDir = cloneString(subDir); + + if (foundFiles == NULL) + return NULL; + } // special code that only gets called in debug mode if (sameString(fileName,"listAll")) { for(oneFile=foundFiles;oneFile;oneFile=oneFile->next) warn("%s",oneFile->fileName); return NULL; } // Look up the file and return it struct fileDb *newList = NULL; while (foundFiles) { oneFile = slPopHead(&foundFiles); if (sameString(fileName,oneFile->fileName)) break; // Found means removed from list: shorter list for next file. @@ -363,107 +455,140 @@ { dyStringPrintf(dyLink,"<A HREF='hgEncodeVocab?%s=",tagsNotVals?"tag":"term"); struct slPair *oneVal = valsAndLabels; for(;oneVal!=NULL;oneVal=oneVal->next) { if (oneVal != valsAndLabels) dyStringAppendC(dyLink,','); dyStringAppend(dyLink,mdbPairVal(oneVal)); } dyStringPrintf(dyLink,"' title='Click for details of each \"%s\"' TARGET=ucscVocab>%s</A>",title,title); } return dyStringCannibalize(&dyLink); } static int filterBoxesForFilesList(char *db,struct mdbObj *mdbObjs,sortOrder_t *sortOrder) -{ // Will create filterBoxes for each sortOrder field. Returns count of filterBoxes made +{ // Will create filterBoxes for each sortOrder field. Returns bitmask of sortOrder colums included int count = 0; +int filterableBits = 0; if (sortOrder != NULL) { struct dyString *dyFilters = dyStringNew(256); int sIx=0; for(sIx = 0;sIx<sortOrder->count;sIx++) { char *var = sortOrder->column[sIx]; enum cvSearchable searchBy = cvSearchMethod(var); - if (searchBy != cvSearchBySingleSelect && searchBy != cvSearchByMultiSelect) + //#define FILTERBY_ALL_SEARCHABLE + #ifdef FILTERBY_ALL_SEARCHABLE + if (searchBy == cvNotSearchable) + #else///ifndef FILTERBY_ALL_SEARCHABLE + if (searchBy == cvNotSearchable || searchBy == cvSearchByFreeText) + #endif///ndef FILTERBY_ALL_SEARCHABLE continue; // Only single selects and multi-select make good candidates for filtering // get all vals for var, then convert to tag/label pairs for filterBys - struct slName *vals = mdbObjsFindAllVals(mdbObjs, var); + struct slName *vals = mdbObjsFindAllVals(mdbObjs, var, CV_LABEL_EMPTY_IS_NONE); + if (searchBy != cvSearchByMultiSelect && searchBy != cvSearchBySingleSelect) + { + int valCount = slCount(vals); + if (valCount > 80 || valCount > (slCount(mdbObjs) * 0.8)) + { + slNameFreeList(&vals); + continue; + } + } struct slPair *tagLabelPairs = NULL; while(vals != NULL) { + char buf[256]; struct slName *term = slPopHead(&vals); char *tag = (char *)cvTag(var,term->name); - if (tag == NULL) - tag = term->name; + if (tag == NULL) // Does not require cv defined! + { + safecpy(buf,sizeof buf,term->name); + tag = buf; + eraseNonAlphaNum(tag); // Bad news if tag has special chars, unfortunately this does not pretect us from dups + if (searchBy != cvSearchByMultiSelect + && searchBy != cvSearchBySingleSelect + && searchBy != cvSearchByDateRange) + { // filtering by terms not in cv or regularized should be abandanded at the first sign of trouble! + if (strlen(term->name) > strlen(tag)) + { + slNameFreeList(&vals); + slNameFree(&term); + break; + } + } + } slPairAdd(&tagLabelPairs,tag,cloneString((char *)cvLabel(var,term->name))); slNameFree(&term); } // If there is more than one val for this var then create filterBy box for it if (slCount(tagLabelPairs) > 1) { // should have a list sorted on the label enum cvDataType eCvDataType = cvDataType(var); if (eCvDataType == cvInteger) slPairValAtoiSort(&tagLabelPairs); else slPairValSortCase(&tagLabelPairs); char extraClasses[256]; safef(extraClasses,sizeof extraClasses,"filterTable %s",var); #ifdef NEW_JQUERY char *dropDownHtml = cgiMakeMultiSelectDropList(var,tagLabelPairs,NULL,"All",extraClasses,"onchange='filterTable(this);' style='font-size:.9em;'"); #else///ifndef NEW_JQUERY char *dropDownHtml = cgiMakeMultiSelectDropList(var,tagLabelPairs,NULL,"All",extraClasses,"onchange='filterTable();' onclick='filterTableExclude(this);'"); #endif///ndef NEW_JQUERY // Note filterBox has classes: filterBy & {var} if (dropDownHtml) { dyStringPrintf(dyFilters,"<td align='left'>\n<B>%s</B>:<BR>\n%s</td><td width=10> </td>\n", labelWithVocabLink(var,sortOrder->title[sIx],tagLabelPairs,TRUE),dropDownHtml); // TRUE were sending tags, not values freeMem(dropDownHtml); count++; + filterableBits |= (0x1<<(sIx)); + //warn("count:%d sIx:%d retBits:%X",count,sIx,filterableBits); } } - if (slCount(tagLabelPairs) > 0) + if (tagLabelPairs != NULL) slPairFreeValsAndList(&tagLabelPairs); } // Finally ready to print the filterBys out if (count) { webIncludeResourceFile("ui.dropdownchecklist.css"); jsIncludeFile("ui.dropdownchecklist.js",NULL); #define FILTERBY_HELP_LINK "<A HREF=\"../goldenPath/help/multiView.html\" TARGET=ucscHelp>help</A>" cgiDown(0.9); printf("<B>Filter files by:</B> (select multiple %sitems - %s)\n<table><tr valign='bottom'>\n", (count >= 1 ? "categories and ":""),FILTERBY_HELP_LINK); printf("%s\n",dyStringContents(dyFilters)); printf("</tr></table>\n"); #ifdef NEW_JQUERY jsIncludeFile("ddcl.js",NULL); printf("<script type='text/javascript'>var newJQuery=true;</script>\n"); #else///ifndef NEW_JQUERY printf("<script type='text/javascript'>var newJQuery=false;</script>\n"); printf("<script type='text/javascript'>$(document).ready(function() { $('.filterBy').each( function(i) { $(this).dropdownchecklist({ firstItemChecksAll: true, noneIsAll: true, maxDropHeight: filterByMaxHeight(this) });});});</script>\n"); #endif///ndef NEW_JQUERY } dyStringFree(&dyFilters); } -return count; +return filterableBits; } static void filesDownloadsPreamble(char *db, struct trackDb *tdb) // Replacement for preamble.html which should expose parent dir, files.txt and supplemental, but // not have any specialized notes per composite. Specialized notes belong in track description. { char *server = hDownloadsServer(); char *subDir = ""; if (hIsBetaHost()) { server = "hgdownload-test.cse.ucsc.edu"; // NOTE: Force this case because beta may think subDir = "/beta"; // it's downloads server is "hgdownload.cse.ucsc.edu" } cgiDown(0.9); @@ -481,31 +606,31 @@ struct fileDb *oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, tdb->track, "supplemental"); if (oneFile != NULL) { printf("<BR>• <B><A HREF='http://%s/goldenPath/%s/%s/%s%s/supplemental/' TARGET=ucscDownloads>supplemental materials</A></B> - any related files provided by the laboratory.\n", server,db,ENCODE_DCC_DOWNLOADS, tdb->track, subDir); } if (hIsPreviewHost()) printf("<BR><b>WARNING</b>: This data is provided for early access via the Preview Browser -- it is unreviewed and subject to change. For high quality reviewed annotations, see the <a target=_blank href='http://%s/cgi-bin/hgTracks?db=%s'>Genome Browser</a>.", "genome.ucsc.edu", db); else printf("<BR><b>NOTE</b>: Early access to additional track data may be available on the <a target=_blank href='http://%s/cgi-bin/hgFileUi?db=%s&g=%s'>Preview Browser</A>.", "genome-preview.ucsc.edu", db, tdb->track); } -static int filesPrintTable(char *db, struct trackDb *parentTdb, struct fileDb *fileList, sortOrder_t *sortOrder,boolean filterable) +static int filesPrintTable(char *db, struct trackDb *parentTdb, struct fileDb *fileList, sortOrder_t *sortOrder,int filterable) // Prints filesList as a sortable table. Returns count { // Table class=sortable int columnCount = 0; int restrictedColumn = 0; char *nowrap = (sortOrder->setting != NULL ? " nowrap":""); // Sort order trackDb setting found so rely on <BR> in titles for wrapping printf("<TABLE class='sortable' style='border: 2px outset #006600;'>\n"); printf("<THEAD class='sortable'>\n"); printf("<TR class='sortable' valign='bottom'>\n"); printf("<TD align='center' valign='center'> "); int filesCount = slCount(fileList); if (filesCount > 5) printf("<em><span class='filesCount'></span>%d files</em>",filesCount); //if (sortOrder) // NOTE: This could be done to preserve sort order FIXME: However hgFileUi would need form OR changes would need to be ajaxed over AND hgsid would be needed. @@ -554,31 +679,31 @@ // Now the files... char *server = hDownloadsServer(); char *subDir = ""; if (hIsBetaHost()) { server = "hgdownload-test.cse.ucsc.edu"; // NOTE: Force this case because beta may think it's downloads server is "hgdownload.cse.ucsc.edu" subDir = "/beta"; } struct fileDb *oneFile = fileList; printf("<TBODY class='sortable sorting'>\n"); // 'sorting' is a fib but it conveniently greys the list till the table is initialized. for( ;oneFile!= NULL;oneFile=oneFile->next) { oneFile->mdb->next = NULL; // mdbs were in list for generating sortOrder, but list no longer needed char *field = NULL; - printf("<TR valign='top'%s>",filterable?" class='filterable'":""); + printf("<TR valign='top'%s>",(filterable != 0) ?" class='filterable'":""); // Download button printf("<TD nowrap>"); if (parentTdb) field = parentTdb->track; else field = mdbObjFindValue(oneFile->mdb,"composite"); assert(field != NULL); printf("<input type='button' value='Download' onclick=\"window.location='http://%s/goldenPath/%s/%s/%s%s/%s';\" title='Download %s ...'>", server,db,ENCODE_DCC_DOWNLOADS, field, subDir, oneFile->fileName, oneFile->fileName); #define SHOW_FOLDER_FRO_COMPOSITE_DOWNLOADS #ifdef SHOW_FOLDER_FRO_COMPOSITE_DOWNLOADS if (parentTdb == NULL) printf(" <A href='../cgi-bin/hgFileUi?db=%s&g=%s' title='Navigate to downloads page for %s set...'><IMG SRC='../images/folderC.png'></a> ", db,field,field); @@ -592,46 +717,47 @@ { char *align = (sameString("labVersion",sortOrder->column[ix]) || sameString("softwareVersion",sortOrder->column[ix]) ? " align='left'":" align='center'"); if (sameString("fileSize",sortOrder->column[ix])) { char niceNumber[128]; sprintWithGreekByte(niceNumber, sizeof(niceNumber), oneFile->fileSize); field = oneFile->sortFields[sortOrder->order[ix] - 1]; printf("<TD abbr='%s' align='right' nowrap>%s</td>",field,niceNumber); } else { field = oneFile->sortFields[sortOrder->order[ix] - 1]; boolean isFieldEmpty = cvTermIsEmpty(sortOrder->column[ix],field); char class[128]; class[0] = '\0'; - if (filterable) - { - enum cvSearchable searchBy = cvSearchMethod(sortOrder->column[ix]); - if (searchBy == cvSearchBySingleSelect || searchBy == cvSearchByMultiSelect) + if (filterable & (0x1<<ix)) { char *cleanClass = NULL; + char buf[256]; if (isFieldEmpty) - cleanClass = "None"; + cleanClass = CV_LABEL_EMPTY_IS_NONE; else { cleanClass = (char *)cvTag(sortOrder->column[ix],field); // class should be tag if (cleanClass == NULL) - cleanClass = field; + { + safecpy(buf,sizeof buf,field); + cleanClass = buf; + eraseNonAlphaNum(cleanClass);// This may not be needed because the filterBy code already eliminated these. } - safef(class,sizeof class," class='%s %s'",sortOrder->column[ix],cleanClass); } + safef(class,sizeof class," class='%s %s'",sortOrder->column[ix],cleanClass); } if (sameString("dateUnrestricted",sortOrder->column[ix]) && field && dateIsOld(field,"%F")) printf("<TD%s nowrap style='color: #BBBBBB;'%s>%s</td>",align,class,field); else { // use label if (!isFieldEmpty && cvTermIsCvDefined(sortOrder->column[ix])) field = (char *)cvLabel(sortOrder->column[ix],field); printf("<TD%s nowrap%s>%s</td>",align,class,isFieldEmpty?" ":field); } if (!sameString("fileType",sortOrder->column[ix])) mdbObjRemoveVars(oneFile->mdb,sortOrder->column[ix]); // Remove this from mdb now so that it isn't displayed in "extras' } } @@ -867,46 +993,46 @@ if (debug) { warn("The following files are in goldenPath/%s/%s/%s/ but NOT in the mdb:",db,ENCODE_DCC_DOWNLOADS, tdb->track); fileDbGet(db, ENCODE_DCC_DOWNLOADS, tdb->track, "listAll"); } jsIncludeFile("hui.js",NULL); jsIncludeFile("ajax.js",NULL); // standard preamble filesDownloadsPreamble(db,tdb); // Now update all files with their sortable fields and sort the list mdbObjReorderByCv(mdbList,FALSE);// Start with cv defined order for visible vars. NOTE: will not need to reorder during print! sortOrder_t *sortOrder = fileSortOrderGet(cart,tdb,mdbList); -boolean filterable = FALSE; +int filterable = 0; if (sortOrder != NULL) { char *vars = removeCommonMdbVarsNotInSortOrder(mdbList,sortOrder); if (vars) { if (debug) warn("These terms are common:%s",vars); freeMem(vars); } // Fill in and sort fileList fileDbSortList(&fileList,sortOrder); // FilterBoxes - filterable = (filterBoxesForFilesList(db,mdbList,sortOrder) > 0); + filterable = filterBoxesForFilesList(db,mdbList,sortOrder); } // Print table filesPrintTable(db,tdb,fileList,sortOrder,filterable); //fileDbFree(&fileList); // Why bother on this very long running cgi? //mdbObjsFree(&mdbList); } int fileSearchResults(char *db, struct sqlConnection *conn, struct slPair *varValPairs, char *fileType) // Prints list of files in downloads directories matching mdb search terms. Returns count { struct sqlConnection *connLocal = conn; if (conn == NULL) connLocal = hAllocConn(db); @@ -956,24 +1082,24 @@ //jsIncludeFile("ajax.js",NULL); // Print table printf("<DIV id='filesFound'>"); if (exceededLimit) { // What is the expected count? Difficult to say because of comma delimited list in fileName. if (filesExpected <= FOUND_FILE_LIMIT) filesExpected = FOUND_FILE_LIMIT + 1; printf("<DIV class='redBox' style='width: 380px;'>Too many files found. Displaying first %d of at least %d.<BR>Narrow search parameters and try again.</DIV><BR>\n", fileCount,filesExpected); //warn("Too many files found. Displaying first %d of at least %d.<BR>Narrow search parameters and try again.\n", fileCount,filesExpected); } -fileCount = filesPrintTable(db,NULL,fileList,sortOrder,FALSE); // FALSE=Don't offer more filtering on the file search page +fileCount = filesPrintTable(db,NULL,fileList,sortOrder,0); // FALSE=Don't offer more filtering on the file search page printf("</DIV><BR>\n"); //fileDbFree(&fileList); // Why bother on this very long running cgi? //mdbObjsFree(&mdbList); return fileCount; }