dd297edc013940b391151471d86ee5a17cd39be6 tdreszer Tue Apr 19 14:45:46 2011 -0700 Adding CGI support for mdb fimeName being a comma delimited list. Also includes support for md5sum as comma delimited list. diff --git src/hg/lib/fileUi.c src/hg/lib/fileUi.c index 9ca8db7..8e0673a 100644 --- src/hg/lib/fileUi.c +++ src/hg/lib/fileUi.c @@ -351,32 +351,30 @@ for(sIx = 0; sIx<sortOrder->count && differentString(commonTerms[tIx],sortOrder->column[sIx]); sIx++) ; if (sIx<sortOrder->count) // Found in sort Order so leave it in mdbObjs continue; char *val = mdbRemoveCommonVar(mdbObjs, commonTerms[tIx]); // All mdbs have it and have the same val for it. if (val) dyStringPrintf(dyCommon,"%s=%s ",commonTerms[tIx],val); } return dyStringCannibalize(&dyCommon); } return NULL; } -#define FILTER_THE_FILES -#ifdef FILTER_THE_FILES static char *labelWithVocabLink(char *var,char *title,struct slPair *valsAndLabels,boolean tagsNotVals) /* If the parentTdb has a controlledVocabulary setting and the vocabType is found, then label will be wrapped with the link to all relevent terms. Return string is cloned. */ { // Determine if the var is cvDefined. If not, simple link boolean cvDefined = FALSE; struct hash *cvTypesOfTerms = (struct hash *)cvTermTypeHash(); if (cvTypesOfTerms != NULL) { struct hash *cvTermDef = hashFindVal(cvTypesOfTerms,var); if (cvTermDef) cvDefined = SETTING_IS_ON(hashFindVal(cvTermDef,"cvDefined")); } struct dyString *dyLink = dyStringNew(256); @@ -450,31 +448,30 @@ if (count) { webIncludeResourceFile("ui.dropdownchecklist.css"); jsIncludeFile("ui.dropdownchecklist.js",NULL); #define FILTERBY_HELP_LINK "<A HREF=\"../goldenPath/help/multiView.html\" TARGET=ucscHelp>help</A>" printf("<B>Filter files by:</B> (select multiple %sitems - %s)\n<table><tr valign='bottom'>\n", (count >= 1 ? "categories and ":""),FILTERBY_HELP_LINK); printf("%s\n",dyStringContents(dyFilters)); printf("</tr></table>\n"); printf("<script type='text/javascript'>$(document).ready(function() { $('.filterBy').each( function(i) { $(this).dropdownchecklist({ firstItemChecksAll: true, noneIsAll: true });});});</script>\n"); } dyStringFree(&dyFilters); } return count; } -#endif///def FILTER_THE_FILES static void filesDownloadsPreamble(char *db, struct trackDb *tdb) { // Do not bother getting preamble.html // 1) It isn't on the RR (yet) // 2) It will likely refer back to the composite for Description info which is included in this page // 3) The rsync-to-tmp-file hurdle isn't worth the effort. puts("<p><B>Data is <A HREF='http://genome.ucsc.edu/ENCODE/terms.html'>RESTRICTED FROM USE</a>"); puts("in publication until the restriction date noted for the given data file.</B></p>"); char *server = hDownloadsServer(); char *subDir = ""; if (hIsBetaHost()) { server = "hgdownload-test.cse.ucsc.edu"; // NOTE: Force this case because beta may think it's downloads server is "hgdownload.cse.ucsc.edu" @@ -661,291 +658,331 @@ // Restriction policy link in later column? if (restrictedColumn > 1) printf("</TD><TH colspan=%d align='left'><A HREF='%s' TARGET=BLANK style='font-size:.9em;'>Restriction Policy</A>", columnCount,ENCODE_DATA_RELEASE_POLICY); printf("</TD></TR>\n"); printf("</TFOOT></TABLE><BR>\n"); if (parentTdb == NULL) printf("<script type='text/javascript'>{$(document).ready(function() {sortTableInitialize($('table.sortable')[0],true,true);});}</script>\n"); return filesCount; } -void filesDownloadUi(char *db, struct cart *cart, struct trackDb *tdb) -// UI for a "composite like" track: This will list downloadable files associated with -// a single trackDb entry (composite or of type "downloadsOnly". The list of files -// will have links to their download and have metadata information associated. -// The list will be a sortable table and there may be filtering controls. -{ - // The basic idea: - // 1) tdb of composite or type=downloadsOnly tableless track - // 2) All mdb Objs associated with "composite=tdb->track" and having fileName - // 3) Verification of each file in its discovered location - // 4) Lookup of 'fileSortOrder' - // 5) TODO: present filter controls - // 6) Presort of files list - // 7) make table class=sortable - // 8) Final file count - // 9) Use trackDb settings to get at html description - // Nice to have: Make filtering and sorting persistent (saved to cart) - -// FIXME: Trick while developing: -if (tdb->table != NULL) - tdb->track = tdb->table; - -boolean debug = cartUsualBoolean(cart,"debug",FALSE); - -struct sqlConnection *conn = hAllocConn(db); -char *mdbTable = mdbTableName(conn,TRUE); // Look for sandBox name first -if(mdbTable == NULL) - errAbort("TABLE NOT FOUND: '%s.%s'.\n",db,MDB_DEFAULT_NAME); - -// Get an mdbObj list of all that belong to this track and have a fileName -char buf[256]; -safef(buf,sizeof(buf),"%s=%s %s=?",MDB_VAR_COMPOSITE,tdb->track,MDB_VAR_FILENAME); -struct mdbByVar *mdbVars = mdbByVarsLineParse(buf); -struct mdbObj *mdbList = mdbObjsQueryByVars(conn,mdbTable,mdbVars); - -// Now get Indexes But be sure not to duplicate entries in the list!!! -safef(buf,sizeof(buf),"%s=%s %s= %s!=",MDB_VAR_COMPOSITE,tdb->track,MDB_VAR_FILEINDEX,MDB_VAR_FILENAME); -mdbVars = mdbByVarsLineParse(buf); -mdbList = slCat(mdbList, mdbObjsQueryByVars(conn,mdbTable,mdbVars)); -mdbObjRemoveHiddenVars(mdbList); -hFreeConn(&conn); - -if (slCount(mdbList) == 0) +static int filesFindInDir(char *db, struct mdbObj **pmdbFiles, struct fileDb **pFileList, char *fileType, int limit,boolean *exceededLimit) +// Prints list of files in downloads directories matching mdb search terms. Returns count { - warn("No files specified in metadata for: %s\n%s",tdb->track,tdb->longLabel); - return; - } - +int fileCount = 0; // Verify file existance and make fileList of those found struct fileDb *fileList = NULL, *oneFile = NULL; // Will contain found files struct mdbObj *mdbFiles = NULL; // Will caontain a list of mdbs for the found files -while(mdbList) +struct mdbObj *mdbList = *pmdbFiles; +while (mdbList && (limit == 0 || fileCount < limit)) { - char buf[512]; boolean found = FALSE; struct mdbObj *mdbFile = slPopHead(&mdbList); + char *composite = mdbObjFindValue(mdbFile,MDB_VAR_COMPOSITE); + if (composite == NULL) + { + mdbObjsFree(&mdbFile); + continue; + } + // First for FileName char *fileName = mdbObjFindValue(mdbFile,MDB_VAR_FILENAME); - if (fileName != NULL) + if (fileName == NULL) { - oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, tdb->track, fileName); - if (oneFile) + mdbObjsFree(&mdbFile); + continue; + } + +//#define NO_FILENAME_LISTS +#ifdef NO_FILENAME_LISTS + oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, composite, fileName); + if (oneFile == NULL) + { + mdbObjsFree(&mdbFile); + continue; + } + + //warn("%s == %s",fileType,oneFile->fileType); + if (isEmpty(fileType) || sameWord(fileType,"Any") + || (oneFile->fileType && sameWord(fileType,oneFile->fileType))) + { + slAddHead(&fileList,oneFile); + oneFile->mdb = mdbFile; + slAddHead(&mdbFiles,oneFile->mdb); + found = TRUE; + fileCount++; + if (limit > 0 && fileCount >= limit) + break; + } + else + fileDbFree(&oneFile); + +#else///ifndef NO_FILENAME_LISTS + + struct slName *fileSet = slNameListFromComma(fileName); + struct slName *md5Set = NULL; + char *md5sums = mdbObjFindValue(mdbFile,MDB_VAR_MD5SUM); + if (md5sums != NULL) + md5Set = slNameListFromComma(md5sums); + while (fileSet != NULL) + { + struct slName *file = slPopHead(&fileSet); + struct slName *md5 = NULL; + if (md5Set) + md5 = slPopHead(&md5Set); + oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, composite, file->name); + if (oneFile == NULL) + { + slNameFree(&file); + if (md5) + slNameFree(&md5); + continue; + } + + //warn("%s == %s",fileType,oneFile->fileType); + if (isEmpty(fileType) || sameWord(fileType,"Any") + || (oneFile->fileType && startsWithWordByDelimiter(fileType,'.',oneFile->fileType))) // Starts with! This ensures both bam and bam.bai are found. { slAddHead(&fileList,oneFile); + if (found) // if already found then need two mdbObjs (assertable but then this is metadata) + oneFile->mdb = mdbObjClone(mdbFile); // Yes clone this as differences will occur + else oneFile->mdb = mdbFile; + if (md5 != NULL) + mdbObjSetVar(oneFile->mdb,MDB_VAR_MD5SUM,md5->name); + else + mdbObjRemoveVars(oneFile->mdb,MDB_VAR_MD5SUM); slAddHead(&mdbFiles,oneFile->mdb); found = TRUE; + fileCount++; + if (limit > 0 && fileCount >= limit) + { + slNameFreeList(&fileSet); + if (md5Set) + slNameFreeList(&md5Set); + break; } - else if (debug) - warn("goldenPath/%s/%s/%s/%s in mdb but not found in directory",db,ENCODE_DCC_DOWNLOADS, tdb->track,fileName); } + else + fileDbFree(&oneFile); + + slNameFree(&file); + if (md5) + slNameFree(&md5); + } +#endif///ndef NO_FILENAME_LISTS + + // FIXME: This support of fileIndex and implicit bam.bai's should be removed when mdb is cleaned up. // Now for FileIndexes - if (fileName && endsWith(fileName,".bam")) // Special to fill in missing .bam.bai's + if (limit == 0 || fileCount < limit) + { + char buf[512]; + if (strchr(fileName,',') == NULL && endsWith(fileName,".bam")) // Special to fill in missing .bam.bai's { safef(buf,sizeof(buf),"%s.bai",fileName); fileName = buf; } else - fileName = mdbObjFindValue(mdbFile,MDB_VAR_FILEINDEX); - if (fileName != NULL) { + fileName = mdbObjFindValue(mdbFile,MDB_VAR_FILEINDEX); // This mdb var should be going away. + if (fileName == NULL) + continue; + } + // Verify existance first - oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, tdb->track, fileName); - if (oneFile) + oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, composite, fileName); // NOTE: won't be found if already found in comma delimited fileName! + if (oneFile == NULL) + continue; + + //warn("%s == %s",fileType,oneFile->fileType); + if (isEmpty(fileType) || sameWord(fileType,"Any") + || (oneFile->fileType && sameWord(fileType,oneFile->fileType)) + || (oneFile->fileType && sameWord(fileType,"bam") && sameWord("bam.bai",oneFile->fileType))) // TODO: put fileType matching into search.c lib code to segregate index logic. { slAddHead(&fileList,oneFile); if (found) // if already found then need two mdbObjs (assertable but then this is metadata) - oneFile->mdb = mdbObjClone(mdbFile); // Do we really need to clone this? + oneFile->mdb = mdbObjClone(mdbFile); else oneFile->mdb = mdbFile; + mdbObjRemoveVars(oneFile->mdb,MDB_VAR_MD5SUM); slAddHead(&mdbFiles,oneFile->mdb); + fileCount++; found = TRUE; + continue; } - else if (debug) - warn("goldenPath/%s/%s/%s/%s in mdb but not found in directory",db,ENCODE_DCC_DOWNLOADS, tdb->track,fileName); + else + fileDbFree(&oneFile); } + if (!found) mdbObjsFree(&mdbFile); } +*pmdbFiles = mdbFiles; +*pFileList = fileList; +if (exceededLimit != NULL) + *exceededLimit = FALSE; +if (mdbList != NULL) + { + if (exceededLimit != NULL) + *exceededLimit = TRUE; + mdbObjsFree(&mdbList); + } +return fileCount; +} -if (slCount(fileList) == 0) +void filesDownloadUi(char *db, struct cart *cart, struct trackDb *tdb) +// UI for a "composite like" track: This will list downloadable files associated with +// a single trackDb entry (composite or of type "downloadsOnly". The list of files +// will have links to their download and have metadata information associated. +// The list will be a sortable table and there may be filtering controls. +{ +boolean debug = cartUsualBoolean(cart,"debug",FALSE); + +struct sqlConnection *conn = hAllocConn(db); +char *mdbTable = mdbTableName(conn,TRUE); // Look for sandBox name first +if(mdbTable == NULL) + errAbort("TABLE NOT FOUND: '%s.%s'.\n",db,MDB_DEFAULT_NAME); + +// Get an mdbObj list of all that belong to this track and have a fileName +char buf[256]; +safef(buf,sizeof(buf),"%s=%s %s=?",MDB_VAR_COMPOSITE,tdb->track,MDB_VAR_FILENAME); +struct mdbByVar *mdbVars = mdbByVarsLineParse(buf); +struct mdbObj *mdbList = mdbObjsQueryByVars(conn,mdbTable,mdbVars); + +// Now get Indexes But be sure not to duplicate entries in the list!!! +safef(buf,sizeof(buf),"%s=%s %s= %s!=",MDB_VAR_COMPOSITE,tdb->track,MDB_VAR_FILEINDEX,MDB_VAR_FILENAME); +mdbVars = mdbByVarsLineParse(buf); +mdbList = slCat(mdbList, mdbObjsQueryByVars(conn,mdbTable,mdbVars)); +mdbObjRemoveHiddenVars(mdbList); +hFreeConn(&conn); + +if (slCount(mdbList) == 0) + { + warn("No files specified in metadata for: %s\n%s",tdb->track,tdb->longLabel); + return; + } + +// Verify file existance and make fileList of those found +struct fileDb *fileList = NULL; // Will contain found files + +int fileCount = filesFindInDir(db, &mdbList, &fileList, NULL, 0, NULL); +assert(fileCount == slCount(fileList)); + +if (fileCount == 0) { warn("No downloadable files currently available for: %s\n%s",tdb->track,tdb->longLabel); return; // No files so nothing to do. } if (debug) { warn("The following files are in goldenPath/%s/%s/%s/ but NOT in the mdb:",db,ENCODE_DCC_DOWNLOADS, tdb->track); fileDbGet(db, ENCODE_DCC_DOWNLOADS, tdb->track, "listAll"); } jsIncludeFile("hui.js",NULL); jsIncludeFile("ajax.js",NULL); // standard preamble filesDownloadsPreamble(db,tdb); // Now update all files with their sortable fields and sort the list -mdbObjReorderByCv(mdbFiles,FALSE);// Start with cv defined order for visible vars. NOTE: will not need to reorder during print! -sortOrder_t *sortOrder = fileSortOrderGet(cart,tdb,mdbFiles); +mdbObjReorderByCv(mdbList,FALSE);// Start with cv defined order for visible vars. NOTE: will not need to reorder during print! +sortOrder_t *sortOrder = fileSortOrderGet(cart,tdb,mdbList); boolean filterable = FALSE; if (sortOrder != NULL) { - char *vars = removeCommonMdbVarsNotInSortOrder(mdbFiles,sortOrder); + char *vars = removeCommonMdbVarsNotInSortOrder(mdbList,sortOrder); if (vars) { if (debug) warn("These terms are common:%s",vars); freeMem(vars); } // Fill in and sort fileList fileDbSortList(&fileList,sortOrder); - // FilterBoxes ? -#ifdef FILTER_THE_FILES - filterable = (filterBoxesForFilesList(db,mdbFiles,sortOrder) > 0); -#endif///def FILTER_THE_FILES + // FilterBoxes + filterable = (filterBoxesForFilesList(db,mdbList,sortOrder) > 0); } // Print table filesPrintTable(db,tdb,fileList,sortOrder,filterable); //fileDbFree(&fileList); // Why bother on this very long running cgi? +//mdbObjsFree(&mdbList); } int fileSearchResults(char *db, struct sqlConnection *conn, struct slPair *varValPairs, char *fileType) // Prints list of files in downloads directories matching mdb search terms. Returns count { struct sqlConnection *connLocal = conn; if (conn == NULL) connLocal = hAllocConn(db); struct mdbObj *mdbList = mdbObjRepeatedSearch(connLocal,varValPairs,FALSE,TRUE); if (conn == NULL) hFreeConn(&connLocal); if (slCount(mdbList) == 0) { printf("<DIV id='filesFound'><BR>No files found.<BR></DIV><BR>\n"); return 0; } // Now sort mdbObjs so that composites will stay together and lookup of files will be most efficient mdbObjsSortOnVars(&mdbList, MDB_VAR_COMPOSITE); mdbObjRemoveHiddenVars(mdbList); #define FOUND_FILE_LIMIT 1000 -int fileCount = 0; -// Verify file existance and make fileList of those found -struct fileDb *fileList = NULL, *oneFile = NULL; // Will contain found files -struct mdbObj *mdbFiles = NULL; // Will caontain a list of mdbs for the found files -while(mdbList && fileCount < FOUND_FILE_LIMIT) // TODO: collapse this and the hgFileUi version into a single function. - { - char buf[512]; - boolean found = FALSE; - struct mdbObj *mdbFile = slPopHead(&mdbList); - char *composite = mdbObjFindValue(mdbFile,MDB_VAR_COMPOSITE); - if (composite != NULL) - { - // First for FileName - char *fileName = mdbObjFindValue(mdbFile,MDB_VAR_FILENAME); - if (fileName != NULL) - { - oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, composite, fileName); - if (oneFile) - { - //warn("%s == %s",fileType,oneFile->fileType); - if (isEmpty(fileType) || sameWord(fileType,"Any") - || (oneFile->fileType && sameWord(fileType,oneFile->fileType))) - { - slAddHead(&fileList,oneFile); - oneFile->mdb = mdbFile; - slAddHead(&mdbFiles,oneFile->mdb); - fileCount++; - found = TRUE; - if (fileCount == FOUND_FILE_LIMIT) - break; - } - } - else - fileDbFree(&oneFile); - } - // Now for FileIndexes - if (fileName && endsWith(fileName,".bam")) // Special to fill in missing .bam.bai's - { - safef(buf,sizeof(buf),"%s.bai",fileName); - fileName = buf; - } - else - fileName = mdbObjFindValue(mdbFile,MDB_VAR_FILEINDEX); - if (fileName != NULL) - { - // Verify existance first - oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, composite, fileName); - if (oneFile) - { - //warn("%s == %s",fileType,oneFile->fileType); - if (isEmpty(fileType) || sameWord(fileType,"Any") - || (oneFile->fileType && sameWord(fileType,oneFile->fileType)) - || (oneFile->fileType && sameWord(fileType,"bam") && sameWord("bam.bai",oneFile->fileType))) // TODO: put fileType matching into search.c lib code to segregate index logic. - { - slAddHead(&fileList,oneFile); - if (found) // if already found then need two mdbObjs (assertable but then this is metadata) - oneFile->mdb = mdbObjClone(mdbFile); // Do we really need to clone this? - else - oneFile->mdb = mdbFile; - slAddHead(&mdbFiles,oneFile->mdb); - fileCount++; - found = TRUE; - continue; - } - else - fileDbFree(&oneFile); - } - } - } - if (!found) - mdbObjsFree(&mdbFile); - } -if (slCount(fileList) == 0) +struct fileDb *fileList = NULL; // Will contain found files +int filesExpected = slCount(mdbList); +boolean exceededLimit = FALSE; +int fileCount = filesFindInDir(db, &mdbList, &fileList, fileType, FOUND_FILE_LIMIT, &exceededLimit); +assert(fileCount == slCount(fileList)); + +if (fileCount == 0) { printf("<DIV id='filesFound'><BR>No files found.<BR></DIV><BR>\n"); return 0; // No files so nothing to do. } // TODO Could sort on varValPairs by creating a sortOrder struct of them //// Now update all files with their sortable fields and sort the list -mdbObjReorderByCv(mdbFiles,FALSE);// Start with cv defined order for visible vars. NOTE: will not need to reorder during print! -sortOrder_t *sortOrder = fileSortOrderGet(NULL,NULL,mdbFiles); // No cart, no tdb +mdbObjReorderByCv(mdbList,FALSE);// Start with cv defined order for visible vars. NOTE: will not need to reorder during print! +sortOrder_t *sortOrder = fileSortOrderGet(NULL,NULL,mdbList); // No cart, no tdb if (sortOrder != NULL) { // Fill in and sort fileList fileDbSortList(&fileList,sortOrder); } -mdbObjRemoveVars(mdbFiles,"tableName"); // Remove this from mdb now so that it isn't displayed in "extras' +mdbObjRemoveVars(mdbList,"tableName"); // Remove this from mdb now so that it isn't displayed in "extras' //jsIncludeFile("hui.js",NULL); //jsIncludeFile("ajax.js",NULL); // Print table printf("<DIV id='filesFound'>"); -if (mdbList != NULL) +if (exceededLimit) { - printf("<DIV class='redBox' style='width: 380px;'>Too many files found. Displaying first %d of potentially %d.<BR>Narrow search parameters and try again.</DIV><BR>\n", - fileCount,(fileCount+slCount(mdbList)*2)); // Multiply*2 because of fileIndexes - //warn("Too many files found. Displaying first %d of potentially %d.<BR>Narrow search parameters and try again.\n", fileCount,(fileCount+slCount(mdbList)*2)); // Multiply because of fileIndexes - mdbObjsFree(&mdbList); + // What is the expected count? Difficult to say because of comma delimited list in fileName. + if (filesExpected <= FOUND_FILE_LIMIT) + filesExpected = FOUND_FILE_LIMIT + 1; + + printf("<DIV class='redBox' style='width: 380px;'>Too many files found. Displaying first %d of at least %d.<BR>Narrow search parameters and try again.</DIV><BR>\n", + fileCount,filesExpected); + //warn("Too many files found. Displaying first %d of at least %d.<BR>Narrow search parameters and try again.\n", fileCount,filesExpected); } fileCount = filesPrintTable(db,NULL,fileList,sortOrder,FALSE); // FALSE=Don't offer more filtering on the file search page printf("</DIV><BR>\n"); //fileDbFree(&fileList); // Why bother on this very long running cgi? +//mdbObjsFree(&mdbList); + return fileCount; }