e877a7961cdfb1b5b9540a0f51f08d3b715f0717 tdreszer Wed Mar 9 14:06:54 2011 -0800 For download file lookup, beta should look on hgdownloads-test but in beta/ subdirectory. diff --git src/hg/lib/fileUi.c src/hg/lib/fileUi.c index 70662ab..70aed8a 100644 --- src/hg/lib/fileUi.c +++ src/hg/lib/fileUi.c @@ -44,92 +44,100 @@ || savedDb == NULL || differentString(savedDb, db) || savedDir == NULL || differentString(savedDir,dir) || savedSubDir == NULL || differentString(savedSubDir,subDir)) { // free up any static mem freeMem(savedDb); freeMem(savedDir); freeMem(savedSubDir); fileDbFree(&foundFiles); FILE *scriptOutput = NULL; char buf[1024]; char cmd[512]; char *words[10]; char *server = hDownloadsServer(); - if (sameString(server,"hgdownload-test.cse.ucsc.edu")) // genome-test is different + + boolean useRsync = TRUE; + if (hIsPrivateHost()) { - // Does not work: rsync -avn rsync://hgdownload-test.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeBroadHistone + // For hgwdev (which is the same machine as "hgdownload-test.cse.ucsc.edu") rsync does not work // Use ls -log --time=ctime --time-style=long-iso /usr/local/apache/htdocs-hgdownload/goldenPath/hg19/encodeDCC/wgEncodeBroadHistone safef(cmd,sizeof(cmd),"ls -log --time-style=long-iso /usr/local/apache/htdocs-hgdownload/goldenPath/%s/%s/%s/", db,dir,subDir); + useRsync = FALSE; + } + else if (hIsBetaHost()) + { + // For hgwbeta, the files are being looked for one test in a "beta/" subdir. Have to rsync + server = "hgdownload-test.cse.ucsc.edu"; // NOTE: Force this case because beta may think it's downloads server is "hgdownload.cse.ucsc.edu" + safef(cmd,sizeof(cmd),"rsync -avn rsync://%s/goldenPath/%s/%s/%s/beta/", server, db, dir, subDir); } else // genome and hgwbeta can use rsync { // Works: rsync -avn rsync://hgdownload.cse.ucsc.edu/goldenPath/hg18/encodeDCC/wgEncodeBroadChipSeq/ - //safef(cmd,sizeof(cmd),"rsync -avn rsync://%s/goldenPath/%s/%s/%s/%s | grep %s", server, db, subDir,dir,fileName,fileName); safef(cmd,sizeof(cmd),"rsync -avn rsync://%s/goldenPath/%s/%s/%s/", server, db, dir, subDir); } //warn("cmd: %s",cmd); scriptOutput = popen(cmd, "r"); while(fgets(buf, sizeof(buf), scriptOutput)) { eraseTrailingSpaces(buf); if (!endsWith(buf,".md5sum")) // Just ignore these { int count = chopLine(buf, words); - if (count >= 6 && sameString(server,"hgdownload-test.cse.ucsc.edu")) // genome-test is different + if (count >= 6 && useRsync == FALSE) // hgwdev is same as hgdownloads-test so can't use rsync { //-rw-rw-r-- 5 502826550 2010-10-22 16:51 /usr/local/apache/htdocs-hgdownload/goldenPath/hg19/encodeDCC/wgEncodeBroadHistone/wgEncodeBroadHistoneGm12878ControlStdRawDataRep1.fastq.gz AllocVar(oneFile); oneFile->fileSize = sqlUnsignedLong(words[2]); oneFile->fileDate = cloneString(words[3]); char *atSlash = strrchr(words[5], '/'); if (atSlash != NULL) oneFile->fileName = cloneString(atSlash + 1); else oneFile->fileName = cloneString(words[5]); slAddHead(&foundFiles,oneFile); } - else if (count == 5 && differentString(server,"hgdownload-test.cse.ucsc.edu"))// genome and hgwbeta can use rsync + else if (count == 5 && useRsync == TRUE)// genome and hgwbeta can use rsync because files are on different machine { //-rw-rw-r-- 26420982 2009/09/29 14:53:30 wgEncodeBroadChipSeq/wgEncodeBroadChipSeqSignalNhlfH4k20me1.wig.gz AllocVar(oneFile); oneFile->fileSize = sqlUnsignedLong(words[1]); oneFile->fileDate = cloneString(words[2]); strSwapChar(oneFile->fileDate,'/','-');// Standardize YYYY-MM-DD, no time oneFile->fileName = cloneString(words[4]); slAddHead(&foundFiles,oneFile); } //warn("File:%s size:%ld",foundFiles->fileName,foundFiles->fileSize); } } pclose(scriptOutput); + // mark this as done to avoid excessive io + savedDb = cloneString(db); + savedDir = cloneString(dir); + savedSubDir = cloneString(subDir); + if (foundFiles == NULL) { AllocVar(oneFile); oneFile->fileName = cloneString("No files found!"); oneFile->fileDate = cloneString(cmd); slAddHead(&foundFiles,oneFile); warn("No files found for command:\n%s",cmd); return NULL; } - - // mark this as done to avoid excessive io - savedDb = cloneString(db); - savedDir = cloneString(dir); - savedSubDir = cloneString(subDir); } // special code that only gets called in debug mode if (sameString(fileName,"listAll")) { for(oneFile=foundFiles;oneFile;oneFile=oneFile->next) warn("%s",oneFile->fileName); return NULL; } // Look up the file and return it struct fileDb *newList = NULL; while (foundFiles) { oneFile = slPopHead(&foundFiles); if (sameString(fileName,oneFile->fileName)) @@ -142,30 +150,31 @@ foundFiles = slCat(foundFiles,newList); // Order does not remain the same if (oneFile != NULL && oneFile->fileType == NULL) { char *suffix = strchr(oneFile->fileName, '.'); if (suffix != NULL && strlen(suffix) > 2) { oneFile->fileType = cloneString(suffix + 1); if (endsWith(oneFile->fileType,".gz")) chopSuffix(oneFile->fileType); } } return oneFile; } + static sortOrder_t *fileSortOrderGet(struct cart *cart,struct trackDb *parentTdb,struct mdbObj *mdbObjs) /* Parses 'fileSortOrder' trackDb/cart instructions and returns a sort order struct or NULL. Some trickiness here. sortOrder->sortOrder is from cart (changed by user action), as is sortOrder->order, But columns are in original tdb order (unchanging)! However, if cart is null, all is from trackDb.ra */ { int ix; sortOrder_t *sortOrder = NULL; char *carveSetting = NULL,*setting = NULL; if (parentTdb) setting = trackDbSetting(parentTdb, FILE_SORT_ORDER); if(setting) { sortOrder = needMem(sizeof(sortOrder_t)); sortOrder->setting = cloneString(setting); carveSetting = sortOrder->setting; @@ -315,50 +324,84 @@ oneFile->sortFields[sortOrder->order[ix] - 1] = field; oneFile->reverse[ sortOrder->order[ix] - 1] = (sortOrder->forward[ix] == FALSE); } else { oneFile->sortFields[sortOrder->order[ix] - 1] = NULL; oneFile->reverse[ sortOrder->order[ix] - 1] = FALSE; } oneFile->sortFields[sortOrder->count] = NULL; } } slSort(fileList,fileDbSortCmp); } } +static char *removeCommonMdbVarsNotInSortOrder(struct mdbObj *mdbObjs,sortOrder_t *sortOrder) +{ // Removes varaibles common to all mdbObjs and not found in sortOrder. Returns allocated string oh removed var=val pairs +if (sortOrder != NULL) + { + // Remove common vars from mdbs grant=Bernstein; lab=Broad; dataType=ChipSeq; setType=exp; control=std; + // However, keep the term if it is in the sortOrder + struct dyString *dyCommon = dyStringNew(256); + char *commonTerms[] = { "grant", "lab", "dataType", "control", "setType" }; + int tIx=0,sIx = 0; + for(;tIx<ArraySize(commonTerms);tIx++) + { + for(sIx = 0; + sIx<sortOrder->count && differentString(commonTerms[tIx],sortOrder->column[sIx]); + sIx++) ; + if (sIx<sortOrder->count) // Found in sort Order so leave it in mdbObjs + continue; + + char *val = mdbRemoveCommonVar(mdbObjs, commonTerms[tIx]); // All mdbs have it and have the same val for it. + if (val) + dyStringPrintf(dyCommon,"%s=%s ",commonTerms[tIx],val); + } + return dyStringCannibalize(&dyCommon); + } +return NULL; +} + static void filesDownloadsPreamble(char *db, struct trackDb *tdb) { // Do not bother getting preamble.html // 1) It isn't on the RR (yet) // 2) It will likely refer back to the composite for Description info which is included in this page // 3) The rsync-to-tmp-file hurdle isn't worth the effort. puts("<p><B>Data is <A HREF='http://genome.ucsc.edu/ENCODE/terms.html'>RESTRICTED FROM USE</a>"); puts("in publication until the restriction date noted for the given data file.</B></p>"); +char *server = hDownloadsServer(); +char *subDir = ""; +if (hIsBetaHost()) + { + server = "hgdownload-test.cse.ucsc.edu"; // NOTE: Force this case because beta may think it's downloads server is "hgdownload.cse.ucsc.edu" + subDir = "/beta"; + } + struct fileDb *oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, tdb->track, "supplemental"); if (oneFile != NULL) { - printf("<p>\n<B>Supplemental materials</b> may be found <A HREF='http://%s/goldenPath/%s/%s/%s/supplemental/' TARGET=ucscDownloads>here</A>.</p>\n", - hDownloadsServer(),db,ENCODE_DCC_DOWNLOADS, tdb->track); + printf("<p>\n<B>Supplemental materials</b> may be found <A HREF='http://%s/goldenPath/%s/%s/%s%s/supplemental/' TARGET=ucscDownloads>here</A>.</p>\n", + server,db,ENCODE_DCC_DOWNLOADS, tdb->track, subDir); } puts("<p>\nThere are two files within this directory that contain information about the downloads:"); -printf("<BR>• <A HREF='http://%s/goldenPath/%s/%s/%s/files.txt' TARGET=ucscDownloads>files.txt</A> which is a tab-separated file with the name and metadata for each download.</LI>\n", - hDownloadsServer(),db,ENCODE_DCC_DOWNLOADS, tdb->track); -printf("<BR>• <A HREF='http://%s/goldenPath/%s/%s/%s/md5sum.txt' TARGET=ucscDownloads>md5sum.txt</A> which is a list of the md5sum output for each download.</LI>\n", - hDownloadsServer(),db,ENCODE_DCC_DOWNLOADS, tdb->track); +printf("<BR>• <A HREF='http://%s/goldenPath/%s/%s/%s%s/files.txt' TARGET=ucscDownloads>files.txt</A> which is a tab-separated file with the name and metadata for each download.</LI>\n", + server,db,ENCODE_DCC_DOWNLOADS, tdb->track, subDir); +printf("<BR>• <A HREF='http://%s/goldenPath/%s/%s/%s%s/md5sum.txt' TARGET=ucscDownloads>md5sum.txt</A> which is a list of the md5sum output for each download.</LI>\n", + server,db,ENCODE_DCC_DOWNLOADS, tdb->track, subDir); puts("<P>"); } static int filesPrintTable(char *db, struct trackDb *parentTdb, struct fileDb *fileList, sortOrder_t *sortOrder) // Prints filesList as a sortable table. Returns count { // Table class=sortable int columnCount = 0; int restrictedColumn = 0; char *nowrap = (sortOrder->setting != NULL ? " nowrap":""); // Sort order trackDb setting found so rely on <BR> in titles for wrapping printf("<TABLE class='sortable' style='border: 2px outset #006600;'>\n"); printf("<THEAD class='sortable'>\n"); printf("<TR class='sortable' valign='bottom'>\n"); @@ -398,48 +441,55 @@ } } //#define INCLUDE_FILENAMES #ifndef INCLUDE_FILENAMES else #endif///defn INCLUDE_FILENAMES { printf("<TH class='sortable sort%d' nowrap>File Name</TH>\n",++curOrder); columnCount++; } printf("<TH class='sortable sort%d' align='left' nowrap>Additional Details</TH>\n",++curOrder); columnCount++; printf("</TR></THEAD>\n"); // Now the files... +char *server = hDownloadsServer(); +char *subDir = ""; +if (hIsBetaHost()) + { + server = "hgdownload-test.cse.ucsc.edu"; // NOTE: Force this case because beta may think it's downloads server is "hgdownload.cse.ucsc.edu" + subDir = "/beta"; + } struct fileDb *oneFile = fileList; printf("<TBODY class='sortable sorting'>\n"); // 'sorting' is a fib but it conveniently greys the list till the table is initialized. for( ;oneFile!= NULL;oneFile=oneFile->next) { oneFile->mdb->next = NULL; // mdbs were in list for generating sortOrder, but list no longer needed char *field = NULL; printf("<TR valign='top'>"); // TODO: BUILD IN THE CLASSES TO ALLOW FILTERBOXES TO WORK!!! // Download button printf("<TD nowrap>"); if (parentTdb) field = parentTdb->track; else field = mdbObjFindValue(oneFile->mdb,"composite"); assert(field != NULL); - printf("<A HREF='http://%s/goldenPath/%s/%s/%s/%s' title='Download %s ...' TARGET=ucscDownloads>", - hDownloadsServer(),db,ENCODE_DCC_DOWNLOADS, field, oneFile->fileName, oneFile->fileName); + printf("<A HREF='http://%s/goldenPath/%s/%s/%s%s/%s' title='Download %s ...' TARGET=ucscDownloads>", + server,db,ENCODE_DCC_DOWNLOADS, field, subDir, oneFile->fileName, oneFile->fileName); printf("<input type='button' value='Download'>"); printf("</a>"); #define SHOW_FOLDER_FRO_COMPOSITE_DOWNLOADS #ifdef SHOW_FOLDER_FRO_COMPOSITE_DOWNLOADS if (parentTdb == NULL) printf(" <A href='../cgi-bin/hgFileUi?db=%s&g=%s' title='Navigate to downloads page for %s set...'><IMG SRC='../images/folderC.png'></a> ", db,field,field); #endif///def SHOW_FOLDER_FRO_COMPOSITE_DOWNLOADS printf("</TD>\n"); // Each of the pulled out mdb vars if (sortOrder) { for(ix=0;ix<sortOrder->count;ix++) { @@ -522,68 +572,54 @@ // 2) All mdb Objs associated with "composite=tdb->track" and having fileName // 3) Verification of each file in its discovered location // 4) Lookup of 'fileSortOrder' // 5) TODO: present filter controls // 6) Presort of files list // 7) make table class=sortable // 8) Final file count // 9) Use trackDb settings to get at html description // Nice to have: Make filtering and sorting persistent (saved to cart) // FIXME: Trick while developing: if (tdb->table != NULL) tdb->track = tdb->table; boolean debug = cartUsualBoolean(cart,"debug",FALSE); -int ix; struct sqlConnection *conn = sqlConnect(db); char *mdbTable = mdbTableName(conn,TRUE); // Look for sandBox name first if(mdbTable == NULL) errAbort("TABLE NOT FOUND: '%s.%s'.\n",db,MDB_DEFAULT_NAME); // Get an mdbObj list of all that belong to this track and have a fileName char buf[256]; safef(buf,sizeof(buf),"composite=%s fileName=?",tdb->track); struct mdbByVar *mdbVars = mdbByVarsLineParse(buf); struct mdbObj *mdbList = mdbObjsQueryByVars(conn,mdbTable,mdbVars); // Now get Indexes But be sure not to duplicate entries in the list!!! safef(buf,sizeof(buf),"composite=%s fileIndex= fileName!=",tdb->track); mdbVars = mdbByVarsLineParse(buf); mdbList = slCat(mdbList, mdbObjsQueryByVars(conn,mdbTable,mdbVars)); sqlDisconnect(&conn); if (slCount(mdbList) == 0) { warn("No files specified in metadata for: %s\n%s",tdb->track,tdb->longLabel); return; } -// Remove common vars from mdbs grant=Bernstein; lab=Broad; dataType=ChipSeq; setType=exp; control=std; -char *commonTerms[] = { "grant", "lab", "dataType", "control", "setType" }; -struct dyString *dyCommon = dyStringNew(256); -for(ix=0;ix<ArraySize(commonTerms);ix++) - { - char *val = mdbRemoveCommonVar(mdbList, commonTerms[ix]);// All mdbs have it and have the same val for it. - if (val) - dyStringPrintf(dyCommon,"%s=%s ",commonTerms[ix],val); - } -if (debug && dyStringLen(dyCommon)) - warn("These terms are common:%s",dyStringContents(dyCommon)); -dyStringFree(&dyCommon); - // Verify file existance and make fileList of those found struct fileDb *fileList = NULL, *oneFile = NULL; // Will contain found files struct mdbObj *mdbFiles = NULL; // Will caontain a list of mdbs for the found files while(mdbList) { char buf[512]; boolean found = FALSE; struct mdbObj *mdbFile = slPopHead(&mdbList); // First for FileName char *fileName = mdbObjFindValue(mdbFile,"fileName"); if (fileName != NULL) { oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, tdb->track, fileName); if (oneFile) { @@ -625,44 +661,52 @@ if (!found) mdbObjsFree(&mdbFile); } if (slCount(fileList) == 0) { warn("No downloadable files currently available for: %s\n%s",tdb->track,tdb->longLabel); return; // No files so nothing to do. } if (debug) { warn("The following files are in goldenPath/%s/%s/%s/ but NOT in the mdb:",db,ENCODE_DCC_DOWNLOADS, tdb->track); fileDbGet(db, ENCODE_DCC_DOWNLOADS, tdb->track, "listAll"); } -// FilterBoxes ? - // Dimensions - // cart contents - //boolean filterAble = dimensionsExist(tdb); - //membersForAll_t* membersForAll = membersForAllSubGroupsGet(tdb,cart); - // Now update all files with their sortable fields and sort the list sortOrder_t *sortOrder = fileSortOrderGet(cart,tdb,mdbFiles); if (sortOrder != NULL) { + char *vars = removeCommonMdbVarsNotInSortOrder(mdbFiles,sortOrder); + if (vars) + { + if (debug) + warn("These terms are common:%s",vars); + freeMem(vars); + } + // Fill in and sort fileList fileDbSortList(&fileList,sortOrder); } +// FilterBoxes ? + // Dimensions + // cart contents + //boolean filterAble = dimensionsExist(tdb); + //membersForAll_t* membersForAll = membersForAllSubGroupsGet(tdb,cart); + jsIncludeFile("hui.js",NULL); jsIncludeFile("ajax.js",NULL); // standard preamble filesDownloadsPreamble(db,tdb); // Print table filesPrintTable(db,tdb,fileList,sortOrder); //fileDbFree(&fileList); // Why bother on this very long running cgi? } int fileSearchResults(char *db, struct sqlConnection *conn, struct slPair *varValPairs, char *fileType) // Prints list of files in downloads directories matching mdb search terms. Returns count {