d3e338b79532a1ba0415e2a65ef1e069ba84caf1
tdreszer
Tue Jan 11 13:20:47 2011 -0800
Added suppport for file searching
diff --git src/hg/lib/fileUi.c src/hg/lib/fileUi.c
index cd0d693..dfae66a 100644
--- src/hg/lib/fileUi.c
+++ src/hg/lib/fileUi.c
@@ -1,543 +1,764 @@
/* fileUi.c - human genome file downloads common controls. */
#include "common.h"
#include "hash.h"
#include "cheapcgi.h"
#include "jsHelper.h"
#include "cart.h"
+#include "hdb.h"
#include "fileUi.h"
#include "hui.h"
#include "obscure.h"
#include "mdb.h"
// FIXME: Move to hui.h since hui.c also needs this
#define ENCODE_DCC_DOWNLOADS "encodeDCC"
+
void fileDbFree(struct fileDb **pFileList)
// free one or more fileDb objects
{
while (pFileList && *pFileList)
{
struct fileDb *oneFile = slPopHead(pFileList);
freeMem(oneFile->fileName);
+ freeMem(oneFile->fileType);
freeMem(oneFile->fileDate);
freeMem(oneFile->sortFields);
freeMem(oneFile->reverse);
mdbObjsFree(&(oneFile->mdb));
freeMem(oneFile);
}
}
struct fileDb *fileDbGet(char *db, char *dir, char *subDir, char *fileName)
// Returns NULL or if found a fileDb struct with name, size and date filled in.
{
static char *savedDb = NULL;
static char *savedDir = NULL;
static char *savedSubDir = NULL;
static struct fileDb *foundFiles = NULL;// Use static list to save excess IO
struct fileDb *oneFile = NULL;
if (foundFiles == NULL
|| savedDb == NULL || differentString(savedDb, db)
|| savedDir == NULL || differentString(savedDir,dir)
|| savedSubDir == NULL || differentString(savedSubDir,subDir))
{
// free up any static mem
freeMem(savedDb);
freeMem(savedDir);
freeMem(savedSubDir);
fileDbFree(&foundFiles);
FILE *scriptOutput = NULL;
char buf[1024];
char cmd[512];
char *words[10];
char *server = hDownloadsServer();
if (sameString(server,"hgdownload-test.cse.ucsc.edu")) // genome-test is different
{
// Does not work: rsync -avn rsync://hgdownload-test.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeBroadHistone
// Use ls -log --time=ctime --time-style=long-iso /usr/local/apache/htdocs-hgdownload/goldenPath/hg19/encodeDCC/wgEncodeBroadHistone
safef(cmd,sizeof(cmd),"ls -log --time-style=long-iso /usr/local/apache/htdocs-hgdownload/goldenPath/%s/%s/%s/", db,dir,subDir);
}
else // genome and hgwbeta can use rsync
{
- // Works: rsync -avn rsync://hgdownload.cse.ucsc.edu/goldenPath/hg18/encodeDCC/wgEncodeBroadChipSeq
- safef(cmd,sizeof(cmd),"rsync -avn rsync://%s/goldenPath/%s/%s/%s/%s | grep %s", server, db, subDir,dir,fileName,fileName);
+ // Works: rsync -avn rsync://hgdownload.cse.ucsc.edu/goldenPath/hg18/encodeDCC/wgEncodeBroadChipSeq/
+ //safef(cmd,sizeof(cmd),"rsync -avn rsync://%s/goldenPath/%s/%s/%s/%s | grep %s", server, db, subDir,dir,fileName,fileName);
+ safef(cmd,sizeof(cmd),"rsync -avn rsync://%s/goldenPath/%s/%s/%s/", server, db, dir, subDir);
}
//warn("cmd: %s",cmd);
scriptOutput = popen(cmd, "r");
while(fgets(buf, sizeof(buf), scriptOutput))
{
eraseTrailingSpaces(buf);
if (!endsWith(buf,".md5sum")) // Just ignore these
{
int count = chopLine(buf, words);
if (count >= 6 && sameString(server,"hgdownload-test.cse.ucsc.edu")) // genome-test is different
{
//-rw-rw-r-- 5 502826550 2010-10-22 16:51 /usr/local/apache/htdocs-hgdownload/goldenPath/hg19/encodeDCC/wgEncodeBroadHistone/wgEncodeBroadHistoneGm12878ControlStdRawDataRep1.fastq.gz
AllocVar(oneFile);
oneFile->fileSize = sqlUnsignedLong(words[2]);
oneFile->fileDate = cloneString(words[3]);
char *atSlash = strrchr(words[5], '/');
if (atSlash != NULL)
oneFile->fileName = cloneString(atSlash + 1);
else
oneFile->fileName = cloneString(words[5]);
slAddHead(&foundFiles,oneFile);
}
else if (count == 5 && differentString(server,"hgdownload-test.cse.ucsc.edu"))// genome and hgwbeta can use rsync
{
//-rw-rw-r-- 26420982 2009/09/29 14:53:30 wgEncodeBroadChipSeq/wgEncodeBroadChipSeqSignalNhlfH4k20me1.wig.gz
AllocVar(oneFile);
oneFile->fileSize = sqlUnsignedLong(words[1]);
oneFile->fileDate = cloneString(words[2]);
strSwapChar(oneFile->fileDate,'/','-');// Standardize YYYY-MM-DD, no time
oneFile->fileName = cloneString(words[4]);
slAddHead(&foundFiles,oneFile);
}
//warn("File:%s size:%ld",foundFiles->fileName,foundFiles->fileSize);
}
}
pclose(scriptOutput);
- //warn("found %d files",slCount(foundFiles));
if (foundFiles == NULL)
{
AllocVar(oneFile);
oneFile->fileName = cloneString("No files found!");
oneFile->fileDate = cloneString(cmd);
slAddHead(&foundFiles,oneFile);
warn("No files found for command:\n%s",cmd);
return NULL;
}
// mark this as done to avoid excessive io
savedDb = cloneString(db);
savedDir = cloneString(dir);
savedSubDir = cloneString(subDir);
}
// special code that only gets called in debug mode
if (sameString(fileName,"listAll"))
{
for(oneFile=foundFiles;oneFile;oneFile=oneFile->next)
warn("%s",oneFile->fileName);
return NULL;
}
// Look up the file and return it
struct fileDb *newList = NULL;
while (foundFiles)
{
oneFile = slPopHead(&foundFiles);
if (sameString(fileName,oneFile->fileName))
- break;
+ break; // Found means removed from list: shorter list for next file.
slAddHead(&newList,oneFile);
oneFile = NULL;
}
if (newList)
- foundFiles = slCat(newList,foundFiles);
+ foundFiles = slCat(foundFiles,newList); // Order does not remain the same
+if (oneFile != NULL && oneFile->fileType == NULL)
+ {
+ char *suffix = strchr(oneFile->fileName, '.');
+ if (suffix != NULL && strlen(suffix) > 2)
+ {
+ oneFile->fileType = cloneString(suffix + 1);
+ if (endsWith(oneFile->fileType,".gz"))
+ chopSuffix(oneFile->fileType);
+ }
+ }
return oneFile;
}
sortOrder_t *fileSortOrderGet(struct cart *cart,struct trackDb *parentTdb)
/* Parses 'fileSortOrder' trackDb/cart instructions and returns a sort order struct or NULL.
Some trickiness here. sortOrder->sortOrder is from cart (changed by user action), as is sortOrder->order,
But columns are in original tdb order (unchanging)! However, if cart is null, all is from trackDb.ra */
{
int ix;
char *setting = trackDbSetting(parentTdb, FILE_SORT_ORDER);
if(setting == NULL) // Must be in trackDb or not a sortable list of files
return NULL;
sortOrder_t *sortOrder = needMem(sizeof(sortOrder_t));
sortOrder->setting = cloneString(setting);
sortOrder->htmlId = needMem(strlen(parentTdb->track)+20);
safef(sortOrder->htmlId, (strlen(parentTdb->track)+20), "%s.%s", parentTdb->track,FILE_SORT_ORDER);
if(cart != NULL)
sortOrder->sortOrder = cloneString(cartOptionalString(cart, sortOrder->htmlId));
sortOrder->count = chopByWhite(sortOrder->setting,NULL,0); // Get size
sortOrder->column = needMem(sortOrder->count*sizeof(char*));
sortOrder->count = chopByWhite(sortOrder->setting,sortOrder->column,sortOrder->count);
sortOrder->title = needMem(sortOrder->count*sizeof(char*));
sortOrder->forward = needMem(sortOrder->count*sizeof(boolean));
sortOrder->order = needMem(sortOrder->count*sizeof(int));
for (ix = 0; ix Data is RESTRICTED FROM USE");
-puts("in publication until the restriction date noted for the given data file.
\nSupplemental materials may be found here.
\n", hDownloadsServer(),db,ENCODE_DCC_DOWNLOADS, tdb->track); } puts("\nThere are two files within this directory that contain information about the downloads:");
printf("
files.txt which is a tab-separated file with the name and metadata for each download.\n",
hDownloadsServer(),db,ENCODE_DCC_DOWNLOADS, tdb->track);
printf("
md5sum.txt which is a list of the md5sum output for each download.\n",
hDownloadsServer(),db,ENCODE_DCC_DOWNLOADS, tdb->track);
+
+
puts("
");
}
void filesDownloadUi(char *db, struct cart *cart, struct trackDb *tdb)
// UI for a "composite like" track: This will list downloadable files associated with
// a single trackDb entry (composite or of type "downloadsOnly". The list of files
// will have links to their download and have metadata information associated.
// The list will be a sortable table and there may be filtering controls.
{
// The basic idea:
// 1) tdb of composite or type=downloadsOnly tableless track
// 2) All mdb Objs associated with "composite=tdb->track" and having fileName
// 3) Verification of each file in its discovered location
// 4) Lookup of 'fileSortOrder'
// 5) TODO: present filter controls
// 6) Presort of files list
// 7) make table class=sortable
// 8) Final file count
// 9) Use trackDb settings to get at html description
// Nice to have: Make filtering and sorting persistent (saved to cart)
// FIXME: Trick while developing:
if (tdb->table != NULL)
tdb->track = tdb->table;
boolean debug = cartUsualBoolean(cart,"debug",FALSE);
int ix;
struct sqlConnection *conn = sqlConnect(db);
char *mdbTable = mdbTableName(conn,TRUE); // Look for sandBox name first
if(mdbTable == NULL)
errAbort("TABLE NOT FOUND: '%s.%s'.\n",db,MDB_DEFAULT_NAME);
struct fileDb *fileList = NULL, *oneFile = NULL;
// Get an mdbObj list of all that belong to this track and have a fileName
char buf[256];
safef(buf,sizeof(buf),"composite=%s fileName=?",tdb->track);
struct mdbByVar *mdbVars = mdbByVarsLineParse(buf);
struct mdbObj *mdbList = mdbObjsQueryByVars(conn,mdbTable,mdbVars);
// Now get Indexes But be sure not to duplicate entries in the list!!!
safef(buf,sizeof(buf),"composite=%s fileIndex= fileName!=",tdb->track);
mdbVars = mdbByVarsLineParse(buf);
mdbList = slCat(mdbList, mdbObjsQueryByVars(conn,mdbTable,mdbVars));
sqlDisconnect(&conn);
if (slCount(mdbList) == 0)
{
warn("No files specified in metadata for: %s\n%s",tdb->track,tdb->longLabel);
return;
}
// Remove common vars from mdbs grant=Bernstein; lab=Broad; dataType=ChipSeq; setType=exp; control=std;
char *commonTerms[] = { "grant", "lab", "dataType", "control", "setType" };
struct dyString *dyCommon = dyStringNew(256);
for(ix=0;ix\n");
printf("\n");
printf("
\n");
printf(" \n");
// Now the files...
printf("\n"); // 'sorting' is a fib but it conveniently greys the list till the table is initialized.
for(oneFile = fileList;oneFile!= NULL;oneFile=oneFile->next)
{
char *field = NULL;
printf(" ");
int filesCount = slCount(fileList);
if (filesCount > 5)
printf("%d files",filesCount); //puts(""); // Use this style when filterboxes are up and running
//if (sortOrder) // NOTE: This could be done to preserve sort order FIXME: However hgFileUi would need form OR changes would need to be ajaxed over AND hgsid would be needed.
// printf("",sortOrder->htmlId, sortOrder->sortOrder);
printf(" \n");
columnCount++;
// Now the columns
int curOrder = 0;
if (sortOrder)
{
curOrder = sortOrder->count;
for(ix=0;ix%s \n",
sortOrder->order[ix],(sortOrder->forward[ix]?"":" sortRev"),
(sameString("fileSize",sortOrder->column[ix])?"abbr='use' ":""),
sortOrder->title[ix]); // keeing track of sortOrder
columnCount++;
if (sameWord(sortOrder->column[ix],"dateUnrestricted"))
restrictedColumn = columnCount;
}
}
//#define INCLUDE_FILENAMES
#ifndef INCLUDE_FILENAMES
else
#endif///defn INCLUDE_FILENAMES
{
printf("File Name \n",++curOrder);
columnCount++;
}
printf("Additional Details \n",++curOrder);
columnCount++;
printf(""); // TODO: BUILD IN THE CLASSES TO ALLOW FILTERBOXES TO WORK!!!
// Download button
printf(" \n");
}
printf("\n");
printf("");
printf("",
hDownloadsServer(),db,ENCODE_DCC_DOWNLOADS, tdb->track, oneFile->fileName, oneFile->fileName);
printf("");
printf(" \n");
// Each of the pulled out mdb vars
if (sortOrder)
{
for(ix=0;ix%s ",field,niceNumber);
}
else
{
field = oneFile->sortFields[sortOrder->order[ix] - 1];
+ if (sameString("dateUnrestricted",sortOrder->column[ix]) && dateIsOld(field,"%F"))
+ field = NULL;
printf("%s ",field?field:" ");
if (!sameString("fileType",sortOrder->column[ix]))
mdbObjRemoveVars(oneFile->mdb,sortOrder->column[ix]); // Remove this from mdb now so that it isn't displayed in "extras'
}
}
}
#ifndef INCLUDE_FILENAMES
else
#endif///ndef INCLUDE_FILENAMES
{ // fileName
printf("%s",oneFile->fileName);
//// FIXME: " The "..." encapsulation could be rebuilt so it could be called here
//printf(" ...",
// oneFile->mdb->obj,oneFile->mdb->obj);
//printf(" ",oneFile->mdb->obj);
}
// Extras grant=Bernstein; lab=Broad; dataType=ChipSeq; setType=exp; control=std;
- mdbObjRemoveVars(oneFile->mdb,"fileName fileIndex composite project"); // Remove this from mdb now so that it isn't displayed in "extras'
+ mdbObjRemoveVars(oneFile->mdb,"fileName fileIndex composite project dccInternalNotes"); // Remove this from mdb now so that it isn't displayed in "extras'
mdbObjReorderVars(oneFile->mdb,"grant lab dataType cell treatment antibody protocol replicate view",FALSE); // Bring to front
mdbObjReorderVars(oneFile->mdb,"subId submittedDataVersion dateResubmitted dataVersion setType inputType controlId tableName",TRUE); // Send to back
field = mdbObjVarValPairsAsLine(oneFile->mdb,TRUE);
printf("%s ",field?field:" ");
printf("");
// Restriction policy link in first column?
if (restrictedColumn == 1)
printf(" \n");
printf("Restriction Policy ", (columnCount - restrictedColumn),ENCODE_DATA_RELEASE_POLICY);
printf(" ",(restrictedColumn > 1 ? (restrictedColumn - 1) : columnCount));
// Total
if (filesCount > 5)
printf("%d files\n",filesCount);
// Restriction policy link in later column?
if (restrictedColumn > 1)
printf(" Restriction Policy", columnCount,ENCODE_DATA_RELEASE_POLICY);
printf("
\n");
// Free mem?
}
+int fileSearchResults(char *db, struct sqlConnection *conn, struct slPair *varValPairs, char *fileType)
+// Prints list of files in downloads directories matching mdb search terms. Returns count
+{
+struct sqlConnection *connLocal = conn;
+if (conn == NULL)
+ connLocal = hAllocConn(db);
+struct mdbObj *mdbList = mdbObjRepeatedSearch(connLocal,varValPairs,FALSE,TRUE);
+if (conn == NULL)
+ hFreeConn(&connLocal);
+if (slCount(mdbList) == 0)
+ {
+ printf("
No files found.
\n");
+ return 0;
+ }
+
+// Now sort mdbObjs so that composites will stay together and lookup of files will be most efficient
+mdbObjsSortOnVars(&mdbList, "composite");
+
+// Verify file existance and make fileList of those found
+struct fileDb *fileList = NULL, *oneFile = NULL;
+while(mdbList)
+ {
+ boolean found = FALSE;
+ struct mdbObj *mdbFile = slPopHead(&mdbList);
+ char *composite = mdbObjFindValue(mdbFile,"composite");
+ if (composite != NULL)
+ {
+ // First for FileName
+ char *fileName = mdbObjFindValue(mdbFile,"fileName");
+ if (fileName != NULL)
+ {
+ oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, composite, fileName);
+ if (oneFile)
+ {
+ //warn("%s == %s",fileType,oneFile->fileType);
+ if (isEmpty(fileType) || sameWord(fileType,"Any")
+ || (oneFile->fileType && sameWord(fileType,oneFile->fileType)))
+ {
+ slAddHead(&fileList,oneFile);
+ oneFile->mdb = mdbFile;
+ found = TRUE;
+ continue;
+ }
+ }
+ else
+ fileDbFree(&oneFile);
+ }
+ // Now for FileIndexes
+ fileName = mdbObjFindValue(mdbFile,"fileIndex");
+ if (fileName != NULL)
+ {
+ // Verify existance first
+ oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, composite, fileName);
+ if (oneFile)
+ {
+ //warn("%s == %s",fileType,oneFile->fileType);
+ if (isEmpty(fileType) || sameWord(fileType,"Any")
+ || (oneFile->fileType && sameWord(fileType,oneFile->fileType)))
+ {
+ slAddHead(&fileList,oneFile);
+ if (found) // if already found then need two mdbObjs (assertable but then this is metadata)
+ oneFile->mdb = mdbObjClone(mdbFile); // Do we really need to clone this?
+ else
+ oneFile->mdb = mdbFile;
+ found = TRUE;
+ continue;
+ }
+ else
+ fileDbFree(&oneFile);
+ }
+ }
+ }
+ if (!found)
+ mdbObjsFree(&mdbFile);
+ }
+if (slCount(fileList) == 0)
+ {
+ printf("
No files found.
\n");
+ return 0; // No files so nothing to do.
+ }
+
+// TODO Could sort on varValPairs by creating a sortOrder struct of them
+//// Now update all files with their sortable fields and sort the list
+//sortOrder_t *sortOrder = fileSortOrderGet(cart,tdb);
+//if (sortOrder != NULL)
+// {
+// // Fill in and sort fileList
+// fileDbSortList(&fileList,sortOrder);
+// }
+
+//jsIncludeFile("hui.js",NULL);
+//jsIncludeFile("ajax.js",NULL);
+
+// Table class=sortable
+int columnCount = 0;
+//int restrictedColumn = 0;
+printf("\n");
+printf("\n");
+printf("
\n");
+printf(" \n");
+
+// Now the files...
+printf("\n");
+for(oneFile = fileList;oneFile!= NULL;oneFile=oneFile->next)
+ {
+ char *field = NULL;
+
+ printf(" ");
+int filesCount = slCount(fileList);
+if (filesCount > 5)
+ printf("%d files",filesCount); //puts(""); // Use this style when filterboxes are up and running
+printf(" \n");
+columnCount++;
+
+// Now the columns
+struct slPair *onePair;
+for(onePair=varValPairs; onePair != NULL; onePair=onePair->next)
+ {
+ printf("%s \n",onePair->name);
+ columnCount++;
+ }
+printf("Size \n");
+columnCount++;
+printf("File Type \n");
+columnCount++;
+printf("Submitted \n");
+columnCount++;
+printf("Restricted \n", ENCODE_DATA_RELEASE_POLICY);
+//printf("
UntilRestricted \n");
+columnCount++;
+//restrictedColumn = columnCount;
+//#define INCLUDE_FILENAMES
+#ifdef INCLUDE_FILENAMES
+printf("
UntilFile Name \n");
+columnCount++;
+#endif///defn INCLUDE_FILENAMES
+printf("Additional Details \n");
+columnCount++;
+printf(""); // TODO: BUILD IN THE CLASSES TO ALLOW FILTERBOXES TO WORK!!!
+
+ // Download button
+ printf(" \n");
+ }
+
+printf("\n");
+printf("");
+ field = mdbObjFindValue(oneFile->mdb,"composite");
+ assert(field != NULL);
+ printf("",
+ hDownloadsServer(),db,ENCODE_DCC_DOWNLOADS, field?field:" ", oneFile->fileName, oneFile->fileName);
+ printf("");
+ printf("");
+ field = mdbObjFindValue(oneFile->mdb,"composite");
+ if (field)
+ {
+ // TODO Look up trackDb.fileSortOrder. If found, then offer "folder" icon with link to page
+ }
+ puts(" ");
+
+ // Each of the pulled out mdb vars
+ for(onePair=varValPairs; onePair != NULL; onePair=onePair->next)
+ {
+ field = mdbObjFindValue(oneFile->mdb,onePair->name);
+ printf("%s ",field?field:" ");
+ mdbObjRemoveVars(oneFile->mdb,onePair->name); // Remove this from mdb now so that it isn't displayed in "extras'
+ }
+ char niceNumber[128];
+ sprintWithGreekByte(niceNumber, sizeof(niceNumber), oneFile->fileSize);
+ printf("%s ",niceNumber);
+ printf("%s ",oneFile->fileType?oneFile->fileType:" ");
+ field = mdbObjFindValue(oneFile->mdb,"dateSubmitted");
+ printf("%s ",field?field:" ");
+ field = mdbObjFindValue(oneFile->mdb,"dateUnrestricted");
+ printf("%s ",field?dateIsOld(field,"%F")?" ":field:" ");
+#ifdef INCLUDE_FILENAMES
+ printf("%s ",oneFile->fileName);
+#endif///def INCLUDE_FILENAMES
+
+ // Extras grant=Bernstein; lab=Broad; dataType=ChipSeq; setType=exp; control=std;
+ mdbObjRemoveVars(oneFile->mdb,"dateSubmitted dateUnrestricted fileName fileIndex composite project dccInternalNotes"); // Remove this from mdb now so that it isn't displayed in "extras'
+ mdbObjReorderVars(oneFile->mdb,"grant lab dataType cell treatment antibody protocol replicate view",FALSE); // Bring to front
+ mdbObjReorderVars(oneFile->mdb,"subId submittedDataVersion dateResubmitted dataVersion setType inputType controlId tableName",TRUE); // Send to back
+ field = mdbObjVarValPairsAsLine(oneFile->mdb,TRUE);
+ printf("%s ",field?field:" ");
+
+ printf("");
+
+printf(" \n");
+printf(" ",columnCount);
+//printf(" ",(restrictedColumn > 1 ? (restrictedColumn - 1) : columnCount));
+
+// Total
+if (filesCount > 5)
+ printf("%d files\n",filesCount);
+
+// Restriction policy link in later column?
+//if (restrictedColumn > 1)
+// printf(" Restriction Policy", columnCount,ENCODE_DATA_RELEASE_POLICY);
+
+printf("