7466d1654758a960990796f5c049e09e475f7766
tdreszer
  Thu Dec 1 17:12:06 2011 -0800
More timing in hgFileUi and hgFileSearch led to important efficiency changes.  Most notable, I am relying upon cached rsync results now to improve speed.
diff --git src/hg/lib/fileUi.c src/hg/lib/fileUi.c
index 5bb3c09..c8b0010 100644
--- src/hg/lib/fileUi.c
+++ src/hg/lib/fileUi.c
@@ -3,30 +3,32 @@
 #include "common.h"
 #include "hash.h"
 #include "cheapcgi.h"
 #include "jsHelper.h"
 #include "cart.h"
 #include "hdb.h"
 #include "fileUi.h"
 #include "hui.h"
 #include "obscure.h"
 #include "mdb.h"
 #include "jsHelper.h"
 #include "web.h"
 #include "trashDir.h"
 
 
+static boolean timeIt = FALSE; // Can remove when satisfied with timing.
+
 void fileDbFree(struct fileDb **pFileList)
 // free one or more fileDb objects
 {
 while (pFileList && *pFileList)
     {
     struct fileDb *oneFile = slPopHead(pFileList);
 
     freeMem(oneFile->fileName);
     freeMem(oneFile->fileType);
     freeMem(oneFile->fileDate);
     freeMem(oneFile->sortFields);
     freeMem(oneFile->reverse);
     mdbObjsFree(&(oneFile->mdb));
     freeMem(oneFile);
     }
@@ -47,35 +49,37 @@
     {
     AllocVar(oneFile);
     oneFile->fileName = cloneString(words[0]);
     oneFile->fileSize = sqlUnsignedLong(words[1]);
     oneFile->fileDate = cloneString(words[2]);
     slAddHead(&fileList,oneFile);
     }
 lineFileClose(&lf);
 if (fileList == NULL)
     unlink(buFile.forCgi);    // remove empty file
 
 return fileList;
 }
 
 // Cache is not faster, so just use it as a backup
-//#define CACHE_IS_FASTER_THAN_RSYNC
+#define CACHE_IS_FASTER_THAN_RSYNC
 #ifdef CACHE_IS_FASTER_THAN_RSYNC
 static boolean fileDbBackupAvailable(char *db, char *dir, char *subDir)
 { // Checks if there is a recent enough cache file
-  // TODO: Add some other trick to invalidate cache at will.
+if (cgiVarExists("clearCache")) // Trick to invalidate cache at will.
+    return FALSE;
+
 struct tempName buFile;
 boolean exists = trashDirReusableFile(&buFile, dir, subDir, db);  // encodeDCC/composite.db
 if (exists)
     {
     struct stat mystat;
     ZeroVar(&mystat);
     if (stat(buFile.forCgi,&mystat)==0)
         {
         // how old is old?
         int secs = (clock1() - mystat.st_ctime); // seconds since created
         if (secs < (24 * 60 * 60)) // one date
             return TRUE;
         }
     }
 return FALSE;
@@ -174,31 +178,35 @@
             }
         pclose(scriptOutput);
         if (foundFiles == NULL)
             {
             foundFiles = fileDbReadFromBackup(db, dir, subDir);
             if (foundFiles == NULL)
                 {
                 AllocVar(oneFile);
                 oneFile->fileName = cloneString("No files found!");
                 oneFile->fileDate = cloneString(cmd);
                 slAddHead(&foundFiles,oneFile);
                 warn("No files found for command:\n%s",cmd);
                 }
             }
         else
+            {
             fileDbWriteToBackup(db, dir, subDir,foundFiles);
+            if (timeIt)
+                uglyTime("Successful rsync found %d files",slCount(foundFiles));
+            }
         }
 
     // mark this as done to avoid excessive io
     savedDb     = cloneString(db);
     savedDir    = cloneString(dir);
     savedSubDir = cloneString(subDir);
 
     if (foundFiles == NULL)
         return NULL;
     }
 
 // special code that only gets called in debug mode
 if (sameString(fileName,"listAll"))
     {
     for(oneFile=foundFiles;oneFile;oneFile=oneFile->next)
@@ -274,31 +282,30 @@
             setting = dyStringCannibalize(&dySortFields);
             }
         else
             dyStringFree(&dySortFields);
         mdbObjsFree(&commonVars);
         }
     if(setting == NULL) // Must be in trackDb or not a sortable list of files
         {
         #define FILE_SORT_ORDER_DEFAULT "cell=Cell_Line lab=Lab view=View replicate=Rep fileSize=Size fileType=File_Type dateSubmitted=Submitted dateUnrestricted=RESTRICTED<BR>Until"
         setting = FILE_SORT_ORDER_DEFAULT;
         }
     sortOrder = needMem(sizeof(sortOrder_t));
     carveSetting = cloneString(setting);
     sortOrder->setting = NULL;
     }
-
 if (parentTdb)
     {
     sortOrder->htmlId = needMem(strlen(parentTdb->track)+20);
     safef(sortOrder->htmlId, (strlen(parentTdb->track)+20), "%s.%s", parentTdb->track,FILE_SORT_ORDER);
     if(cart != NULL)
         sortOrder->sortOrder = cloneString(cartOptionalString(cart, sortOrder->htmlId));
     }
 
 sortOrder->count = chopByWhite(carveSetting,NULL,0);  // Get size
 sortOrder->column  = needMem(sortOrder->count*sizeof(char*));
 sortOrder->count = chopByWhite(carveSetting,sortOrder->column,sortOrder->count);
 sortOrder->title   = needMem(sortOrder->count*sizeof(char*));
 sortOrder->forward = needMem(sortOrder->count*sizeof(boolean));
 sortOrder->order   = needMem(sortOrder->count*sizeof(int));
 for (ix = 0; ix<sortOrder->count; ix++)
@@ -335,33 +342,39 @@
     }
 if (sortOrder->sortOrder == NULL)
     sortOrder->sortOrder = cloneString(setting);      // no order in cart, all power to trackDb
 return sortOrder;  // NOTE cloneString:words[0]==*sortOrder->column[0] and will be freed when sortOrder is freed
 }
 
 static int fileDbSortCmp(const void *va, const void *vb)
 // Compare two sortable tdb items based upon sort columns.
 {
 const struct fileDb *a = *((struct fileDb **)va);
 const struct fileDb *b = *((struct fileDb **)vb);
 char **fieldsA = a->sortFields;
 char **fieldsB = b->sortFields;
 int ix=0;
 int compared = 0;
-while(fieldsA[ix] != NULL && fieldsB[ix] != NULL && compared == 0)
+while(fieldsA[ix] != NULL && fieldsB[ix] != NULL)
+    {
+    compared = strcmp(fieldsA[ix], fieldsB[ix]);
+    if (compared != 0)
     {
-    compared = strcmp(fieldsA[ix], fieldsB[ix]) * (a->reverse[ix]? -1: 1);
+        if (a->reverse[ix])
+            compared *= -1;
+        break;
+        }
     ix++;
     }
 return compared;
 }
 
 static void fileDbSortList(struct fileDb **fileList, sortOrder_t *sortOrder)
 {// If sortOrder struct provided, will update sortFields in fileList, then sort it
 if (sortOrder && fileList)
     {
     struct fileDb *oneFile = NULL;
     for(oneFile = *fileList;oneFile != NULL;oneFile=oneFile->next)
         {
         oneFile->sortFields = needMem(sizeof(char *)    * (sortOrder->count + 1)); // +1 Null terminated
         oneFile->reverse    = needMem(sizeof(boolean *) *  sortOrder->count);
         int ix;
@@ -377,61 +390,67 @@
             else if (sameString("fileType",sortOrder->column[ix]))
                 field = oneFile->fileType;
             else
                 field = mdbObjFindValue(oneFile->mdb,sortOrder->column[ix]);
 
             if (field)
                 {
                 oneFile->sortFields[sortOrder->order[ix] - 1] = field;
                 oneFile->reverse[   sortOrder->order[ix] - 1] = (sortOrder->forward[ix] == FALSE);
                 }
             else
                 {
                 oneFile->sortFields[sortOrder->order[ix] - 1] = NULL;
                 oneFile->reverse[   sortOrder->order[ix] - 1] = FALSE;
                 }
-            oneFile->sortFields[sortOrder->count] = NULL;
             }
+        oneFile->sortFields[sortOrder->count] = NULL;
         }
     slSort(fileList,fileDbSortCmp);
     }
 }
 
-static char *removeCommonMdbVarsNotInSortOrder(struct mdbObj *mdbObjs,sortOrder_t *sortOrder)
-{  // Removes varaibles common to all mdbObjs and not found in sortOrder. Returns allocated string oh removed var=val pairs
+static int removeCommonMdbVarsNotInSortOrder(struct mdbObj *mdbObjs,sortOrder_t *sortOrder)
+{  // Removes varaibles common to all mdbObjs and not found in sortOrder. Returns count of vars removed
 if (sortOrder != NULL)
     {
     // Remove common vars from mdbs grant=Bernstein; lab=Broad; dataType=ChipSeq; setType=exp; control=std;
     // However, keep the term if it is in the sortOrder
+    int count = 0;
     struct dyString *dyCommon = dyStringNew(256);
     char *commonTerms[] = { "grant", "lab", "dataType", "control", "setType" };
     int tIx=0,sIx = 0;
     for(;tIx<ArraySize(commonTerms);tIx++)
         {
         for(sIx = 0;
             sIx<sortOrder->count && differentString(commonTerms[tIx],sortOrder->column[sIx]);
             sIx++) ;
         if (sIx<sortOrder->count) // Found in sort Order so leave it in mdbObjs
             continue;
 
-        char *val = mdbRemoveCommonVar(mdbObjs, commonTerms[tIx]); // All mdbs have it and have the same val for it.
-        if (val)
-            dyStringPrintf(dyCommon,"%s=%s ",commonTerms[tIx],val);
+        if (mdbObjsHasCommonVar(mdbObjs, commonTerms[tIx], TRUE)) // val the same or missing
+            {
+            count++;
+            dyStringPrintf(dyCommon,"%s ",commonTerms[tIx]);
         }
-    return dyStringCannibalize(&dyCommon);
     }
-return NULL;
+    if (count > 0)
+        mdbObjRemoveVars(mdbObjs,dyStringContents(dyCommon)); // removes from full list of mdbs
+    dyStringFree(&dyCommon);
+    return count;
+    }
+return 0;
 }
 
 static char *labelWithVocabLink(char *var,char *title,struct slPair *valsAndLabels,boolean tagsNotVals)
 /* If the parentTdb has a controlledVocabulary setting and the vocabType is found,
    then label will be wrapped with the link to all relevent terms.  Return string is cloned. */
 {
 // Determine if the var is cvDefined.  If not, simple link
 boolean cvDefined = FALSE;
 struct hash *cvTypesOfTerms = (struct hash *)cvTermTypeHash();
 if (cvTypesOfTerms != NULL)
     {
     struct hash *cvTermDef = hashFindVal(cvTypesOfTerms,var);
     if (cvTermDef)
         cvDefined = SETTING_IS_ON(hashFindVal(cvTermDef,"cvDefined"));
     }
@@ -602,31 +621,31 @@
 
 struct fileDb *oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, tdb->track, "supplemental");
 if (oneFile != NULL)
     {
     printf("<BR>&#149;&nbsp;<B><A HREF='http://%s/goldenPath/%s/%s/%s%s/supplemental/' TARGET=ucscDownloads>supplemental materials</A></B> - any related files provided by the laboratory.\n",
           server,db,ENCODE_DCC_DOWNLOADS, tdb->track, subDir);
     }
 if (hIsPreviewHost())
     printf("<BR><b>WARNING</b>: This data is provided for early access via the Preview Browser -- it is unreviewed and subject to change. For high quality reviewed annotations, see the <a target=_blank href='http://%s/cgi-bin/hgTracks?db=%s'>Genome Browser</a>.",
         "genome.ucsc.edu", db);
 else
     printf("<BR><b>NOTE</b>: Early access to additional track data may be available on the <a target=_blank href='http://%s/cgi-bin/hgFileUi?db=%s&g=%s'>Preview Browser</A>.",
         "genome-preview.ucsc.edu", db, tdb->track);
 }
 
-static int filesPrintTable(char *db, struct trackDb *parentTdb, struct fileDb *fileList, sortOrder_t *sortOrder,int filterable,boolean timeIt)
+static int filesPrintTable(char *db, struct trackDb *parentTdb, struct fileDb *fileList, sortOrder_t *sortOrder,int filterable)
 // Prints filesList as a sortable table. Returns count
 {
 if (timeIt)
     uglyTime("Start table");
 // Table class=sortable
 int columnCount = 0;
 int restrictedColumn = 0;
 char *nowrap = (sortOrder->setting != NULL ? " nowrap":""); // Sort order trackDb setting found so rely on <BR> in titles for wrapping
 printf("<TABLE class='sortable' style='border: 2px outset #006600;'>\n");
 printf("<THEAD class='sortable'>\n");
 printf("<TR class='sortable' valign='bottom'>\n");
 printf("<TD align='center' valign='center'>&nbsp;");
 int filesCount = slCount(fileList);
 if (filesCount > 5)
     printf("<em><span class='filesCount'></span>%d files</em>",filesCount);
@@ -685,31 +704,34 @@
 struct fileDb *oneFile = fileList;
 printf("<TBODY class='sortable sorting'>\n"); // 'sorting' is a fib but it conveniently greys the list till the table is initialized.
 if (timeIt)
     uglyTime("Finished column headers");
 for( ;oneFile!= NULL;oneFile=oneFile->next)
     {
     oneFile->mdb->next = NULL; // mdbs were in list for generating sortOrder, but list no longer needed
     char *field = NULL;
 
     printf("<TR valign='top'%s>",(filterable != 0) ?" class='filterable'":"");
     // Download button
     printf("<TD nowrap>");
     if (parentTdb)
         field = parentTdb->track;
     else
-        field = mdbObjFindValue(oneFile->mdb,"composite");
+        {
+        field = mdbObjFindValue(oneFile->mdb,MDB_VAR_COMPOSITE);
+        mdbObjRemoveOneVar(oneFile->mdb,MDB_VAR_COMPOSITE,NULL);
+        }
     assert(field != NULL);
 
     printf("<input type='button' value='Download' onclick=\"window.location='http://%s/goldenPath/%s/%s/%s%s/%s';\" title='Download %s ...'>",
           server,db,ENCODE_DCC_DOWNLOADS, field, subDir, oneFile->fileName, oneFile->fileName);
 
 #define SHOW_FOLDER_FRO_COMPOSITE_DOWNLOADS
 #ifdef SHOW_FOLDER_FRO_COMPOSITE_DOWNLOADS
     if (parentTdb == NULL)
         printf("&nbsp;<A href='../cgi-bin/hgFileUi?db=%s&g=%s' title='Navigate to downloads page for %s set...'><IMG SRC='../images/folderC.png'></a>&nbsp;", db,field,field);
 #endif///def SHOW_FOLDER_FRO_COMPOSITE_DOWNLOADS
     printf("</TD>\n");
 
     // Each of the pulled out mdb vars
     if (sortOrder)
         {
@@ -760,43 +782,42 @@
                 char *align = (sameString("labVersion",sortOrder->column[ix]) || sameString("softwareVersion",sortOrder->column[ix]) ? " align='left'":" align='center'");
                 if (sameString("dateUnrestricted",sortOrder->column[ix]) && field && dateIsOld(field,"%F"))
                     printf("<TD%s nowrap style='color: #BBBBBB;'%s>%s</td>",align,class,field);
                 else
                     {
                     // use label
                     if (!isFieldEmpty && termHash)
                         {
                         char *label = hashFindVal(termHash,CV_LABEL);
                         if (label != NULL)
                             field = label;
                         }
                     printf("<TD%s nowrap%s>%s</td>",align,class,isFieldEmpty?" &nbsp;":field);
                     }
                 if (!sameString("fileType",sortOrder->column[ix]))
-                    mdbObjRemoveVars(oneFile->mdb,sortOrder->column[ix]); // Remove this from mdb now so that it isn't displayed in "extras'
+                    mdbObjRemoveOneVar(oneFile->mdb,sortOrder->column[ix],NULL);
                 }
             }
         }
 #ifndef INCLUDE_FILENAMES
     else
 #endif///ndef INCLUDE_FILENAMES
         { // fileName
         printf("<TD nowrap>%s",oneFile->fileName);
         }
 
     // Extras  grant=Bernstein; lab=Broad; dataType=ChipSeq; setType=exp; control=std;
-    mdbObjRemoveVars(oneFile->mdb,MDB_VAR_FILENAME " " MDB_VAR_FILEINDEX " " MDB_VAR_COMPOSITE " " MDB_VAR_PROJECT); // Remove this from mdb now so that it isn't displayed in "extras'
     field = mdbObjVarValPairsAsLine(oneFile->mdb,TRUE,FALSE);
     printf("<TD nowrap>%s</td>",field?field:" &nbsp;");
 
     printf("</TR>\n");
     }
 if (timeIt)
     uglyTime("Finished files");
 
 printf("</TBODY><TFOOT class='bgLevel1'>\n");
 printf("<TR valign='top'>");
 
 // Restriction policy link in first column?
 if (restrictedColumn == 1)
     printf("<TH colspan=%d><A HREF='%s' TARGET=BLANK style='font-size:.9em;'>Restriction Policy</A></TH>", (columnCount - restrictedColumn),ENCODE_DATA_RELEASE_POLICY);
 
@@ -877,31 +898,31 @@
             continue;
             }
 
         //warn("%s == %s",fileType,oneFile->fileType);
         if (isEmpty(fileType) || sameWord(fileType,"Any")
         || (oneFile->fileType && startsWithWordByDelimiter(fileType,'.',oneFile->fileType))) // Starts with!  This ensures both bam and bam.bai are found.
             {
             slAddHead(&fileList,oneFile);
             if (found) // if already found then need two mdbObjs (assertable but then this is metadata)
                 oneFile->mdb = mdbObjClone(mdbFile);  // Yes clone this as differences will occur
             else
                 oneFile->mdb = mdbFile;
             if (md5 != NULL)
                 mdbObjSetVar(oneFile->mdb,MDB_VAR_MD5SUM,md5->name);
             else
-                mdbObjRemoveVars(oneFile->mdb,MDB_VAR_MD5SUM);
+                mdbObjRemoveOneVar(oneFile->mdb,MDB_VAR_MD5SUM,NULL);
             slAddHead(&mdbFiles,oneFile->mdb);
             found = TRUE;
             fileCount++;
             if (limit > 0 && fileCount >= limit)
                 {
                 slNameFreeList(&fileSet);
                 if (md5Set)
                     slNameFreeList(&md5Set);
                 break;
                 }
             }
         else
             fileDbFree(&oneFile);
 
         slNameFree(&file);
@@ -919,31 +940,31 @@
 
         // Verify existance first
         oneFile = fileDbGet(db, ENCODE_DCC_DOWNLOADS, composite, fileName);  // NOTE: won't be found if already found in comma delimited fileName!
         if (oneFile == NULL)
             continue;
 
         if (isEmpty(fileType) || sameWord(fileType,"Any")
         || (oneFile->fileType && sameWord(fileType,oneFile->fileType))
         || (oneFile->fileType && sameWord(fileType,"bam") && sameWord("bam.bai",oneFile->fileType))) // TODO: put fileType matching into search.c lib code to segregate index logic.
             {
             slAddHead(&fileList,oneFile);
             if (found) // if already found then need two mdbObjs (assertable but then this is metadata)
                 oneFile->mdb = mdbObjClone(mdbFile);
             else
                 oneFile->mdb = mdbFile;
-            mdbObjRemoveVars(oneFile->mdb,MDB_VAR_MD5SUM);
+            mdbObjRemoveOneVar(oneFile->mdb,MDB_VAR_MD5SUM,NULL);
             slAddHead(&mdbFiles,oneFile->mdb);
             fileCount++;
             found = TRUE;
             continue;
             }
         else
             fileDbFree(&oneFile);
         }
     // FIXME: This support of fileIndex should be removed when mdb is cleaned up.
 
     if (!found)
         mdbObjsFree(&mdbFile);
     }
 *pmdbFiles = mdbFiles;
 *pFileList = fileList;
@@ -952,172 +973,191 @@
 if (mdbList != NULL)
     {
     if (exceededLimit != NULL)
         *exceededLimit = TRUE;
     mdbObjsFree(&mdbList);
     }
 return fileCount;
 }
 
 void filesDownloadUi(char *db, struct cart *cart, struct trackDb *tdb)
 // UI for a "composite like" track: This will list downloadable files associated with
 // a single trackDb entry (composite or of type "downloadsOnly". The list of files
 // will have links to their download and have metadata information associated.
 // The list will be a sortable table and there may be filtering controls.
 {
-boolean timeIt = cartUsualBoolean(cart, "measureTiming",FALSE);
+timeIt = cartUsualBoolean(cart, "measureTiming",FALSE); // static to file
 if (timeIt)
-    uglyTime(NULL); // kicks off timing now
+    uglyTime("Starting file search");
+
 boolean debug = cartUsualBoolean(cart,"debug",FALSE);
 
 struct sqlConnection *conn = hAllocConn(db);
 char *mdbTable = mdbTableName(conn,TRUE); // Look for sandBox name first
 if(mdbTable == NULL)
     errAbort("TABLE NOT FOUND: '%s.%s'.\n",db,MDB_DEFAULT_NAME);
 
 // Get an mdbObj list of all that belong to this track and have a fileName
 char buf[256];
 safef(buf,sizeof(buf),"%s=%s %s=?",MDB_VAR_COMPOSITE,tdb->track,MDB_VAR_FILENAME);
 struct mdbByVar *mdbVars = mdbByVarsLineParse(buf);
 struct mdbObj *mdbList = mdbObjsQueryByVars(conn,mdbTable,mdbVars);
 
 // Now get Indexes  But be sure not to duplicate entries in the list!!!
 safef(buf,sizeof(buf),"%s=%s %s= %s!=",MDB_VAR_COMPOSITE,tdb->track,MDB_VAR_FILEINDEX,MDB_VAR_FILENAME);
 mdbVars = mdbByVarsLineParse(buf);
 mdbList = slCat(mdbList, mdbObjsQueryByVars(conn,mdbTable,mdbVars));
 mdbObjRemoveHiddenVars(mdbList);
 hFreeConn(&conn);
 
 if (mdbList)
     (void)mdbObjsFilter(&mdbList,"objStatus","re*",TRUE); // revoked, replaced, renamed
 
 if (slCount(mdbList) == 0)
     {
     warn("No files specified in metadata for: %s\n%s",tdb->track,tdb->longLabel);
     return;
     }
+if (timeIt)
+    uglyTime("Found %d mdb objects",slCount(mdbList));
 
 // Verify file existance and make fileList of those found
 struct fileDb *fileList = NULL; // Will contain found files
 
 int fileCount = filesFindInDir(db, &mdbList, &fileList, NULL, 0, NULL);
 if (timeIt)
     uglyTime("Found %d files in dir",fileCount);
 assert(fileCount == slCount(fileList));
 
 if (fileCount == 0)
     {
     warn("No downloadable files currently available for: %s\n%s",tdb->track,tdb->longLabel);
     return;  // No files so nothing to do.
     }
 if (debug)
     {
     warn("The following files are in goldenPath/%s/%s/%s/ but NOT in the mdb:",db,ENCODE_DCC_DOWNLOADS, tdb->track);
     fileDbGet(db, ENCODE_DCC_DOWNLOADS, tdb->track, "listAll");
     }
 
 jsIncludeFile("hui.js",NULL);
 jsIncludeFile("ajax.js",NULL);
 
 // standard preamble
 filesDownloadsPreamble(db,tdb);
 
+// remove these now to get them out of the way
+mdbObjRemoveVars(mdbList,MDB_VAR_FILENAME " " MDB_VAR_FILEINDEX " " MDB_VAR_COMPOSITE " " MDB_VAR_PROJECT);
+if (timeIt)
+    uglyTime("<BR>Removed 4 unwanted vars");
+
 // Now update all files with their sortable fields and sort the list
 mdbObjReorderByCv(mdbList,FALSE);// Start with cv defined order for visible vars. NOTE: will not need to reorder during print!
 sortOrder_t *sortOrder = fileSortOrderGet(cart,tdb,mdbList);
 int filterable = 0;
 if (sortOrder != NULL)
     {
-    char *vars = removeCommonMdbVarsNotInSortOrder(mdbList,sortOrder);
-    if (vars)
-        {
-        if (debug)
-            warn("These terms are common:%s",vars);
-        freeMem(vars);
-        }
+    int removed = removeCommonMdbVarsNotInSortOrder(mdbList,sortOrder);
+    if (removed && debug)
+        warn("%d terms are common and were removed",removed);
+    if (timeIt)
+        uglyTime("Removed %d common vars",removed);
 
     // Fill in and sort fileList
     fileDbSortList(&fileList,sortOrder);
     if (timeIt)
-        uglyTime("<BR>Sorted %d files on %d columns",fileCount,sortOrder->count);
+        uglyTime("Sorted %d files on %d columns",fileCount,sortOrder->count);
 
     // FilterBoxes
     filterable = filterBoxesForFilesList(db,mdbList,sortOrder);
     if (timeIt && filterable)
-        uglyTime("Created filter boxes 0x%X",filterable);
+        uglyTime("Created filter boxes");
     }
 
 // Print table
-filesPrintTable(db,tdb,fileList,sortOrder,filterable,timeIt);
+filesPrintTable(db,tdb,fileList,sortOrder,filterable);
 
 //fileDbFree(&fileList); // Why bother on this very long running cgi?
 //mdbObjsFree(&mdbList);
 }
 
-int fileSearchResults(char *db, struct sqlConnection *conn, struct slPair *varValPairs, char *fileType)
+int fileSearchResults(char *db, struct sqlConnection *conn, struct cart *cart, struct slPair *varValPairs, char *fileType)
 // Prints list of files in downloads directories matching mdb search terms. Returns count
 {
+timeIt = cartUsualBoolean(cart, "measureTiming",FALSE); // static to file
+if (timeIt)
+    uglyTime("Starting file search");
+
 struct sqlConnection *connLocal = conn;
 if (conn == NULL)
     connLocal = hAllocConn(db);
 struct mdbObj *mdbList = mdbObjRepeatedSearch(connLocal,varValPairs,FALSE,TRUE);
 if (conn == NULL)
     hFreeConn(&connLocal);
 
+mdbObjRemoveHiddenVars(mdbList);
 if (mdbList)
     (void)mdbObjsFilter(&mdbList,"objStatus","re*",TRUE); // revoked, replaced, renamed
 
 if (slCount(mdbList) == 0)
     {
     printf("<DIV id='filesFound'><BR>No files found.<BR></DIV><BR>\n");
     return 0;
     }
+if (timeIt)
+    uglyTime("Found %d mdb objects",slCount(mdbList));
 
 // Now sort mdbObjs so that composites will stay together and lookup of files will be most efficient
 mdbObjsSortOnVars(&mdbList, MDB_VAR_COMPOSITE);
-mdbObjRemoveHiddenVars(mdbList);
 
 #define FOUND_FILE_LIMIT 1000
 struct fileDb *fileList = NULL; // Will contain found files
 int filesExpected = slCount(mdbList);
 boolean exceededLimit = FALSE;
 int fileCount = filesFindInDir(db, &mdbList, &fileList, fileType, FOUND_FILE_LIMIT, &exceededLimit);
+if (timeIt)
+    uglyTime("Found %d files in dir",fileCount);
 assert(fileCount == slCount(fileList));
 
 if (fileCount == 0)
     {
     printf("<DIV id='filesFound'><BR>No files found.<BR></DIV><BR>\n");
     return 0;  // No files so nothing to do.
     }
 
+// remove these now to get them out of the way
+mdbObjRemoveVars(mdbList,MDB_VAR_FILENAME " " MDB_VAR_FILEINDEX " " MDB_VAR_PROJECT " " MDB_VAR_TABLENAME);
+if (timeIt)
+    uglyTime("Removed 4 unwanted vars");
+
 // Now update all files with their sortable fields and sort the list
 mdbObjReorderByCv(mdbList,FALSE);// Start with cv defined order for visible vars. NOTE: will not need to reorder during print!
 sortOrder_t *sortOrder = fileSortOrderGet(NULL,NULL,mdbList); // No cart, no tdb
 if (sortOrder != NULL)
     {
     // Fill in and sort fileList
     fileDbSortList(&fileList,sortOrder);
+    if (timeIt)
+        uglyTime("Sorted %d files on %d columns",fileCount,sortOrder->count);
     }
 
-mdbObjRemoveVars(mdbList,"tableName"); // Remove this from mdb now so that it isn't displayed in "extras'
-
 // Print table
 printf("<DIV id='filesFound'>");
 if (exceededLimit)
     {
     // What is the expected count?  Difficult to say because of comma delimited list in fileName.
     if (filesExpected <= FOUND_FILE_LIMIT)
         filesExpected = FOUND_FILE_LIMIT + 1;
 
     printf("<DIV class='redBox' style='width: 380px;'>Too many files found.  Displaying first %d of at least %d.<BR>Narrow search parameters and try again.</DIV><BR>\n",
            fileCount,filesExpected);
     }
 
-fileCount = filesPrintTable(db,NULL,fileList,sortOrder,0,FALSE); // 0=No columns are 'filtered' on the file search page
+fileCount = filesPrintTable(db,NULL,fileList,sortOrder,0); // 0=No columns are 'filtered' on the file search page
 printf("</DIV><BR>\n");
 
 //fileDbFree(&fileList); // Why bother on this very long running cgi?
 //mdbObjsFree(&mdbList);
 
 return fileCount;
 }