src/hg/instinct/lib/featuresLib.c 1.30

1.30 2009/06/04 03:42:49 jsanborn
added copyright notices, removed cluster library
Index: src/hg/instinct/lib/featuresLib.c
===================================================================
RCS file: /projects/compbio/cvsroot/kent/src/hg/instinct/lib/featuresLib.c,v
retrieving revision 1.29
retrieving revision 1.30
diff -b -B -U 1000000 -r1.29 -r1.30
--- src/hg/instinct/lib/featuresLib.c	12 Jan 2009 20:48:00 -0000	1.29
+++ src/hg/instinct/lib/featuresLib.c	4 Jun 2009 03:42:49 -0000	1.30
@@ -1,881 +1,886 @@
+/********************************************************************************/
+/* Copyright 2007-2009 -- The Regents of the University of California           */
+/********************************************************************************/
+
+
 /* ispyFeatures.c -- enables access of ispy clinical features for feature sorter */
 
 #include "common.h"
 #include "hash.h"
 #include "linefile.h"
 #include "cheapcgi.h"
 #include "localmem.h"
 #include "dystring.h"
 #include "obscure.h"
 #include "jksql.h"
 #include "hdb.h"
 #include "sqlList.h"
 #include "hPrint.h"
 #include "hgHeatmapLib.h"
 #include "featuresLib.h"
 #include "filterFeatures.h"
 #include "ra.h"
 
 #define CODES_TABLE "codes"
 #define CODES_VAL "val"
 #define CODES_CODE "code"
 #define CODES_COLNAME "colName"
 #define CODES_UNDEFINED "Undefined"
 
 /*** Global Variables defined here ***/
 struct hash *columnHash;           /* Hash of active columns keyed by name. */
 struct cart *cart;	/* This holds cgi and other variables between clicks. */
 char *genome;
 char *database;
 
 char *configVarName(struct column *col, char *varName, char *patDb)
 /* Return variable name for configuration. */
 {
 static char name[256];
 safef(name, sizeof(name), "%s.%s.%s.%s", colConfigPrefix, col->name, varName, patDb);
 return name;
 }
 
 char *configVarVal(struct column *col, char *varName, char *patDb)
 /* Return value for configuration variable.  Return NULL if it
  * doesn't exist or if it is "" */
 {
 if (!cart) 
     return NULL;
 
 char *name = configVarName(col, varName,patDb);
 return cartNonemptyString(cart, name);
 }
 
 char *lookupItemUrlVal(struct column *col, char *sVal,
                        struct sqlConnection *conn)
 {
 char query[512];
 safef(query, sizeof(query), col->itemUrlQuery, sVal);
 return sqlQuickString(conn, query);
 }
 
 char *columnSetting(struct column *col, char *name, char *defaultVal)
 /* Return value of named setting in column, or default if it doesn't exist. */
 {
 char *result = hashFindVal(col->settings, name);
 if (result == NULL)
     result = defaultVal;
 return result;
 }
 
 int columnSettingInt(struct column *col, char *name, int defaultVal)
 /* Return value of named integer setting or default if it doesn't exist. */
 {
 char *result = hashFindVal(col->settings, name);
 if (result == NULL)
     return defaultVal;
 return atoi(result);
 }
 
 static void hPrintSpaces(int count)
 /* Print count number of spaces. */
 {
 while (--count >= 0)
     hPrintf("&nbsp");
 }
 
 char *colInfoUrl(struct column *col)
 /* Return URL to get column info.  freeMem this when done. */
 {
 if (!cart)
     return NULL;
 
 char *labelUrl;
 if ((labelUrl = columnSetting(col, "labelUrl", NULL)) != NULL)
     return labelUrl;
 else
     {
     char url[512];
     safef(url, sizeof(url), "../cgi-bin/hgHeatmap?%s&%s=%s",
 	  cartSidUrlString(cart), colInfoVarName, col->name);
     return cloneString(url);
     }
 }
 
 void colInfoAnchor(struct column *col)
 /* Print anchor tag that leads to column info page. */
 {
 char *url = colInfoUrl(col);
 hPrintf("<A HREF=\"%s\">", url);
 freeMem(url);
 }
 
 void colInfoLink(struct column *col)
 /* Print link to column. */
 {
 colInfoAnchor(col);
 hPrintf("%s</A>", col->shortLabel);
 }
 
 
 void labelSimplePrint(struct column *col)
 /* This just prints cell->shortLabel.  If colWidth is
  * set it will add spaces, center justifying it.  */
 {
 int colWidth = columnSettingInt(col, "colWidth", 0);
   
 hPrintf("<TH ALIGN=LEFT VALIGN=BOTTOM><B><PRE>");
 /* The <PRE> above helps Internet Explorer avoid wrapping
  * in the label column, which helps us avoid wrapping in
  * the data columns below.  Wrapping in the data columns
  * makes the expression display less effective so we try                                          
  * to minimize it.  -jk */
 if (colWidth == 0)
     {
     colInfoLink(col);
     }
 else
     {
     int labelLen = strlen(col->shortLabel);
     int diff = colWidth - labelLen;
     if (diff < 0) diff = 0;
     colInfoLink(col);
     hPrintSpaces(diff);
     }
 hPrintf("</PRE></B></TH>");
 }
 
 void cellSimplePrintExt(struct column *col, struct slName *id,
                         struct sqlConnection *conn, boolean lookupForUrl)
      /* This just prints one field from table. */
 {
 char *s = col->cellVal(col, id, conn);
 hPrintf("<TD>");
 if (s == NULL)
     {
     hPrintf("n/a");
     }
 else
     {
     if (col->itemUrl != NULL)
         {
 	hPrintf("<A HREF=\"");
 	char *sVal = id->name;
 	if (lookupForUrl)
             sVal = s;
 	if (col->itemUrlQuery)
             sVal = lookupItemUrlVal(col, sVal, conn);
 	hPrintf(col->itemUrl, sVal);
 	if (col->itemUrlQuery)
             freez(&sVal);
 	hPrintf("\"");
 	hPrintf(">");
 	hPrintNonBreak(s);
 	hPrintf("</A>");
         }
     else
         {
 	hPrintNonBreak(s);
         }
     freeMem(s);
     }
 hPrintf("</TD>");
 }
 
 void cellSimplePrint(struct column *col, struct slName *id,
                      struct sqlConnection *conn)
 /* This just prints one field from table. */
 {
 cellSimplePrintExt(col, id, conn, TRUE);
 }
 
 static boolean alwaysExists(struct column *col, struct sqlConnection *conn)
 /* We always exist. */
 {
 return TRUE;
 }
 
 boolean simpleTableExists(struct column *col, struct sqlConnection *conn)
 /* This returns true if col->table exists. */
 {
 return sqlTableExists(conn, col->table);
 }
 
 static char *noVal(struct column *col, struct slName *id, struct sqlConnection *conn)
 /* Return not-available value. */
 {
 return cloneString("n/a");
 }
 
 static char *noColVal(struct column *col, struct sqlConnection *conn)
 /* Return not-available value. */
 {
 return cloneString("n/a");
 }
 
 static int oneColumn(struct column *col)
 /* Return that we have single column. */
 {
 return 1;
 }
 
 void columnDefaultMethods(struct column *col)
 /* Set up default methods. */
 {
 col->exists = alwaysExists;
 col->cellVal = noVal;
 col->cellMinVal = noColVal;
 col->cellMaxVal = noColVal;
 col->cellAvgVal = noColVal;
 col->cellPrint = cellSimplePrint;
 col->labelPrint = labelSimplePrint;
 col->tableColumns = oneColumn;
 }
 
 void cellSelfLinkPrint(struct column *col, struct slName *id,
                        struct sqlConnection *conn)
 /* Print self and hyperlink to make this the search term. */
 {
 char *s = col->cellVal(col, id, conn);
 if (s == NULL)
     s = cloneString("n/a");
 hPrintf("<TD>");
 
 hPrintf("%s</A></TD>", s);
 freeMem(s);
 }
 
 static char *numberVal(struct column *col, struct slName *id,
                        struct sqlConnection *conn)
 /* Return incrementing number. */
 {
 static int ix = 0;
 char buf[15];
 ++ix;
 safef(buf, sizeof(buf), "%d", ix);
 return cloneString(buf);
 }
 
 void setupColumnNum(struct column *col, char *parameters)
 /* Set up column that displays index in displayed list. */
 {
 col->cellVal = numberVal;
 col->cellPrint = cellSelfLinkPrint;
 }
 
 void setupColumnString(struct column *col, char *parameters)
 /* Set up column that displays index in displayed list. */
 {
 return;
 }
 
 
 /* ---- Simple table lookup type columns ---- */
 
 char *cellLookupVal(struct column *col, struct slName *id, 
 		    struct sqlConnection *conn)
 /* Get a field in a table defined by col->table, col->keyField, 
  * col->valField.  If an xrefLookup is specified in col->settings,
  * use that to look up an alternate name for the result. */
 {
 char *xrefDb = hashFindVal(col->settings, "xrefDb");
 char *xrefTable = hashFindVal(col->settings, "xrefTable");
 char *xrefNameField = hashFindVal(col->settings, "xrefNameField");
 char *xrefAliasField = hashFindVal(col->settings, "xrefAliasField");
 char query[1024];
 if (xrefDb)
     safef(query, sizeof(query),
 	  "select %s.%s.%s "
 	  "from %s.%s, %s "
 	  "where %s.%s = '%s' "
 	  "and %s.%s = %s.%s.%s;",
 	  xrefDb, xrefTable, xrefAliasField,
 	  xrefDb, xrefTable,   col->table,
 	  col->table, col->keyField,   id->name,
 	  col->table, col->valField,   xrefDb, xrefTable, xrefNameField);
 else
     safef(query, sizeof(query), "select %s from %s where %s = '%s'",
 	  col->valField, col->table, col->keyField, id->name);
 
 return sqlQuickString(conn, query);
 }
 
 boolean cellLookupCoded(struct column *col, struct sqlConnection *conn)
 {
 if (!sqlTableExists(conn, CODES_TABLE))
     return FALSE;
 
 if (!col->filterType)
     return FALSE;
 
 if (!sameString(col->filterType, "coded"))
     return FALSE;
 
 char query[256];
 safef(query, sizeof(query), 
       "select %s from %s where %s = \"%s\"",
       CODES_VAL, CODES_TABLE, CODES_COLNAME, col->valField);
 
 return sqlExists(conn, query);
 }
 
 char *cellLookupCodedVal(struct column *col, struct slName *id, struct sqlConnection *conn)
 {
 char *cellVal = col->cellVal(col, id, conn);
 if (!sqlTableExists(conn, CODES_TABLE))
     return cellVal;
 
 if (!col->filterType)
     return FALSE;
 
 if (!sameString(col->filterType, "coded"))
     return cellVal;
 
 char query[256];
 safef(query, sizeof(query),
       "select %s from %s where %s = '%s' and %s = %s",
       CODES_VAL, CODES_TABLE, CODES_COLNAME, col->name, CODES_CODE, cellVal);
 
 return sqlQuickString(conn, query);
 }  
 
 struct slName *cellLookupCodedVals(struct column *col, struct sqlConnection *conn)
 {
 if (!sqlTableExists(conn, CODES_TABLE))
     return NULL;
 
 char query[256];
 safef(query, sizeof(query), 
       "select %s from %s where %s = \"%s\"",
       CODES_VAL, CODES_TABLE, CODES_COLNAME, col->valField);
 struct slName *slList = sqlQuickList(conn, query);
 
 /* Need to add "Undefined" code at end of list, to allow users to select NULL
  * clinical data */
 slNameAddTail(&slList, CODES_UNDEFINED); 
 
 return slList;
 }
 
 char *cellLookupMinVal(struct column *col, struct sqlConnection *conn)
 /* Get minimum value of column in database */
 {
 char *min = cloneString(hashFindVal(col->settings, "min"));
 if (min)
     return min;
 char query[512];
 safef(query, sizeof(query), "select min(%s) from %s", col->valField, col->table);
 return sqlQuickString(conn, query);
 }
 
 char *cellLookupMaxVal(struct column *col, struct sqlConnection *conn)
 /* Get maximum value of lookup column */
 {
 char *max = cloneString(hashFindVal(col->settings, "max"));
 if (max)
     return max;
 
 char query[512];
 safef(query, sizeof(query), "select max(%s) from %s", col->valField, col->table);
 return sqlQuickString(conn, query);
 }
 
 char *cellLookupMinCutVal(struct column *col, struct sqlConnection *conn)
 /* Get minimum cutoff value of column in database, any value smaller than this will be displayed as null */
 {
 char *minCut = cloneString(hashFindVal(col->settings, "minCut"));
 if (minCut)
     return minCut;
 else
     return NULL;
 }
 
 char *cellLookupMaxCutVal(struct column *col, struct sqlConnection *conn)
 /* Get maximum cutoff value of column in database, any value greater than this will be displayed as null */
 {
 char *maxCut = cloneString(hashFindVal(col->settings, "maxCut"));
 if (maxCut)
     return maxCut;
 else
     return NULL;
 }
 
 char *cellLookupAvgVal(struct column *col, struct sqlConnection *conn)
 /* Get average value of column in database */
 {
 char query[512];
 safef(query, sizeof(query), "select avg(%s) from %s", col->valField, col->table);
 return sqlQuickString(conn, query);
 }
 
 char *cellLookupOffsetVal(struct column *col)
 /* Get minimum cutoff value of column in database, any value smaller than this will be displayed as null */
 {
 char *offset = cloneString(hashFindVal(col->settings, "offset"));
 if (offset)
     return offset;
 else
     return "0";
 }
 
 char *cellLookupColorReverseVal(struct column *col)
 /* Get minimum cutoff value of column in database, any value smaller than this will be displayed as null */
 {
 char *reverse = cloneString(hashFindVal(col->settings, "colorReverse"));
 if (reverse)
     return reverse;
 else
     return "1";
 }
 
 char *cellLookupColorScheme(struct column *col)
 /* Get color scheme for column in database */
 {
 char *color = cloneString(hashFindVal(col->settings, "colorScheme"));
 if (color)
     return color;
 else
     return "default";
 }
 
 static void cellLookupConfigControls(struct column *col, char *patDb)
 {
 hPrintf("<TD>");
 char *name = configVarName(col, "sortType", patDb);
 char *curVal = "unsorted";
 if (cart)
     curVal = cartUsualString(cart, name, "unsorted");
 
 static char *choices[3] = {"ascending", "unsorted", "descending"};
 hPrintf("sort: ");
 cgiMakeDropList(name, choices, ArraySize(choices), curVal);    
 hPrintf("</TD>");
 }
 
 void setupColumnLookup(struct column *col, char *parameters, char *patDb)
 /* Set up column that just looks up one field in a table
  * keyed by the geneId. */
 {
 //char *xrefLookup = cloneString(hashFindVal(col->settings, "xrefLookup"));
 col->table = cloneString(nextWord(&parameters));
 col->keyField = cloneString(nextWord(&parameters));
 col->valField = cloneString(nextWord(&parameters));
 if (col->valField == NULL)
     errAbort("Not enough fields in type lookup for %s", col->name);
 col->exists = simpleTableExists;
 col->cellVal = cellLookupVal;
 col->cellAvgVal = cellLookupAvgVal;
 col->cellMinVal = cellLookupMinVal;
 col->cellMaxVal = cellLookupMaxVal;
 col->cellMinCutVal= cellLookupMinCutVal;
 col->cellMaxCutVal= cellLookupMaxCutVal;
 col->cellOffsetVal= cellLookupOffsetVal;
 col->cellColorReverseVal= cellLookupColorReverseVal;
 col->cellColorSchemeVal = cellLookupColorScheme;
 
 col->cellCoded = cellLookupCoded;
 col->cellCodedVal = cellLookupCodedVal;
 col->cellCodedVals = cellLookupCodedVals;
 
 char *val = "unsorted";
 if (cart)
     val = cartUsualString(cart, configVarName(col, "sortType", patDb), "unsorted");
 
 if (sameString(val, "ascending"))
     col->cellSortDirection = 1;
 else if (sameString(val, "descending"))
     col->cellSortDirection = -1;
 else
     col->cellSortDirection = 0;
 
 col->configControls = cellLookupConfigControls;
 }
 
 struct slName *weedUnlessInHash(struct slName *inList, struct hash *hash)
 /* Return a new list with stuff not in hash removed. */
 {
 struct slName *outList = NULL, *sample, *cloneSample;
 
 for (sample = inList; sample != NULL; sample = sample->next)
     {
     if (hashLookup(hash,sample->name))
 	{
 	cloneSample = slNameNew(sample->name);
 	slSafeAddHead(&outList, cloneSample);
 	}
     }
 slReverse(&outList);
 return outList;
 }
 
 
 struct slName *advFilterFromQuery(struct sqlConnection *conn, char *query,
 				  struct slName *list)
 /* Return list of smaples from list that are returned by query */
 {
 struct hash *passHash = newHash(16);  /* Hash of samples that pass. */
 struct sqlResult *sr = sqlGetResult(conn, query);
 char **row;
 
 while ((row = sqlNextRow(sr)) != NULL)
     hashAdd(passHash, row[0], NULL);
 
 struct slName *newList = weedUnlessInHash(list, passHash);
 sqlFreeResult(&sr);
 hashFree(&passHash);
 return newList;
 }
 
 
 struct slName *lookupFloatAdvFilter(struct column *col, 
 				    struct sqlConnection *conn, 
 				    struct slName *list, 
 				    char *ghName, int subset)
 /* Do advanced filter on columns of lookup type with keys 
    in list and values as floats */
 {
 char *minString = advFilterVal(col, "min", ghName, subset);
 char *maxString = advFilterVal(col, "max", ghName, subset);
 struct slName *newList=NULL;
 
 if (minString != NULL || maxString != NULL)
     {
     struct dyString *dy = newDyString(512);
     dyStringPrintf(dy, "select %s from %s where ", col->keyField, col->table);
     if (minString && maxString)
 	dyStringPrintf(dy, "%s >= %s and %s <= %s",
 		       col->valField, minString, col->valField, maxString);
     else if (minString)
 	dyStringPrintf(dy, "%s >= %s", col->valField, minString);
     else
 	dyStringPrintf(dy, "%s <= %s", col->valField, maxString);
 
     newList = advFilterFromQuery(conn, dy->string, list);
     dyStringFree(&dy);
     return newList;
     }
 return list;
 }
 
 struct slName *lookupCodedAdvFilter(struct column *col, 
 				    struct sqlConnection *conn, 
 				    struct slName *list, 
 				    char *ghName, int subset)
 /* Do advanced filter on columns of lookup type with keys 
    in list and values as floats */
 {
 char *codes = advFilterVal(col, "codes", ghName, subset);
 struct slName *newList = NULL;
 
 if (codes != NULL)
     {
     struct slName *sl, *slList = slNameListFromComma(codes);
 
     struct dyString *dy = newDyString(512);
     dyStringPrintf(dy, "select %s from %s where ", col->keyField, col->table);
 
     for (sl = slList; sl; sl = sl->next)
 	{
 	if (sameString(sl->name, CODES_UNDEFINED))
 	    { /* If UNDEFINED code is set, must use "name is NULL" MySQL string */
 	    dyStringPrintf(dy, "%s is NULL ", col->valField);
 	    }
 	else
 	    {
 	    char query[128];
 	    safef(query, sizeof(query), 
 		  "select %s from %s where %s = \"%s\" and %s = \"%s\"", 
 		  CODES_CODE, CODES_TABLE, CODES_COLNAME, col->valField, CODES_VAL, sl->name);
 	    struct sqlResult *sr = sqlGetResult(conn, query);
 	    char **row;
 	    row = sqlNextRow(sr);
 	    if (!row)
 		return list;  // something screwy with codes, return original list?
 	    char *result = row[0];
 	    
 	    dyStringPrintf(dy, "%s = %s ", col->valField, result);
 	    sqlFreeResult(&sr);
 	    }
 	if (sl->next)
 	    dyStringPrintf(dy, "or ");
 	}
  
     newList = advFilterFromQuery(conn, dy->string, list);
     dyStringFree(&dy);
     return newList;
     }
 return list;
 }
 
 void minMaxAdvFilterControls(struct column *col, char *ghName, int subset) 
 /* Print out controls for min/max advanced filter. */
 {
 hPrintf("min: ");
 advFilterRemakeTextVar(col, "min", ghName, subset, 1);
 hPrintf("<Br>max: ");
 advFilterRemakeTextVar(col, "max", ghName, subset, 1);
 }
 
 void codedAdvFilterControls(struct column *col, char *ghName, int subset) 
 /* Print out controls for min/max advanced filter. */
 {
 return;
 }
 
 void setupColumnFilterType(struct column *col)
 /* Set up methods and column-specific variables based on
  * track type. */
 {
 if (!col->filterType)
     return;
 
 if (sameString(col->filterType, "minMax"))
     {
     col->advFilter = lookupFloatAdvFilter;
     col->filterControls = minMaxAdvFilterControls;
     }
 else if (sameString(col->filterType, "coded"))
     {
     col->advFilter = lookupCodedAdvFilter;
     col->filterControls = codedAdvFilterControls;
     }
 else
     return;
 }
 
 void setupColumnType(struct column *col, char *patDb)
 /* Set up methods and column-specific variables based on
  * track type. */
 {
 char *dupe = cloneString(col->type);
 char *s = dupe;
 char *type = nextWord(&s);
 
 columnDefaultMethods(col);
 
 if (type == NULL)
     errAbort("Missing type value for column %s", col->name);
 if (sameString(type, "num"))
     setupColumnNum(col, s);
 else if (sameString(type, "lookup"))
     setupColumnLookup(col, s, patDb);
 else if (sameString(type, "string"))
     setupColumnString(col, s);
 else
     errAbort("Unrecognized type %s for %s", col->type, col->name);
 freez(&dupe);
 }
 
 boolean columnSettingExists(struct column *col, char *name)
 /* Return TRUE if setting exists in column. */
 {
 return hashFindVal(col->settings, name) != NULL;
 }
 
 
 int columnCmpPriority(const void *va, const void *vb)
 /* Compare to sort columns based on priority. */
 {
 const struct column *a = *((struct column **)va);
 const struct column *b = *((struct column **)vb);
 float dif = a->priority - b->priority;
 if (dif < 0)
     return -1;
 else if (dif > 0)
     return 1;
 else
     return 0;
 }
 
 void refinePriorities(struct hash *colHash, char *patDb)
 /* Consult colOrderVar if it exists to reorder priorities. */
 {
 if (!cart)
     return;
 
 char varName[256];
 safef(varName, sizeof(varName),"%s.%s", colOrderVar, patDb);
 char *orig = cartOptionalString(cart, varName);
 if (orig != NULL)
     {
     char *dupe = cloneString(orig);
     char *s = dupe;
     char *name, *val;
     struct column *col;
     while ((name = nextWord(&s)) != NULL)
         {
 	if ((val = nextWord(&s)) == NULL || !isdigit(val[0]))
             {
 	    warn("Bad priority list: %s", orig);
 	    cartRemove(cart, varName);
 	    break;
             }
 	col = hashFindVal(colHash, name);
 	if (col != NULL)
             col->priority = atof(val);
         }
     freez(&dupe);
     }
 }
 
 void refineVisibility(struct column *colList)
 /* Consult cart to set on/off visibility. */
 {
 if (!cart)
     return;
 
 char varName[256], *val;
 struct column *col;
 
 for (col = colList; col != NULL; col = col->next)
     {
     safef(varName, sizeof(varName), "%s%s.vis", colConfigPrefix, col->name);
     val = cartOptionalString(cart, varName);
     if (val != NULL)
         col->on = sameString(val, "1");
     }
 }
 
 char *mustFindInRaHash(char *fileName, struct hash *raHash, char *name)
 /* Look up in ra hash or die trying. */
 {
 char *val = hashFindVal(raHash, name);
 if (val == NULL)
     errAbort("Missing required %s field in %s", name, fileName);
 return val;
 }
 
 void columnVarsFromSettings(struct column *col, char *fileName)
 /* Grab a bunch of variables from col->settings and                                             
  * move them into col proper. */
 {
 struct hash *settings = col->settings;
 col->name = mustFindInRaHash(fileName, settings, "name");
 
 col->shortLabel = mustFindInRaHash(fileName, settings, "shortLabel");
 col->longLabel = mustFindInRaHash(fileName, settings, "longLabel");
 col->priority = atof(mustFindInRaHash(fileName, settings, "priority"));
 col->on = col->defaultOn =
     sameString(mustFindInRaHash(fileName, settings, "visibility"), "on");
 col->type = mustFindInRaHash(fileName, settings, "type");
 col->filterType = hashFindVal(settings, "filterType");
 col->itemUrl = hashFindVal(settings, "itemUrl");
 col->itemUrlQuery = hashFindVal(settings, "itemUrlQuery");
 col->group = hashFindVal(settings, "group");
 }
 
 /* Return a hash of columns keyed by name. */
 static struct hash *hashColumns(struct column *colList)
 {
 struct column *col;
 struct hash *hash = hashNew(8);
 for (col = colList; col != NULL; col = col->next)
     {
     if (hashLookup(hash, col->name))
         warn("duplicate %s in column list", col->name);
     hashAdd(hash, col->name, col);
     }
 return hash;
 }
 
 struct slName *getIncludeFiles(char *raFile)
 {
 struct lineFile *lf = lineFileOpen(raFile, TRUE);
 char *line, *file;
 
 struct slName *sl, *slList = NULL;
 
 for (;;)
     {
     if (!lineFileNext(lf, &line, NULL))                                                         
 	break;                                                                                   
 
     line = skipLeadingSpaces(line);
 
     if (startsWith("#include", line) || startsWith("include", line))
 	{
 	nextWord(&line);
 	file = nextQuotedWord(&line);
 	sl = slNameNew(file);
 	slAddHead(&slList, sl);
 	}
     }
 
 lineFileClose(&lf);
 
 return slList;
 }
 
 static char *rootDir = "hgHeatmapData";
 
 struct hash *readRa(char *rootName)
 /* read datasets.ra file, including any #include .ra files */
 {
 //return hgReadRa(genome, database, rootDir, rootName, NULL);
 
 struct hash *hashOfHash = newHash(10); 
 struct hashEl *helList, *hel;
 struct hash *raList = NULL, *ra;
 
 char fileName[HDB_MAX_PATH_STRING];
 safef(fileName, sizeof(fileName), "%s/%s", rootDir, rootName);
 struct slName *sl, *slList = getIncludeFiles(fileName);
 
 raFoldIn(fileName, hashOfHash);
 for (sl = slList; sl; sl = sl->next)
     {
     safef(fileName, sizeof(fileName), "%s/%s", rootDir, sl->name);
     raFoldIn(fileName, hashOfHash);
     }
 
 /* Create list. */
 helList = hashElListHash(hashOfHash);
 for (hel = helList; hel != NULL; hel = hel->next)
     {
     ra = hel->val;
     slAddHead(&raList, ra);
     hel->val = NULL;
     }
 hashElFreeList(&helList);
 return raList;
 }
 
 struct column *getColumns(struct sqlConnection *conn, char *raName, char *patDb)
 /* Return list of columns for big table. */
 {
 if (!raName)
     return NULL;
 struct column *col, *colList = NULL;
 struct hash *raList = readRa(raName), *raHash = NULL;
 
 /* Create built-in columns. */
 if (raList == NULL)
     errAbort("Couldn't find anything from %s", raName);
 
 for (raHash = raList; raHash != NULL; raHash = raHash->next)
     {
     AllocVar(col);
     col->settings = raHash;
     columnVarsFromSettings(col, raName);
     
     if (hashFindVal(raHash, "hide"))
 	continue;
 
     setupColumnFilterType(col);    
     setupColumnType(col, patDb);
     if (col->exists(col, conn))
 	{
 	slAddHead(&colList, col);
 	}
 
     /* when the feature/col can not be found in the database, the column/feature will simply be ignored, instead of print out an error message
        This is useful for situation that part of the clinical data are private and part are public. 
        Private data tables do not exist on a public browser site. We do not want to trigger error message in such situation 
     */ 
     }
 /* Put columns in hash */
 columnHash = hashColumns(colList);
 
 /* Tweak ordering and visibilities as per user settings. */
 refinePriorities(columnHash, patDb);
 refineVisibility(colList);
 slSort(&colList, columnCmpPriority);
 
 return colList;
 }