src/hg/instinct/bioInt2/bioIntUI.c 1.17

1.17 2009/04/27 06:15:48 jsanborn
updated lots of stuff, will break older implementation of database
Index: src/hg/instinct/bioInt2/bioIntUI.c
===================================================================
RCS file: /projects/compbio/cvsroot/kent/src/hg/instinct/bioInt2/bioIntUI.c,v
retrieving revision 1.16
retrieving revision 1.17
diff -b -B -U 1000000 -r1.16 -r1.17
--- src/hg/instinct/bioInt2/bioIntUI.c	16 Apr 2009 19:53:55 -0000	1.16
+++ src/hg/instinct/bioInt2/bioIntUI.c	27 Apr 2009 06:15:48 -0000	1.17
@@ -1,976 +1,974 @@
 /* bioIntUI */
 #include "common.h"
 #include "bed.h"
 #include "cart.h"
 #include "linefile.h"
 #include "customTrack.h"
 #include "genoLay.h"
 #include "hash.h"
 #include "hCommon.h"
 #include "hdb.h"
 #include "hPrint.h"
 #include "htmshell.h"
 #include "hui.h"
 #include "trackLayout.h"
 #include "web.h"
 #include "microarray.h"
 #include "ra.h"
 #include "hgStatsLib.h"
 #include "featuresLib.h" 
 #include "json.h"
 #include "bioIntDb.h"
 #include "bioIntDriver.h"
 #include "bioIntUI.h"
 
 
 static char const rcsid[] = "$Id$";
 /* ---- Global variables. ---- */
 struct cart *cart;	         /* This holds cgi and other variables between clicks. */
 struct hash *oldVars;	         /* Old cart hash. */
 
 char *db = "bioInt";
 char *localDb = "localDb";
 
 void usage()
 /* Explain usage and exit. */
 {
 errAbort(
   "bioIntUI\n"
   "usage:\n"
   "   bioIntUI\n"
   );
 }
 
 /****** BEGIN HELPER FUNCTIONS *******/
 
 struct analyses *getAnalysesById(struct sqlConnection *conn, int analysis_id)
 {
 char query[256];
 safef(query, sizeof(query), 
       "select * from %s where id = %d;", 
       AN_TABLE, analysis_id);
 
 return analysesLoadByQuery(conn, query);
 }
 
 struct analyses *getAnalysesByCohortId(struct sqlConnection *conn, int cohort_id)
 {
 char query[256];
 safef(query, sizeof(query), 
       "select * from %s where cohort_id = %d;", 
       AN_TABLE, cohort_id);
 
 return analysesLoadByQuery(conn, query);
 }
 
 struct analysisFeatures *getAnalysisFeaturesByName(struct sqlConnection *conn, 
 						   char *name)
 {
 char query[256];
 safef(query, sizeof(query), 
       "select * from %s where feature_name = \"%s\"", 
       AF_TABLE, name);
 
 return analysisFeaturesLoadByQuery(conn, query);
 }
 
 struct analysisFeatures *getAnalysisFeaturesById(struct sqlConnection *conn, 
 						 int id)
 {
 char query[256];
 safef(query, sizeof(query), 
       "select * from %s where id = %d", 
       AF_TABLE, id);
 
 return analysisFeaturesLoadByQuery(conn, query);
 }
 
 
 struct features *getFeaturesByName(struct sqlConnection *conn, 
 				   char *name)
 {
 char query[256];
 safef(query, sizeof(query), 
       "select * from %s where name = \"%s\"", 
       FE_TABLE, name);
 
 return featuresLoadByQuery(conn, query);
 }
 
 struct features *getFeaturesById(struct sqlConnection *conn, 
 				 int id)
 {
 char query[256];
 safef(query, sizeof(query), 
       "select * from %s where id = %d", 
       FE_TABLE, id);
 
 return featuresLoadByQuery(conn, query);
 }
 
 struct tissues *getTissuesById(struct sqlConnection *conn, 
 			       int id)
 {
 char query[256];
 safef(query, sizeof(query), 
       "select * from %s where id = %d", 
       TI_TABLE, id);
 
 return tissuesLoadByQuery(conn, query);
 }
 
 struct datasets *getDatasetsByCohortId(struct sqlConnection *conn, int cohort_id)
 {
 char query[256];
 safef(query, sizeof(query), 
       "select %s.* from %s join %s on %s.id = %s.dataset_id "
       "where %s.cohort_id = %d",
       DA_TABLE, DA_TABLE, DC_TABLE, DA_TABLE, DC_TABLE, DC_TABLE, cohort_id);
 
 return datasetsLoadByQuery(conn, query);
 }
 
 struct cohortCorr *getCohortCorrByCohortId(struct sqlConnection *conn, int cohort_id)
 {
 char query[256];
 safef(query, sizeof(query), 
       "select * from %s where cohort_id = %d",
       CC_TABLE, cohort_id);
 
 return cohortCorrLoadByQuery(conn, query);
 }
 
 char *getFieldFromKgXref(struct sqlConnection *conn, char *geneSymbol, 
 			 char *field)
 {
 char query[256];
 safef(query, sizeof(query), 
       "select %s from %s where geneSymbol = \"%s\" ",
       field, KX_TABLE, geneSymbol);
 
 return sqlQuickString(conn, query);
 }
 
-struct pathways *getPathwayByName(struct sqlConnection *conn, char *name)
+struct genesets *getGenesetByName(struct sqlConnection *conn, char *name)
 {
 char query[256];
 safef(query, sizeof(query), 
       "select * from %s where name = \"%s\" ",
-      PA_TABLE, name);
+      GE_TABLE, name);
 
-return pathwaysLoadByQuery(conn, query);
+return genesetsLoadByQuery(conn, query);
 }
 
 void setAnalysisFeatureDesc(struct sqlConnection *conn, struct json *js, 
 			    struct analysisFeatures *af)
 {
 char *desc = getFieldFromKgXref(conn, af->feature_name, "description");
 if (!desc)
     desc = cloneString(af->feature_name);
 
 /* String 'isoform xxx' from description... almost all entries have them */
 char *ptr = rStringIn("isoform", desc);
 if (ptr)
     *ptr = '\0'; 
 
 jsonAddString(js, "description", desc);
 }
 
 void setAnalysisFeatureLink(struct sqlConnection *conn, struct json *js,
 			    struct analysisFeatures *af)
 {
 char *name, *source;
-name = getFieldFromKgXref(conn, af->feature_name, "kgId");
+name = getFieldFromKgXref(conn, af->feature_name, "geneSymbol");
 if (name)
     source = "UCSC";
 else
     {
-    struct pathways *pa = getPathwayByName(conn, af->feature_name);
-    if (!pa)
+    struct genesets *gs = getGenesetByName(conn, af->feature_name);
+    if (!gs)
 	return;
-    name = pa->name;
-    source = pa->source;
+    name = gs->name;
+    source = gs->source;
     }
 
 jsonAddString(js, "name", name);
 jsonAddString(js, "source", source);
 }
 
 
 
 /****** END HELPER FUNCTIONS *******/
 
 
 /* get a list of the current analyses */
 
 void getAnalyses()
 {
 char query[256];
 safef(query, sizeof(query),
       "select * from analyses;");
 
 struct sqlConnection *conn = hAllocConnProfile(localDb, db);
 struct analyses *an, *anList = analysesLoadByQuery(conn, query);
 
 struct json *js = newJson();
 struct json *analysis, *analyses = jsonAddContainerList(js, "analyses");
 analysis = analyses;
 for (an = anList; an; an = an->next)
     {
     jsonAddInt(analysis, "id", an->id);
     jsonAddInt(analysis, "cohort_id", an->cohort_id);
     jsonAddInt(analysis, "module_id", an->module_id);
     jsonAddString(analysis, "result_table", an->result_table);
     jsonAddString(analysis, "input_tables", an->input_tables);
     
     if (an->next)
 	analysis = jsonAddContainerToList(&analyses);    
     }
 
 if (js)
     hPrintf("%s\n", js->print(js));
 
 hFreeConn(&conn);
 }
 
 void getCohorts()
 {
 char query[256];
 safef(query, sizeof(query),
       "select * from %s;", CO_TABLE);
 
 struct sqlConnection *conn = hAllocConnProfile(localDb, db);
 struct cohorts *co, *coList = cohortsLoadByQuery(conn, query);
 
 struct json *js = newJson();
 struct json *cohort, *cohorts = jsonAddContainerList(js, "cohorts");
 cohort = cohorts;
 for (co = coList; co; co = co->next)
     {
     jsonAddInt(cohort, "cohort_id", co->id);
     jsonAddString(cohort, "name", co->name);
 
     struct datasets *da, *daList = getDatasetsByCohortId(conn, co->id);
 
     struct tissues *ti = getTissuesById(conn, daList->tissue_id);
     jsonAddString(cohort, "tissue", ti->name);
     tissuesFree(&ti);
 
     struct json *dataset, *datasets = jsonAddContainerList(cohort, "datasets");
     dataset = datasets;
     for (da = daList; da; da = da->next)
 	{
 	jsonAddString(dataset, "name", da->name);
 	jsonAddInt(dataset, "num_samples", da->num_samples);
 
 	if (da->next)
 	    dataset = jsonAddContainerToList(&datasets);
 	}
 
     if (co->next)
 	cohort = jsonAddContainerToList(&cohorts);    
     }
 
 if (js)
     hPrintf("%s\n", js->print(js));
 
 hFreeConn(&conn);
 }
 
 struct searchResults {
     struct searchResults *next;
     char *name;
-    char *source;
+    char *type;
     int length;
 };
 
 int searchResultsCmp(const void *va, const void *vb)
 /* Compare function to sort array of ints. */
 {
 const struct searchResults *a = *((struct searchResults **)va);
 const struct searchResults *b = *((struct searchResults **)vb);
 int diff = a->length - b->length;
 if (diff < 0)
     return -1;
 else if (diff > 0)
     return 1;
 else
     return 0;
 }   
 
 struct searchResults *searchForFeatures(struct sqlConnection *conn, int cohort_id, 
 					struct datasets *daList, char *feature_name)
 {
 int maxResponse = 5;
 /* Check analysis features */
 char query[256];
 safef(query, sizeof(query), 
-      "select feature_name from %s where feature_name like \"%%%s%%\" "
+      "select feature_name,type from %s where feature_name like \"%%%s%%\" "
       "order by length(feature_name);", 
       AF_TABLE, feature_name);
 
 int count = 0;
 struct searchResults *sp, *spList = NULL;
 struct sqlResult *sr = sqlGetResult(conn, query);
 char **row = NULL;
 while ((row = sqlNextRow(sr)) != NULL)
     { 
     char *name = row[0];
+    char *type = row[1];
     AllocVar(sp);
     sp->name = cloneString(name);
-    sp->source = cloneString("gene/geneset");
+    sp->type = cloneString(type);
     sp->length = strlen(sp->name);
     slAddHead(&spList, sp);
     if (count > maxResponse)
 	break;
     count++;
     }
 sqlFreeResult(&sr);
 
 /* Check clinical features */
 struct datasets *da;
 struct dyString *dy = dyStringNew(100);
 dyStringPrintf(dy,
 	       "select DISTINCT %s.name from %s join %s on %s.sample_id = %s.id ",
 	       FE_TABLE, CD_TABLE, SA_TABLE, CD_TABLE, SA_TABLE);
 dyStringPrintf(dy,
 	       "join %s on %s.id = %s.feature_id ",
 	       FE_TABLE, FE_TABLE, CD_TABLE);
 dyStringPrintf(dy,
 	       "where %s.name like \"%%%s%%\" and %s.dataset_id in (",
 	       FE_TABLE, feature_name, SA_TABLE);
 for (da = daList; da; da = da->next)
     {
     dyStringPrintf(dy, "%d", da->id);
     if (da->next)
 	dyStringPrintf(dy, ",");
     }
 dyStringPrintf(dy, ") order by length(%s.name)", FE_TABLE);     
 char *cquery = dyStringCannibalize(&dy);
 
 count = 0;
 sr = sqlGetResult(conn, cquery);
 while ((row = sqlNextRow(sr)) != NULL)
     { 
     char *name = row[0];
     AllocVar(sp);
     sp->name = cloneString(name);
-    sp->source = cloneString("clinical");
+    sp->type = cloneString("clinical");
     sp->length = strlen(sp->name);
     slAddHead(&spList, sp);
     if (count < maxResponse)
 	break;
     count++;
     }
 sqlFreeResult(&sr);
 
 slSort(&spList, searchResultsCmp);
 return spList;
 }
 
 void sendNoMatch(struct json *js)
 {
 return;
 }
 
 void sendAmbiguities(struct json *js, struct searchResults *spList)
 {
 struct searchResults *sp;
 for (sp = spList; sp; sp = sp->next)
-    jsonAddString(js, sp->name, sp->source); 
+    jsonAddString(js, sp->name, sp->type); 
 }
 
 void sendAnalysisFeatureData(struct sqlConnection *conn, struct json *js, 
 			     int cohort_id, char *feature_name)
 {
 struct analysisFeatures *af = getAnalysisFeaturesByName(conn, feature_name);
 
 if (!af)
     {
     hFreeConn(&conn);
     errAbort("Could not find analysisFeature named in %s in db", feature_name);
     }
 
 struct analyses *an, *anList = getAnalysesByCohortId(conn, cohort_id);
 if (!anList)
     {
     hFreeConn(&conn);
     errAbort("No analyses with cohort_id = %d.\n", cohort_id);
     }
 
 char query[512];
 for (an = anList; an; an = an->next)
     {
     safef(query, sizeof(query), 
-	  "select DISTINCT %s.name, %s.val from %s join %s on %s.sample_id = %s.id "
+	  "select DISTINCT %s.name, %s.conf from %s join %s on %s.sample_id = %s.id "
 	  "where %s.feature_id = %d;",
 	  SA_TABLE, an->result_table, an->result_table, SA_TABLE, an->result_table, SA_TABLE,
 	  an->result_table, af->id);
     if (sqlExists(conn, query))
 	break;
     }
 
 setAnalysisFeatureDesc(conn, js, af);
 setAnalysisFeatureLink(conn, js, af);  // e.g. 'hgg_gene' = kgId
 
 struct json *data = jsonAddContainer(js, "data");
 struct sqlResult *sr = sqlGetResult(conn, query);
 char **row = NULL;
 while ((row = sqlNextRow(sr)) != NULL)
     { 
     char *name = row[0];
     double val = atof(row[1]);
     jsonAddDouble(data, name, val);
     }
 sqlFreeResult(&sr);
 }
 
 struct samples *getOverlappingSamples(struct sqlConnection *conn, 
 				      struct datasets *daList)
 {
 struct dyString *dy = dyStringNew(100);
 int count = 1;
 dyStringPrintf(dy, "select * from %s as t%d ", SA_TABLE, count);
 count++;
 
 struct datasets *da = daList->next;
 for (da = daList; da; da = da->next)
     {
     dyStringPrintf(dy, 
 		   "join %s as t%d on t1.id = t%d.id ", 
 		   SA_TABLE, count, count);
     count++;
     }
 count = 1;
 dyStringPrintf(dy, "where ");
 for (da = daList; da; da = da->next)
     {
     dyStringPrintf(dy, "t%d.dataset_id=%d ", count, da->id);
     if (da->next)
 	dyStringPrintf(dy, "and ");
     count++;
     }
 char *query = dyStringCannibalize(&dy);
 return samplesLoadByQuery(conn, query);
 }
 
 void sendClinicalData(struct sqlConnection *conn, struct json *js,
 		      int cohort_id, char *feature_name, struct datasets *daList)
 {
 struct features *fe = getFeaturesByName(conn, feature_name);
 
 if (!fe)
     {
     hFreeConn(&conn);
     errAbort("Could not find clinical feature in db");
     }
 
 struct samples *sa, *samples = getOverlappingSamples(conn, daList);
 struct dyString *dy = dyStringNew(100);
 dyStringPrintf(dy, 
 	       "select DISTINCT %s.name, %s.val, %s.code from %s "
 	       "join %s on %s.sample_id = %s.id " 
 	       "where %s.feature_id = %d and %s.id in (", 
 	       SA_TABLE, CD_TABLE, CD_TABLE, CD_TABLE, SA_TABLE, CD_TABLE, SA_TABLE,
 	       CD_TABLE, fe->id, SA_TABLE);
 for (sa = samples; sa; sa = sa->next)
     {
     dyStringPrintf(dy, "%d", sa->id);
     if (sa->next)
 	dyStringPrintf(dy, ",");
     }
 dyStringPrintf(dy, ")");
 char *query = dyStringCannibalize(&dy);
 
 jsonAddString(js, "description", fe->longLabel);
 jsonAddString(js, "name", fe->name);
 jsonAddString(js, "source", "N/A");
 
 struct slDouble *sd;
 struct hash *codeHash = hashNew(0);
 struct json *data = jsonAddContainer(js, "data");
 struct sqlResult *sr = sqlGetResult(conn, query);
 char **row = NULL;
 while ((row = sqlNextRow(sr)) != NULL)
     { 
     char *name = row[0];
     double val = atof(row[1]);
     char *code = row[2];
 
     if (!sameString(code, "(null)")) // eventually need to fix this.
 	{
 	jsonAddString(data, name, code);
 
 	if (hashLookup(codeHash, code))
 	    continue;
 	sd = slDoubleNew(val);
 	hashAdd(codeHash, code, sd);
 	}
     else
 	jsonAddDouble(data, name, val);       
 	
     }
 if (hashNumEntries(codeHash) == 0)
     return;
 
 struct json *codes = jsonAddContainer(js, "codes");
 struct hashEl *el, *elList = hashElListHash(codeHash);
 for (el = elList; el != NULL; el = el->next)
     {
     char *name = el->name;
     sd = el->val;
 
     jsonAddDouble(codes, name, sd->val);
     }
 hashElFreeList(&elList);
 }
 
 void sendUniqueMatch(struct sqlConnection *conn, struct json *js, 
 		     int cohort_id, char *feature_names, char *sources, 
 		     struct datasets *daList)
 {
 struct slName *s, *sList = slNameListFromComma(sources);
 struct slName *f, *fList = slNameListFromComma(feature_names);
 
 if (slCount(sList) != slCount(fList))
     errAbort("source list length not equal to feature list length\n");
 
 for (s = sList, f = fList; s && f; s = s->next, f = f->next)
     {
     struct json *container = jsonAddContainer(js, f->name);
-    if (sameString(s->name, "gene/geneset"))
+    if (sameString(s->name, "gene") || sameString(s->name, "geneset"))
 	sendAnalysisFeatureData(conn, container, cohort_id, f->name);
     else if (sameString(s->name, "clinical"))
 	sendClinicalData(conn, container, cohort_id, f->name, daList);
     }
 }
 
 void getFeatureData()
 {
 int cohort_id = cartUsualInt(cart, bioIntCohortId, -1);
 if (cohort_id == -1)
     cohort_id = 2;  // hard code for first analysis during testing!
 
 /* feature source = gene,geneset,clinical... */
 char *source = cartOptionalString(cart, bioIntSourceName);
 char *feature_name = cartOptionalString(cart, bioIntFeatureName);
 if (!feature_name || !source)
     errAbort("%s and %s must be set for mode=getFeatureData\n", bioIntFeature, bioIntSourceName);
 
 struct sqlConnection *conn = hAllocConnProfile(localDb, db);
 
 struct datasets *daList = getDatasetsByCohortId(conn, cohort_id);
 if (!daList)
     errAbort("No datasets matching cohort_id = %d", cohort_id);
 
 struct json *js = newJson();
 sendUniqueMatch(conn, js, cohort_id, feature_name, source, daList);
 
 if (js)
     hPrintf("%s\n", js->print(js));
 
 hFreeConn(&conn);
 }
 
 void getSuggestions()
 {
 int cohort_id = cartUsualInt(cart, bioIntCohortId, -1);
 if (cohort_id == -1)
     cohort_id = 2;  // hard code for first analysis during testing!
 
 /* feature source = gene,geneset,clinical... */
 //char *source = cartOptionalString(cart, bioIntSourceName);
 
 char *feature_name = cartOptionalString(cart, bioIntFeatureName);
 if (!feature_name)
     errAbort("%s or %s must be set for mode=getFeatureData\n", bioIntFeature, bioIntFeatureId);
 
 struct sqlConnection *conn = hAllocConnProfile(localDb, db);
 
 struct datasets *daList = getDatasetsByCohortId(conn, cohort_id);
 if (!daList)
     errAbort("No datasets matching cohort_id = %d", cohort_id);
 
 struct json *js = newJson();
 if (sameString(feature_name, ""))  // blank was sent, return no match
     sendNoMatch(js);
 else
     {    
     struct searchResults *spList = searchForFeatures(conn, cohort_id, daList, feature_name);
     int numMatched = slCount(spList);
     if (numMatched == 0)
 	sendNoMatch(js);
     else
 	sendAmbiguities(js, spList);
     }
 
 if (js)
     hPrintf("%s\n", js->print(js));
 
 hFreeConn(&conn); 
 }
 
 void getClinicalFeatures()
 {
 int cohort_id = cartUsualInt(cart, bioIntCohortId, -1);
 if (cohort_id == -1)
     cohort_id = 2;  // hard code for first analysis during testing!
 
 struct sqlConnection *conn = hAllocConnProfile(localDb, db);
 
 struct datasets *da, *daList = getDatasetsByCohortId(conn, cohort_id);
 if (!daList)
     {
     hFreeConn(&conn);
     errAbort("No datasets matching cohort_id = %d", cohort_id);
     }
 
 struct dyString *dy = dyStringNew(100);
 dyStringPrintf(dy, 
 	       "select DISTINCT %s.* from %s join %s on %s.id = %s.feature_id "
 	       "join %s on %s.sample_id = %s.id where %s.dataset_id in (", 
 	       FE_TABLE, FE_TABLE, CD_TABLE, FE_TABLE, CD_TABLE, 
 	       SA_TABLE, CD_TABLE, SA_TABLE, SA_TABLE);
 for (da = daList; da; da = da->next)
     {
     dyStringPrintf(dy, "%d", da->id);
     if (da->next)
 	dyStringPrintf(dy, ",");
     }
 dyStringPrintf(dy, ")");
 
 char *query = dyStringCannibalize(&dy);
 struct features *fe, *feList = featuresLoadByQuery(conn, query);
 
 struct json *js = newJson();
 struct json *feature, *features = jsonAddContainerList(js, "features");
 feature = features;
 for (fe = feList; fe; fe = fe->next)
     {
     jsonAddInt(feature, "id", fe->id);
     jsonAddString(feature, "name", fe->name);
     jsonAddString(feature, "shortLabel", fe->shortLabel);
     jsonAddString(feature, "longLabel", fe->longLabel);
 
     if (fe->next)
 	feature = jsonAddContainerToList(&features);
     }
 
 if (js)
     hPrintf("%s\n", js->print(js));
 
 hFreeConn(&conn);
 }
 
 void getClinicalData()
 {
 int cohort_id = cartUsualInt(cart, bioIntCohortId, -1);
 if (cohort_id == -1)
     cohort_id = 2;  // hard code for first analysis during testing!
 
 char *feature_name = cartOptionalString(cart, bioIntFeatureName);
 if (!feature_name)
     errAbort("%s or %s must be set for mode=getClinicalData\n", bioIntFeature, bioIntFeatureId);
 
 struct sqlConnection *conn = hAllocConnProfile(localDb, db);
 
 struct features *fe = getFeaturesByName(conn, feature_name);
 if (!fe)
     {
     hFreeConn(&conn);
     errAbort("Could not find clinical feature in db");
     }
 
 struct datasets *da, *daList = getDatasetsByCohortId(conn, cohort_id);
 if (!daList)
     {
     hFreeConn(&conn);
     errAbort("No datasets matching cohort_id = %d", cohort_id);
     }
 
 struct dyString *dy = dyStringNew(100);
 dyStringPrintf(dy, 
 	       "select DISTINCT %s.name, %s.val from %s join %s on %s.sample_id = %s.id ", 
 	       SA_TABLE, CD_TABLE, CD_TABLE, SA_TABLE, CD_TABLE, SA_TABLE);
 dyStringPrintf(dy, 
 	       "where %s.feature_id = %d and %s.dataset_id in (", 
 	       CD_TABLE, fe->id, SA_TABLE);
 
 for (da = daList; da; da = da->next)
     {
     dyStringPrintf(dy, "%d", da->id);
     if (da->next)
 	dyStringPrintf(dy, ",");
     }
 dyStringPrintf(dy, ")");
 
 char *query = dyStringCannibalize(&dy);
 struct json *js = newJson();
 struct sqlResult *sr = sqlGetResult(conn, query);
 char **row = NULL;
 while ((row = sqlNextRow(sr)) != NULL)
     { 
     char *name = row[0];
     double val = atof(row[1]);
     jsonAddDouble(js, name, val);
     }
 
 if (js)
     hPrintf("%s\n", js->print(js));
 
 hFreeConn(&conn);
 }
 
 void getMostCorrelated()
 {
 int takeTop = cartUsualInt(cart, bioIntTakeTop, 5);
 
 int cohort_id = cartUsualInt(cart, bioIntCohortId, -1);
 if (cohort_id == -1)
     cohort_id = 2;  // hard code for first analysis during testing!
 
 char *feature_name = cartOptionalString(cart, bioIntFeatureName);
 if (!feature_name)
     errAbort("%s or %s must be set for mode=getMostCorrelated\n", bioIntFeature, bioIntFeatureId);
 
 struct sqlConnection *conn = hAllocConnProfile(localDb, db);
 
 struct analysisFeatures *af = getAnalysisFeaturesByName(conn, feature_name);
 if (!af)
     {
     hFreeConn(&conn);
     errAbort("Could not find analysisFeature in db");
     }
 
 struct cohortCorr *cc = getCohortCorrByCohortId(conn, cohort_id);
 if (!cc)
     {
     hFreeConn(&conn);
     errAbort("No cohort correlation table with cohort_id = %d.\n", cohort_id);
     }
 
 struct dyString *dy = dyStringNew(100);
 dyStringPrintf(dy, "select a1.feature_name, a2.feature_name, %s.val from %s "
 	       "join %s as a1 on %s.feature_id1 = a1.id ", 
 	       cc->result_table, cc->result_table, AF_TABLE, cc->result_table);
 dyStringPrintf(dy, "join %s as a2 on %s.feature_id2 = a2.id ", 
 	       AF_TABLE, cc->result_table);
 dyStringPrintf(dy, "where feature_id1 = %d or feature_id2 = %d order by %s.val DESC", 
 	       af->id, af->id, cc->result_table);
 char *query = dyStringCannibalize(&dy);
 
 struct slPair *sp, *spList = NULL;
 struct sqlResult *sr = sqlGetResult(conn, query);
 char **row = NULL;
 while ((row = sqlNextRow(sr)) != NULL)
     {
     char *name = NULL;
     char *name1 = row[0];
     char *name2 = row[1];
     double val = atof(row[2]);
 
     if (sameString(name1, af->feature_name))
 	name = name2;
     else if (sameString(name2, af->feature_name))
 	name = name1;
     else  // doesn't match either, should *never* get here
 	continue;
 
     AllocVar(sp);
     sp->name = cloneString(name);
     sp->val = slDoubleNew(val);
     slAddHead(&spList, sp);
     }
 slReverse(&spList);
 
 int count;
 struct json *js = newJson();
 struct json *corrs = jsonAddContainer(js, "Top Correlated");
 for (sp = spList, count = 0; sp && (count < takeTop); sp = sp->next, count++)
     {
     struct slDouble *sd = sp->val;
     if (sd->val < 0.0)
 	continue;
 
     jsonAddDouble(corrs, sp->name, sd->val);    
     }
 
 slReverse(&spList);
 corrs = jsonAddContainer(js, "Top Anti-Correlated");
 for (sp = spList, count = 0; sp && (count < takeTop); sp = sp->next, count++)
     {
     struct slDouble *sd = sp->val;
     if (sd->val > 0.0)
 	continue;
     jsonAddDouble(corrs, sp->name, sd->val);    
     }
 
 if (js)
     hPrintf("%s", js->print(js));
 
 hFreeConn(&conn);
 }
 
 void expandFeature()
 {
 int takeTop = cartUsualInt(cart, bioIntTakeTop, 5);
-
 int cohort_id = cartUsualInt(cart, bioIntCohortId, -1);
-if (cohort_id == -1)
-    cohort_id = 2;  // hard code for first analysis during testing!
-
 char *feature_name = cartOptionalString(cart, bioIntFeatureName);
+
 if (!feature_name)
     errAbort("%s must be set for mode=getGenesInGeneset\n", bioIntFeatureName);
 
 struct sqlConnection *conn = hAllocConnProfile(localDb, db);
 
 struct analysisFeatures *af = getAnalysisFeaturesByName(conn, feature_name);
 if (!af)
     {
     hFreeConn(&conn);
     errAbort("Could not find analysisFeature in db");
     }
 
 struct analyses *an, *anList = getAnalysesByCohortId(conn, cohort_id);
 if (!anList)
     {
     hFreeConn(&conn);
     errAbort("No analyses with cohort_id = %d.\n", cohort_id);
     }
 
 char tmpQ[512];
 for (an = anList; an; an = an->next)
     {
     safef(tmpQ, sizeof(tmpQ),
 	  "select * from %s where feature_id = %d;",
 	  an->result_table, af->id);
     if (sqlExists(conn, tmpQ))
 	break;
     }
 
 if (!an)
     {
     hFreeConn(&conn);
     errAbort("No analysis feature with id=%d in any analysis table in cohort with id = %d", 
 	     af->id, cohort_id);
     }
 
 // input_tables may have comma-separated list of raw datasets
 //    we don't want to expand into those yet.
 char *input_tables = an->input_tables;
-
 if (!sqlTableExists(conn, input_tables))
     {
     hFreeConn(&conn);
     errAbort("Table does not exist, %s.\n", input_tables);
     }
 
-/***
-select DISTINCT analysisFeatures.id from pathways join pathwayGenes on pathways.id=pathwayGenes.id join geneLookup on geneLookup.id=pathwayGenes.gene_id join kgXref on kgXref.kgID=geneLookup.kgId join analysisFeatures on analysisFeatures.feature_name=kgXref.geneSymbol where pathways.id=123;
-***/
 uglyTime(NULL);
 struct dyString *dy = newDyString(100);
 dyStringPrintf(dy, 
-	       "select DISTINCT %s.id from %s join %s on %s.id=%s.id "
-	       "join %s on %s.id=%s.gene_id join %s on %s.kgID=%s.kgId "
-	       "join %s on %s.feature_name=%s.geneSymbol where %s.name=\"%s\";",
-	       AF_TABLE, PA_TABLE, PG_TABLE, PA_TABLE, PG_TABLE,
-	       GL_TABLE, GL_TABLE, PG_TABLE, KX_TABLE, KX_TABLE, GL_TABLE,
-	       AF_TABLE, AF_TABLE, KX_TABLE, PA_TABLE, af->feature_name);
+	       "select DISTINCT gene_id from %s where id = %d",
+	       GG_TABLE, af->id);
 char *query = dyStringCannibalize(&dy);
 
 struct slInt *si, *siList = sqlQuickNumList(conn, query);
 
 dy = newDyString(100);
 dyStringPrintf(dy, 
-	       "select %s.feature_name from %s join %s on feature_id=%s.id where feature_id in (",
-	       AF_TABLE, input_tables, AF_TABLE, AF_TABLE);
+	       "select feature_name, type, sum(abs(conf)) as s from %s "
+	       "join %s on feature_id=id where feature_id in (",
+	       AF_TABLE, input_tables);
 for (si = siList; si; si = si->next)
     {
     dyStringPrintf(dy, "%d", si->val);
     if (si->next)
 	dyStringPrintf(dy, ",");
     }
 dyStringPrintf(dy, 
-	       ") group by feature_id order by sum(abs(val)) DESC limit %d;", 
+	       ") group by feature_id order by s DESC limit %d;", 
 	       takeTop);
 query = dyStringCannibalize(&dy);
 
-struct slName *sl, *fList = sqlQuickList(conn, query);
-struct slName *sList = NULL;
-// Only dealing with gene/genesets now
-for (sl = fList; sl; sl = sl->next)
-    slNameAddHead(&sList, "gene/geneset");
+struct slName *fList = NULL, *tList = NULL;
 
+struct sqlResult *sr = sqlGetResult(conn, query);
+char **row = NULL;
+while ((row = sqlNextRow(sr)) != NULL)
+    { 
+    char *name = row[0];
+    char *type = row[1];
+    slNameAddHead(&fList, name);
+    slNameAddHead(&tList, type);
+    }
+slReverse(&fList);
+slReverse(&tList);
 char *feature_names = slNameListToString(fList, ',');
-char *sources = slNameListToString(sList, ',');
+char *types = slNameListToString(tList, ',');
 
 struct json *js = newJson();
-sendUniqueMatch(conn, js, cohort_id, feature_names, sources, NULL);  
+sendUniqueMatch(conn, js, cohort_id, feature_names, types, NULL);  
 
 if (js)
     hPrintf("%s", js->print(js));
 
 hFreeConn(&conn);
 }
 
 void dispatchRoutines()
 /* Look at command variables in cart and figure out which
  * page to draw. */
 {
 /* retrieve cart variables, handle various modes */
 char *mode = cartOptionalString(cart, bioIntMode);
 if (!mode)
     errAbort("%s is required.", bioIntMode);
 
 if (sameString(mode, "getAnalyses"))
     getAnalyses();
 else if (sameString(mode, "getCohorts"))
     getCohorts();
 else if (sameString(mode, "getFeatureData"))
     getFeatureData();
 else if (sameString(mode, "getSuggestions"))
     getSuggestions();
 else if (sameString(mode, "getClinicalData"))
     getClinicalData();
 else if (sameString(mode, "getClinicalFeatures"))
     getClinicalFeatures();
 else if (sameString(mode, "getMostCorrelated"))
     getMostCorrelated();
 else if (sameString(mode, "expandFeature"))
     expandFeature();
 else
     errAbort("Incorrect mode = %s", mode);
 
 cartRemovePrefix(cart, bioIntPrefix);
 
 }
 
 void hghDoUsualHttp()
 /* Wrap html page dispatcher with code that writes out
  * HTTP header and write cart back to database. */
 {
 cartWriteCookie(cart, hUserCookie());
 printf("Content-Type:application/x-javascript\r\n\r\n");
 
 /* Dispatch other pages, that actually want to write HTML. */
 cartWarnCatcher(dispatchRoutines, cart, jsonEarlyWarningHandler);
 cartCheckout(&cart);
 }
 
 char *excludeVars[] = {"Submit", "submit", NULL};
 
 int main(int argc, char *argv[])
 /* Process command line. */
 {
 htmlPushEarlyHandlers();
 cgiSpoof(&argc, argv);
 htmlSetStyle(htmlStyleUndecoratedLink);
 
 oldVars = hashNew(12);
 cart = cartForSession(hUserCookie(), excludeVars, oldVars);
 
 hghDoUsualHttp();
 return 0;
 }