5bfb32fdfc639f9bffb5435044cef164bdde0b28
kent
  Sun Dec 12 08:02:13 2021 -0800
Implementing barChartMerge trackDb option for faceted bar charts.

diff --git src/hg/lib/facetedTable.c src/hg/lib/facetedTable.c
index 3f929f5..be59540 100644
--- src/hg/lib/facetedTable.c
+++ src/hg/lib/facetedTable.c
@@ -1,24 +1,25 @@
 /* facetedTable - routines to help produce a sortable table with facet selection fields.
  * This builds on top of things in tablesTables and facetField. */
 
 #include "common.h"
 #include "hash.h"
 #include "hdb.h"
 #include "web.h"
 #include "trashDir.h"
 #include "hCommon.h"
+#include "htmlColor.h"
 #include "hgColors.h"
 #include "fieldedTable.h"
 #include "tablesTables.h"
 #include "facetField.h"
 #include "hgConfig.h"
 #include "facetedTable.h"
 
 static struct facetedTable *facetedTableNew(char *name, char *varPrefix, char *facets)
 /* Return new, mostly empty faceted table */
 {
 struct facetedTable *facTab;
 AllocVar(facTab);
 facTab->name = cloneString(name);
 facTab->varPrefix = cloneString(varPrefix);
 facTab->facets = cloneString(facets);
@@ -115,84 +116,424 @@
 cartRemove(cart, var);
 safef(var, sizeof(var), "%s_facet_fieldVal", facTab->varPrefix);
 cartRemove(cart, var);
 safef(var, sizeof(var), "%s_facet_fieldName", facTab->varPrefix);
 cartRemove(cart, var);
 }
 
 boolean facetedTableUpdateOnClick(struct facetedTable *facTab, struct cart *cart)
 /* If we got called by a click on a facet deal with that and return TRUE, else do
  * nothing and return false */
 {
 char *selOp = facetedTableSelOp(facTab, cart);
 if (selOp)
     {
     char *selFieldName = facetedTableSelField(facTab, cart);
-    char *selFieldVal = facetedTableSelVal(facTab, cart);
-    if (selFieldName && selFieldVal)
+    if (selFieldName != NULL)
 	{
+	char *selFieldVal = facetedTableSelVal(facTab, cart);
 	char selListVar[256];
 	safef(selListVar, sizeof(selListVar), "%s_facet_selList", facTab->varPrefix);
 	char *selectedFacetValues=cartUsualString(cart, selListVar, "");
 	struct facetField *selList = deLinearizeFacetValString(selectedFacetValues);
 	selectedListFacetValUpdate(&selList, selFieldName, selFieldVal, selOp);
 	char *newSelectedFacetValues = linearizeFacetVals(selList);
 	cartSetString(cart, selListVar, newSelectedFacetValues);
 	facetedTableRemoveOpVars(facTab, cart);
 	}
     return TRUE;
     }
 else
     return FALSE;
 }
 
-struct fieldedTable *facetedTableSelect(struct facetedTable *facTab, struct cart *cart)
+struct mergeGroup
+/* Adds up series of counts and values with values weighted by count*/
+    {
+    struct mergeGroup *next;
+    char *key;	        /* Key - just / separated list of component keys */
+    char **aRow;	/* First row we encounter in input table*/
+    struct slRef *rowRefList;  /* List of refs to all rows in merge */
+    };
+
+struct mergeColHelper
+/* Helper structure for merging - one per column whether merged or not */
+    {
+    struct mergeColHelper *next;
+    char *name;	/* Name of column */
+    int ix;	/* Column index in table */
+    boolean isMerged;	/* If true it's a merging column */
+    boolean isFacet;	/* If true it's a faceting column */
+    boolean sameAfterMerger;  /* If true then all values after merge would be same */
+    boolean isCount;	/* Is count field */
+    boolean isVal;	/* Is val field */
+    boolean isKey;	/* Is it the key field */
+    boolean isColor;	/* Is it the color field? */
+    int mergedIx;	/* Column index in merged table */
+    };
+
+static struct mergeColHelper *makeColList(struct facetedTable *facTab, struct fieldedTable *table,
+    struct facetField *selList, struct mergeColHelper **retCountCol, 
+    struct mergeColHelper **retValCol, struct mergeColHelper **retColorCol)
+/* Create a list of all columns of table annotated with some useful info */
+{
+/* Build up hash of all fields with selection info */
+struct hash *ffHash = hashNew(4);
+struct facetField *ff;
+for (ff = selList; ff != NULL; ff = ff->next)
+    hashAdd(ffHash, ff->fieldName, ff);
+
+/* Build up hash of all faceted fields */
+struct hash *facetHash = hashNew(4);
+struct slName *facetNameList = slNameListFromComma(facTab->facets);
+struct slName *el;
+for (el = facetNameList; el != NULL; el = el->next)
+    hashAdd(facetHash, el->name, NULL);
+
+/* Stream through columns of table making up of helpers, one for each field  */
+struct mergeColHelper *list = NULL, *helper;
+int i;
+for (i=0; i<table->fieldCount; ++i)
+    {
+    char *field = table->fields[i];
+    AllocVar(helper);
+    helper->name = field;
+    helper->ix = i;
+    struct facetField *ff = hashFindVal(ffHash, field);
+    if (ff != NULL)
+	helper->isMerged = ff->isMerged;
+    helper->isFacet = (hashLookup(facetHash, field) != NULL);
+    helper->isVal = sameString(field, "val");
+    if (helper->isVal)
+        *retValCol = helper;
+    helper->isCount = sameString(field, "count");
+    if (helper->isCount)
+        *retCountCol = helper;
+    helper->isColor = sameString(field, "color");
+    if (helper->isColor)
+        *retColorCol = helper;
+    helper->isKey = (i == 0);
+    slAddHead(&list, helper);
+    }
+slReverse(&list);
+
+/* Clean up and go home */
+hashFree(&ffHash);
+hashFree(&facetHash);
+slNameFreeList(&facetNameList);
+return list;
+}
+
+static char *calcMergedKey(struct mergeColHelper *colList, struct fieldedRow *fr, 
+    struct dyString *key)
+/* Calculate merging key for this row. Puts result in key, whose contents will be overwritten */
+{
+dyStringClear(key);
+struct mergeColHelper *col;
+char **row = fr->row;
+for (col = colList; col != NULL; col = col->next)
+    {
+    if (col->isFacet && !col->isMerged)
+	{
+	if (key->stringSize != 0)
+	    dyStringAppendC(key, ' ');
+	dyStringAppend(key, row[col->ix]);
+	}
+    }
+return key->string;
+}
+
+long summedCountAt(int countIx, struct mergeGroup *merge)
+/* Return sum of column across merge group */
+{
+long long total = 0;
+struct slRef *ref;
+for (ref = merge->rowRefList; ref != NULL; ref = ref->next)
+    {
+    char **row = ref->val;
+    int count = sqlUnsigned(row[countIx]);
+    total += count;
+    }
+return total;
+}
+
+static int scaleColorPart(double colSum, long count)
+/* Return component scaled for count but lighten it away if
+ * the count is to low. */
+{
+int maxVal = 240;
+int threshold = 100;
+if (count == 0)
+    return maxVal;  // Which makes us nearly white and invisible
+double fullColor = colSum/count;
+int underThreshold = threshold - count;
+if (underThreshold < 0)
+    return round(fullColor);
+
+/* We are under threshold.  Blend us with white (aka maxVal) */
+double ratioUnderThreshold = (double)underThreshold/(double)threshold;
+return (1 - ratioUnderThreshold)*fullColor + ratioUnderThreshold*maxVal;
+}
+
+long long weightedAveValAt(int countIx, int valIx, struct mergeGroup *merge)
+/* Return wieghted average value */
+{
+long totalCount = 0;
+double weightedSum = 0;
+struct slRef *ref;
+for (ref = merge->rowRefList; ref != NULL; ref = ref->next)
+    {
+    char **row = ref->val;
+    int count = sqlUnsigned(row[countIx]);
+    double val = sqlDouble(row[valIx]);
+    totalCount += count;
+    weightedSum += val*count;
+    }
+return weightedSum/totalCount;
+}
+
+long long aveValAt(int valIx, struct mergeGroup *merge)
+/* Return average value */
+{
+int count = 0;
+double weightedSum = 0;
+struct slRef *ref;
+for (ref = merge->rowRefList; ref != NULL; ref = ref->next)
+    {
+    char **row = ref->val;
+    double val = sqlDouble(row[valIx]);
+    count += 1;
+    weightedSum += val;
+    }
+return weightedSum/count;
+}
+
+void weightedAveColorAt(int countIx, int colorIx, struct mergeGroup *merge, 
+    int *retR, int *retG, int *retB)
+/* Return wieghted average color.  Assumes color fields all #RRGGBB hex format  */
+{
+long totalCount = 0;
+double rSum = 0, gSum=0, bSum = 0;
+struct slRef *ref;
+for (ref = merge->rowRefList; ref != NULL; ref = ref->next)
+    {
+    char **row = ref->val;
+    int count = (countIx >= 0 ? sqlUnsigned(row[countIx]) : 1);
+    char *hexColor = row[colorIx];
+    unsigned htmlColor;	// Format will be 8 bits each of RGB
+    if (!htmlColorForCode(hexColor, &htmlColor))
+	internalErr();
+    int r,g,b;
+    htmlColorToRGB(htmlColor, &r, &g, &b);
+    rSum += r*count;
+    gSum += g*count;
+    bSum += b*count;
+    totalCount += count;
+    }
+*retR = scaleColorPart(rSum,totalCount);
+*retG = scaleColorPart(gSum,totalCount);
+*retB = scaleColorPart(bSum,totalCount);
+}
+
+
+
+static struct fieldedTable *groupColumns(struct facetedTable *facTab, struct fieldedTable *tableIn,
+    struct facetField *selList, struct facetField ***retFfArray)
+/* Create a new table that has somewhat fewer fields than the input table based on grouping
+ * together columns based on isMerged values of selList.  Returns a new ffArray to go with new
+ * table. */
+{
+struct mergeColHelper *countCol = NULL, *valCol = NULL, *colorCol = NULL;
+struct mergeColHelper *inColList = makeColList(facTab, tableIn, selList, 
+    &countCol, &valCol, &colorCol);
+
+/* Create new empty table and merge hash full of mergeGroups */
+struct mergeGroup *mergeList = NULL;
+struct hash *mergeHash = hashNew(0);
+
+/* Stream through table calculating key for the row and updating mergeGroup */
+struct fieldedRow *fr;
+struct dyString *keyBuf = dyStringNew(0);
+for (fr = tableIn->rowList; fr != NULL; fr = fr->next)
+    {
+    char *key = calcMergedKey(inColList, fr, keyBuf);
+    struct mergeGroup *merge = hashFindVal(mergeHash, key);
+    if (merge == NULL)
+        {
+	AllocVar(merge);
+	hashAddSaveName(mergeHash, key, merge, &merge->key);
+	merge->aRow = fr->row;
+	slAddHead(&mergeList, merge);
+	}
+    refAdd(&merge->rowRefList, fr->row);
+    }
+slReverse(&mergeList);
+
+/* Make a trip through merged table noting whether a each column merges cleanly */
+struct mergeColHelper *inCol;
+for (inCol = inColList; inCol != NULL; inCol = inCol->next)
+    {
+    struct mergeGroup *merge;
+    int colIx = inCol->ix;
+    boolean mergeOk = TRUE;
+    for (merge = mergeList; merge != NULL && mergeOk; merge = merge->next)
+        {
+	struct slRef *ref = merge->rowRefList;
+	char **row = ref->val;
+	char *first = row[colIx];
+	for (ref = ref->next; ref != NULL; ref = ref->next)
+	    {
+	    row = ref->val;
+	    if (!sameString(first, row[colIx]))
+	        {
+		mergeOk = FALSE;
+		break;
+		}
+	    }
+	}
+    inCol->sameAfterMerger = mergeOk;
+    }
+
+/* Figure out list of fields for new table 
+ *     We'll keep the first field but repurpose it as new index.
+ *     We'll copy over the non-merged facet fields
+ *     We'll copy over other fields that have the same value for each row in the merge
+ *     We'll compute new count and val fields where count is sum and val is weighted average 
+ *     If there's a color field we'll merge it by weighted average as well */
+
+char *newFields[tableIn->fieldCount];  /* Might not use all of these but it's big enough */
+int newFieldCount = 0;
+for (inCol = inColList; inCol != NULL; inCol = inCol->next)
+    {
+    char *name = inCol->name;
+    if (inCol->sameAfterMerger || inCol->isKey || inCol->isCount || inCol->isVal || inCol->isColor)
+        {
+	newFields[newFieldCount] = name;
+	inCol->mergedIx = newFieldCount;
+	newFieldCount += 1;
+	}
+    else
+        inCol->mergedIx = -1;
+    }
+
+struct fieldedTable *tableOut = fieldedTableNew("merged", newFields, newFieldCount);
+char *newRow[newFieldCount];
+struct mergeGroup *merge;
+for (merge = mergeList; merge != NULL; merge = merge->next)
+    {
+    char countBuf[32], valBuf[32], colorBuf[8];
+    for (inCol = inColList; inCol != NULL; inCol = inCol->next)
+        {
+	if (inCol->mergedIx != -1)
+	    {
+	    if (inCol->isKey)
+	        {
+		newRow[inCol->mergedIx] = merge->key;
+		}
+	    else if (inCol->isCount)
+	        {
+		long totalCount = summedCountAt(inCol->ix, merge);
+		safef(countBuf, sizeof(countBuf), "%ld", totalCount);
+		newRow[inCol->mergedIx] = countBuf;
+		}
+	    else if (inCol->isVal)
+	        {
+		double val;
+		if (countCol == NULL)
+		    val = aveValAt(inCol->ix, merge);
+		else
+		    val = weightedAveValAt(countCol->ix, inCol->ix, merge);
+		safef(valBuf, sizeof(valBuf), "%g", val);
+		newRow[inCol->mergedIx] = valBuf;
+		}
+	    else if (inCol->isColor)
+	        {
+		int r,g,b;
+		weightedAveColorAt(countCol->ix, inCol->ix, merge, &r, &g, &b);
+		safef(colorBuf, sizeof(colorBuf), "#%02X%02X%02X", r, g, b);
+		newRow[inCol->mergedIx] = colorBuf;
+		}
+	    else if (inCol->sameAfterMerger)
+	        {
+		newRow[inCol->mergedIx] = merge->aRow[inCol->ix];
+		}
+	    }
+	}
+    fieldedTableAdd(tableOut, newRow, newFieldCount, tableOut->rowCount);
+    }
+
+/* Recompute ffArray */
+struct facetField **oldFfArray = facTab->ffArray;
+struct facetField **newFfArray;
+AllocArray(newFfArray, newFieldCount);
+for (inCol = inColList; inCol != NULL; inCol = inCol->next)
+    {
+    if (inCol->mergedIx != -1)
+        newFfArray[inCol->mergedIx] = oldFfArray[inCol->ix];
+    }
+
+*retFfArray = newFfArray;
+return tableOut;
+}
+
+struct fieldedTable *facetedTableSelect(struct facetedTable *facTab, struct cart *cart,
+    struct facetField ***retFfArray)
 /* Return table containing rows of table that have passed facet selection */
 {
 char *selList = facetedTableSelList(facTab, cart);
-return facetFieldsFromFieldedTable(facTab->table, selList, facTab->ffArray);
+struct fieldedTable *subtable = facetFieldSelectRows(facTab->table, selList, facTab->ffArray);
+struct facetField *ffSelList  = deLinearizeFacetValString(selList);
+if (facTab->mergeFacetsOk)
+    {
+    return groupColumns(facTab, subtable, ffSelList, retFfArray);
+    }
+else
+    {
+    *retFfArray = facTab->ffArray;
+    return subtable;
+    }
 }
 
 struct slInt *facetedTableSelectOffsets(struct facetedTable *facTab, struct cart *cart)
 /* Return a list of row positions that pass faceting */
 {
 char *selList = facetedTableSelList(facTab, cart);
 struct fieldedTable *ft = facTab->table;
 int fieldCount = ft->fieldCount;
 struct slInt *retList = NULL;
 facetFieldsFromSqlTableInit(ft->fields, fieldCount,
     selList, facTab->ffArray);
 struct fieldedRow *fr;
 int ix = 0;
 for (fr = facTab->table->rowList; fr != NULL; fr = fr->next)
     {
     if (perRowFacetFields(fieldCount, fr->row, "", facTab->ffArray))
 	{
 	struct slInt *el = slIntNew(ix);
 	slAddHead(&retList, el);
 	}
     ++ix;
     }
 slReverse(&retList);
 return retList;
 }
 
 
 void facetedTableWriteHtml(struct facetedTable *facTab, struct cart *cart,
-    struct fieldedTable *selected, char *displayList, 
+    struct fieldedTable *selected, struct facetField **selectedFf, char *displayList, 
     char *returnUrl, int maxLenField, 
     struct hash *tagOutputWrappers, void *wrapperContext, int facetUsualSize)
 /* Write out the main HTML associated with facet selection and table. */
 {
 facetedTableWebInit();
 struct hash *emptyHash = hashNew(0);
 webFilteredFieldedTable(cart, selected, 
     displayList, returnUrl, facTab->varPrefix, 
     maxLenField, tagOutputWrappers, wrapperContext,
     FALSE, NULL,
     selected->rowCount, facetUsualSize,
     NULL, emptyHash,
-    facTab->ffArray, facTab->facets,
-    NULL);
+    selectedFf, facTab->facets,
+    NULL, facTab->mergeFacetsOk);
 hashFree(&emptyHash);
 }