5bfb32fdfc639f9bffb5435044cef164bdde0b28 kent Sun Dec 12 08:02:13 2021 -0800 Implementing barChartMerge trackDb option for faceted bar charts. diff --git src/hg/lib/facetedTable.c src/hg/lib/facetedTable.c index 3f929f5..be59540 100644 --- src/hg/lib/facetedTable.c +++ src/hg/lib/facetedTable.c @@ -1,198 +1,539 @@ /* facetedTable - routines to help produce a sortable table with facet selection fields. * This builds on top of things in tablesTables and facetField. */ #include "common.h" #include "hash.h" #include "hdb.h" #include "web.h" #include "trashDir.h" #include "hCommon.h" +#include "htmlColor.h" #include "hgColors.h" #include "fieldedTable.h" #include "tablesTables.h" #include "facetField.h" #include "hgConfig.h" #include "facetedTable.h" static struct facetedTable *facetedTableNew(char *name, char *varPrefix, char *facets) /* Return new, mostly empty faceted table */ { struct facetedTable *facTab; AllocVar(facTab); facTab->name = cloneString(name); facTab->varPrefix = cloneString(varPrefix); facTab->facets = cloneString(facets); return facTab; } struct facetedTable *facetedTableFromTable(struct fieldedTable *table, char *varPrefix, char *facets) /* Construct a facetedTable around a fieldedTable */ { struct facetedTable *facTab = facetedTableNew(table->name, varPrefix, facets); facTab->table = table; AllocArray(facTab->ffArray, table->fieldCount); return facTab; } void facetedTableFree(struct facetedTable **pFt) /* Free up resources associated with faceted table */ { struct facetedTable *facTab = *pFt; if (facTab != NULL) { freeMem(facTab->name); freeMem(facTab->varPrefix); freeMem(facTab->facets); freeMem(facTab->ffArray); freez(pFt); } } static void facetedTableWebInit() /* Print out scripts and css that we need. We should be in a page body or title. */ { static boolean initted = FALSE; if (initted) return; initted = TRUE; webIncludeResourceFile("facets.css"); printf("\t\t"); printf("\t\t\n" "\n" "\t\t\n" "\t\t\n" "\n" "\t\t\n" "\t\t\n" ); } static char *facetedTableSelList(struct facetedTable *facTab, struct cart *cart) /* Look up list of selected items in facets from cart */ { char var[256]; safef(var, sizeof(var), "%s_facet_selList", facTab->varPrefix); return cartOptionalString(cart, var); } static char *facetedTableSelOp(struct facetedTable *facTab, struct cart *cart) /* Look up selOp in cart */ { char var[256]; safef(var, sizeof(var), "%s_facet_op", facTab->varPrefix); return cartOptionalString(cart, var); } static char *facetedTableSelField(struct facetedTable *facTab, struct cart *cart) /* Look up sel field in cart */ { char var[256]; safef(var, sizeof(var), "%s_facet_fieldName", facTab->varPrefix); return cartOptionalString(cart, var); } static char *facetedTableSelVal(struct facetedTable *facTab, struct cart *cart) /* Look up sel val in cart */ { char var[256]; safef(var, sizeof(var), "%s_facet_fieldVal", facTab->varPrefix); return cartOptionalString(cart, var); } static void facetedTableRemoveOpVars(struct facetedTable *facTab, struct cart *cart) /* Remove sel op/field/name vars from cart */ { char var[256]; safef(var, sizeof(var), "%s_facet_op", facTab->varPrefix); cartRemove(cart, var); safef(var, sizeof(var), "%s_facet_fieldVal", facTab->varPrefix); cartRemove(cart, var); safef(var, sizeof(var), "%s_facet_fieldName", facTab->varPrefix); cartRemove(cart, var); } boolean facetedTableUpdateOnClick(struct facetedTable *facTab, struct cart *cart) /* If we got called by a click on a facet deal with that and return TRUE, else do * nothing and return false */ { char *selOp = facetedTableSelOp(facTab, cart); if (selOp) { char *selFieldName = facetedTableSelField(facTab, cart); - char *selFieldVal = facetedTableSelVal(facTab, cart); - if (selFieldName && selFieldVal) + if (selFieldName != NULL) { + char *selFieldVal = facetedTableSelVal(facTab, cart); char selListVar[256]; safef(selListVar, sizeof(selListVar), "%s_facet_selList", facTab->varPrefix); char *selectedFacetValues=cartUsualString(cart, selListVar, ""); struct facetField *selList = deLinearizeFacetValString(selectedFacetValues); selectedListFacetValUpdate(&selList, selFieldName, selFieldVal, selOp); char *newSelectedFacetValues = linearizeFacetVals(selList); cartSetString(cart, selListVar, newSelectedFacetValues); facetedTableRemoveOpVars(facTab, cart); } return TRUE; } else return FALSE; } -struct fieldedTable *facetedTableSelect(struct facetedTable *facTab, struct cart *cart) +struct mergeGroup +/* Adds up series of counts and values with values weighted by count*/ + { + struct mergeGroup *next; + char *key; /* Key - just / separated list of component keys */ + char **aRow; /* First row we encounter in input table*/ + struct slRef *rowRefList; /* List of refs to all rows in merge */ + }; + +struct mergeColHelper +/* Helper structure for merging - one per column whether merged or not */ + { + struct mergeColHelper *next; + char *name; /* Name of column */ + int ix; /* Column index in table */ + boolean isMerged; /* If true it's a merging column */ + boolean isFacet; /* If true it's a faceting column */ + boolean sameAfterMerger; /* If true then all values after merge would be same */ + boolean isCount; /* Is count field */ + boolean isVal; /* Is val field */ + boolean isKey; /* Is it the key field */ + boolean isColor; /* Is it the color field? */ + int mergedIx; /* Column index in merged table */ + }; + +static struct mergeColHelper *makeColList(struct facetedTable *facTab, struct fieldedTable *table, + struct facetField *selList, struct mergeColHelper **retCountCol, + struct mergeColHelper **retValCol, struct mergeColHelper **retColorCol) +/* Create a list of all columns of table annotated with some useful info */ +{ +/* Build up hash of all fields with selection info */ +struct hash *ffHash = hashNew(4); +struct facetField *ff; +for (ff = selList; ff != NULL; ff = ff->next) + hashAdd(ffHash, ff->fieldName, ff); + +/* Build up hash of all faceted fields */ +struct hash *facetHash = hashNew(4); +struct slName *facetNameList = slNameListFromComma(facTab->facets); +struct slName *el; +for (el = facetNameList; el != NULL; el = el->next) + hashAdd(facetHash, el->name, NULL); + +/* Stream through columns of table making up of helpers, one for each field */ +struct mergeColHelper *list = NULL, *helper; +int i; +for (i=0; ifieldCount; ++i) + { + char *field = table->fields[i]; + AllocVar(helper); + helper->name = field; + helper->ix = i; + struct facetField *ff = hashFindVal(ffHash, field); + if (ff != NULL) + helper->isMerged = ff->isMerged; + helper->isFacet = (hashLookup(facetHash, field) != NULL); + helper->isVal = sameString(field, "val"); + if (helper->isVal) + *retValCol = helper; + helper->isCount = sameString(field, "count"); + if (helper->isCount) + *retCountCol = helper; + helper->isColor = sameString(field, "color"); + if (helper->isColor) + *retColorCol = helper; + helper->isKey = (i == 0); + slAddHead(&list, helper); + } +slReverse(&list); + +/* Clean up and go home */ +hashFree(&ffHash); +hashFree(&facetHash); +slNameFreeList(&facetNameList); +return list; +} + +static char *calcMergedKey(struct mergeColHelper *colList, struct fieldedRow *fr, + struct dyString *key) +/* Calculate merging key for this row. Puts result in key, whose contents will be overwritten */ +{ +dyStringClear(key); +struct mergeColHelper *col; +char **row = fr->row; +for (col = colList; col != NULL; col = col->next) + { + if (col->isFacet && !col->isMerged) + { + if (key->stringSize != 0) + dyStringAppendC(key, ' '); + dyStringAppend(key, row[col->ix]); + } + } +return key->string; +} + +long summedCountAt(int countIx, struct mergeGroup *merge) +/* Return sum of column across merge group */ +{ +long long total = 0; +struct slRef *ref; +for (ref = merge->rowRefList; ref != NULL; ref = ref->next) + { + char **row = ref->val; + int count = sqlUnsigned(row[countIx]); + total += count; + } +return total; +} + +static int scaleColorPart(double colSum, long count) +/* Return component scaled for count but lighten it away if + * the count is to low. */ +{ +int maxVal = 240; +int threshold = 100; +if (count == 0) + return maxVal; // Which makes us nearly white and invisible +double fullColor = colSum/count; +int underThreshold = threshold - count; +if (underThreshold < 0) + return round(fullColor); + +/* We are under threshold. Blend us with white (aka maxVal) */ +double ratioUnderThreshold = (double)underThreshold/(double)threshold; +return (1 - ratioUnderThreshold)*fullColor + ratioUnderThreshold*maxVal; +} + +long long weightedAveValAt(int countIx, int valIx, struct mergeGroup *merge) +/* Return wieghted average value */ +{ +long totalCount = 0; +double weightedSum = 0; +struct slRef *ref; +for (ref = merge->rowRefList; ref != NULL; ref = ref->next) + { + char **row = ref->val; + int count = sqlUnsigned(row[countIx]); + double val = sqlDouble(row[valIx]); + totalCount += count; + weightedSum += val*count; + } +return weightedSum/totalCount; +} + +long long aveValAt(int valIx, struct mergeGroup *merge) +/* Return average value */ +{ +int count = 0; +double weightedSum = 0; +struct slRef *ref; +for (ref = merge->rowRefList; ref != NULL; ref = ref->next) + { + char **row = ref->val; + double val = sqlDouble(row[valIx]); + count += 1; + weightedSum += val; + } +return weightedSum/count; +} + +void weightedAveColorAt(int countIx, int colorIx, struct mergeGroup *merge, + int *retR, int *retG, int *retB) +/* Return wieghted average color. Assumes color fields all #RRGGBB hex format */ +{ +long totalCount = 0; +double rSum = 0, gSum=0, bSum = 0; +struct slRef *ref; +for (ref = merge->rowRefList; ref != NULL; ref = ref->next) + { + char **row = ref->val; + int count = (countIx >= 0 ? sqlUnsigned(row[countIx]) : 1); + char *hexColor = row[colorIx]; + unsigned htmlColor; // Format will be 8 bits each of RGB + if (!htmlColorForCode(hexColor, &htmlColor)) + internalErr(); + int r,g,b; + htmlColorToRGB(htmlColor, &r, &g, &b); + rSum += r*count; + gSum += g*count; + bSum += b*count; + totalCount += count; + } +*retR = scaleColorPart(rSum,totalCount); +*retG = scaleColorPart(gSum,totalCount); +*retB = scaleColorPart(bSum,totalCount); +} + + + +static struct fieldedTable *groupColumns(struct facetedTable *facTab, struct fieldedTable *tableIn, + struct facetField *selList, struct facetField ***retFfArray) +/* Create a new table that has somewhat fewer fields than the input table based on grouping + * together columns based on isMerged values of selList. Returns a new ffArray to go with new + * table. */ +{ +struct mergeColHelper *countCol = NULL, *valCol = NULL, *colorCol = NULL; +struct mergeColHelper *inColList = makeColList(facTab, tableIn, selList, + &countCol, &valCol, &colorCol); + +/* Create new empty table and merge hash full of mergeGroups */ +struct mergeGroup *mergeList = NULL; +struct hash *mergeHash = hashNew(0); + +/* Stream through table calculating key for the row and updating mergeGroup */ +struct fieldedRow *fr; +struct dyString *keyBuf = dyStringNew(0); +for (fr = tableIn->rowList; fr != NULL; fr = fr->next) + { + char *key = calcMergedKey(inColList, fr, keyBuf); + struct mergeGroup *merge = hashFindVal(mergeHash, key); + if (merge == NULL) + { + AllocVar(merge); + hashAddSaveName(mergeHash, key, merge, &merge->key); + merge->aRow = fr->row; + slAddHead(&mergeList, merge); + } + refAdd(&merge->rowRefList, fr->row); + } +slReverse(&mergeList); + +/* Make a trip through merged table noting whether a each column merges cleanly */ +struct mergeColHelper *inCol; +for (inCol = inColList; inCol != NULL; inCol = inCol->next) + { + struct mergeGroup *merge; + int colIx = inCol->ix; + boolean mergeOk = TRUE; + for (merge = mergeList; merge != NULL && mergeOk; merge = merge->next) + { + struct slRef *ref = merge->rowRefList; + char **row = ref->val; + char *first = row[colIx]; + for (ref = ref->next; ref != NULL; ref = ref->next) + { + row = ref->val; + if (!sameString(first, row[colIx])) + { + mergeOk = FALSE; + break; + } + } + } + inCol->sameAfterMerger = mergeOk; + } + +/* Figure out list of fields for new table + * We'll keep the first field but repurpose it as new index. + * We'll copy over the non-merged facet fields + * We'll copy over other fields that have the same value for each row in the merge + * We'll compute new count and val fields where count is sum and val is weighted average + * If there's a color field we'll merge it by weighted average as well */ + +char *newFields[tableIn->fieldCount]; /* Might not use all of these but it's big enough */ +int newFieldCount = 0; +for (inCol = inColList; inCol != NULL; inCol = inCol->next) + { + char *name = inCol->name; + if (inCol->sameAfterMerger || inCol->isKey || inCol->isCount || inCol->isVal || inCol->isColor) + { + newFields[newFieldCount] = name; + inCol->mergedIx = newFieldCount; + newFieldCount += 1; + } + else + inCol->mergedIx = -1; + } + +struct fieldedTable *tableOut = fieldedTableNew("merged", newFields, newFieldCount); +char *newRow[newFieldCount]; +struct mergeGroup *merge; +for (merge = mergeList; merge != NULL; merge = merge->next) + { + char countBuf[32], valBuf[32], colorBuf[8]; + for (inCol = inColList; inCol != NULL; inCol = inCol->next) + { + if (inCol->mergedIx != -1) + { + if (inCol->isKey) + { + newRow[inCol->mergedIx] = merge->key; + } + else if (inCol->isCount) + { + long totalCount = summedCountAt(inCol->ix, merge); + safef(countBuf, sizeof(countBuf), "%ld", totalCount); + newRow[inCol->mergedIx] = countBuf; + } + else if (inCol->isVal) + { + double val; + if (countCol == NULL) + val = aveValAt(inCol->ix, merge); + else + val = weightedAveValAt(countCol->ix, inCol->ix, merge); + safef(valBuf, sizeof(valBuf), "%g", val); + newRow[inCol->mergedIx] = valBuf; + } + else if (inCol->isColor) + { + int r,g,b; + weightedAveColorAt(countCol->ix, inCol->ix, merge, &r, &g, &b); + safef(colorBuf, sizeof(colorBuf), "#%02X%02X%02X", r, g, b); + newRow[inCol->mergedIx] = colorBuf; + } + else if (inCol->sameAfterMerger) + { + newRow[inCol->mergedIx] = merge->aRow[inCol->ix]; + } + } + } + fieldedTableAdd(tableOut, newRow, newFieldCount, tableOut->rowCount); + } + +/* Recompute ffArray */ +struct facetField **oldFfArray = facTab->ffArray; +struct facetField **newFfArray; +AllocArray(newFfArray, newFieldCount); +for (inCol = inColList; inCol != NULL; inCol = inCol->next) + { + if (inCol->mergedIx != -1) + newFfArray[inCol->mergedIx] = oldFfArray[inCol->ix]; + } + +*retFfArray = newFfArray; +return tableOut; +} + +struct fieldedTable *facetedTableSelect(struct facetedTable *facTab, struct cart *cart, + struct facetField ***retFfArray) /* Return table containing rows of table that have passed facet selection */ { char *selList = facetedTableSelList(facTab, cart); -return facetFieldsFromFieldedTable(facTab->table, selList, facTab->ffArray); +struct fieldedTable *subtable = facetFieldSelectRows(facTab->table, selList, facTab->ffArray); +struct facetField *ffSelList = deLinearizeFacetValString(selList); +if (facTab->mergeFacetsOk) + { + return groupColumns(facTab, subtable, ffSelList, retFfArray); + } +else + { + *retFfArray = facTab->ffArray; + return subtable; + } } struct slInt *facetedTableSelectOffsets(struct facetedTable *facTab, struct cart *cart) /* Return a list of row positions that pass faceting */ { char *selList = facetedTableSelList(facTab, cart); struct fieldedTable *ft = facTab->table; int fieldCount = ft->fieldCount; struct slInt *retList = NULL; facetFieldsFromSqlTableInit(ft->fields, fieldCount, selList, facTab->ffArray); struct fieldedRow *fr; int ix = 0; for (fr = facTab->table->rowList; fr != NULL; fr = fr->next) { if (perRowFacetFields(fieldCount, fr->row, "", facTab->ffArray)) { struct slInt *el = slIntNew(ix); slAddHead(&retList, el); } ++ix; } slReverse(&retList); return retList; } void facetedTableWriteHtml(struct facetedTable *facTab, struct cart *cart, - struct fieldedTable *selected, char *displayList, + struct fieldedTable *selected, struct facetField **selectedFf, char *displayList, char *returnUrl, int maxLenField, struct hash *tagOutputWrappers, void *wrapperContext, int facetUsualSize) /* Write out the main HTML associated with facet selection and table. */ { facetedTableWebInit(); struct hash *emptyHash = hashNew(0); webFilteredFieldedTable(cart, selected, displayList, returnUrl, facTab->varPrefix, maxLenField, tagOutputWrappers, wrapperContext, FALSE, NULL, selected->rowCount, facetUsualSize, NULL, emptyHash, - facTab->ffArray, facTab->facets, - NULL); + selectedFf, facTab->facets, + NULL, facTab->mergeFacetsOk); hashFree(&emptyHash); }