d3f9b9ecf2a6d30bd8f97566f72e5650982c308d kent Mon Dec 13 15:22:03 2021 -0800 Adding hgTracks display of merged bar charts. diff --git src/hg/lib/facetedTable.c src/hg/lib/facetedTable.c index be59540..4bedfba 100644 --- src/hg/lib/facetedTable.c +++ src/hg/lib/facetedTable.c @@ -1,539 +1,654 @@ /* facetedTable - routines to help produce a sortable table with facet selection fields. * This builds on top of things in tablesTables and facetField. */ #include "common.h" #include "hash.h" #include "hdb.h" #include "web.h" #include "trashDir.h" #include "hCommon.h" #include "htmlColor.h" #include "hgColors.h" #include "fieldedTable.h" #include "tablesTables.h" #include "facetField.h" #include "hgConfig.h" #include "facetedTable.h" static struct facetedTable *facetedTableNew(char *name, char *varPrefix, char *facets) /* Return new, mostly empty faceted table */ { struct facetedTable *facTab; AllocVar(facTab); facTab->name = cloneString(name); facTab->varPrefix = cloneString(varPrefix); facTab->facets = cloneString(facets); return facTab; } struct facetedTable *facetedTableFromTable(struct fieldedTable *table, char *varPrefix, char *facets) /* Construct a facetedTable around a fieldedTable */ { struct facetedTable *facTab = facetedTableNew(table->name, varPrefix, facets); facTab->table = table; AllocArray(facTab->ffArray, table->fieldCount); return facTab; } void facetedTableFree(struct facetedTable **pFt) /* Free up resources associated with faceted table */ { struct facetedTable *facTab = *pFt; if (facTab != NULL) { freeMem(facTab->name); freeMem(facTab->varPrefix); freeMem(facTab->facets); freeMem(facTab->ffArray); freez(pFt); } } static void facetedTableWebInit() /* Print out scripts and css that we need. We should be in a page body or title. */ { static boolean initted = FALSE; if (initted) return; initted = TRUE; webIncludeResourceFile("facets.css"); printf("\t\t"); printf("\t\t\n" "\n" "\t\t\n" "\t\t\n" "\n" "\t\t\n" "\t\t\n" ); } static char *facetedTableSelList(struct facetedTable *facTab, struct cart *cart) /* Look up list of selected items in facets from cart */ { char var[256]; safef(var, sizeof(var), "%s_facet_selList", facTab->varPrefix); return cartOptionalString(cart, var); } static char *facetedTableSelOp(struct facetedTable *facTab, struct cart *cart) /* Look up selOp in cart */ { char var[256]; safef(var, sizeof(var), "%s_facet_op", facTab->varPrefix); return cartOptionalString(cart, var); } static char *facetedTableSelField(struct facetedTable *facTab, struct cart *cart) /* Look up sel field in cart */ { char var[256]; safef(var, sizeof(var), "%s_facet_fieldName", facTab->varPrefix); return cartOptionalString(cart, var); } static char *facetedTableSelVal(struct facetedTable *facTab, struct cart *cart) /* Look up sel val in cart */ { char var[256]; safef(var, sizeof(var), "%s_facet_fieldVal", facTab->varPrefix); return cartOptionalString(cart, var); } static void facetedTableRemoveOpVars(struct facetedTable *facTab, struct cart *cart) /* Remove sel op/field/name vars from cart */ { char var[256]; safef(var, sizeof(var), "%s_facet_op", facTab->varPrefix); cartRemove(cart, var); safef(var, sizeof(var), "%s_facet_fieldVal", facTab->varPrefix); cartRemove(cart, var); safef(var, sizeof(var), "%s_facet_fieldName", facTab->varPrefix); cartRemove(cart, var); } boolean facetedTableUpdateOnClick(struct facetedTable *facTab, struct cart *cart) /* If we got called by a click on a facet deal with that and return TRUE, else do * nothing and return false */ { char *selOp = facetedTableSelOp(facTab, cart); if (selOp) { char *selFieldName = facetedTableSelField(facTab, cart); if (selFieldName != NULL) { char *selFieldVal = facetedTableSelVal(facTab, cart); char selListVar[256]; safef(selListVar, sizeof(selListVar), "%s_facet_selList", facTab->varPrefix); char *selectedFacetValues=cartUsualString(cart, selListVar, ""); struct facetField *selList = deLinearizeFacetValString(selectedFacetValues); selectedListFacetValUpdate(&selList, selFieldName, selFieldVal, selOp); char *newSelectedFacetValues = linearizeFacetVals(selList); cartSetString(cart, selListVar, newSelectedFacetValues); facetedTableRemoveOpVars(facTab, cart); } return TRUE; } else return FALSE; } struct mergeGroup /* Adds up series of counts and values with values weighted by count*/ { struct mergeGroup *next; char *key; /* Key - just / separated list of component keys */ char **aRow; /* First row we encounter in input table*/ - struct slRef *rowRefList; /* List of refs to all rows in merge */ + struct slRef *rowRefList; /* List of refs to all rows (struct fieldedRow *) in merge */ }; struct mergeColHelper /* Helper structure for merging - one per column whether merged or not */ { struct mergeColHelper *next; char *name; /* Name of column */ int ix; /* Column index in table */ boolean isMerged; /* If true it's a merging column */ boolean isFacet; /* If true it's a faceting column */ boolean sameAfterMerger; /* If true then all values after merge would be same */ boolean isCount; /* Is count field */ boolean isVal; /* Is val field */ boolean isKey; /* Is it the key field */ boolean isColor; /* Is it the color field? */ int mergedIx; /* Column index in merged table */ }; static struct mergeColHelper *makeColList(struct facetedTable *facTab, struct fieldedTable *table, struct facetField *selList, struct mergeColHelper **retCountCol, struct mergeColHelper **retValCol, struct mergeColHelper **retColorCol) /* Create a list of all columns of table annotated with some useful info */ { /* Build up hash of all fields with selection info */ struct hash *ffHash = hashNew(4); struct facetField *ff; for (ff = selList; ff != NULL; ff = ff->next) hashAdd(ffHash, ff->fieldName, ff); /* Build up hash of all faceted fields */ struct hash *facetHash = hashNew(4); struct slName *facetNameList = slNameListFromComma(facTab->facets); struct slName *el; for (el = facetNameList; el != NULL; el = el->next) hashAdd(facetHash, el->name, NULL); /* Stream through columns of table making up of helpers, one for each field */ struct mergeColHelper *list = NULL, *helper; int i; for (i=0; ifieldCount; ++i) { char *field = table->fields[i]; AllocVar(helper); helper->name = field; helper->ix = i; struct facetField *ff = hashFindVal(ffHash, field); if (ff != NULL) helper->isMerged = ff->isMerged; helper->isFacet = (hashLookup(facetHash, field) != NULL); helper->isVal = sameString(field, "val"); if (helper->isVal) *retValCol = helper; helper->isCount = sameString(field, "count"); if (helper->isCount) *retCountCol = helper; helper->isColor = sameString(field, "color"); if (helper->isColor) *retColorCol = helper; helper->isKey = (i == 0); slAddHead(&list, helper); } slReverse(&list); /* Clean up and go home */ hashFree(&ffHash); hashFree(&facetHash); slNameFreeList(&facetNameList); return list; } -static char *calcMergedKey(struct mergeColHelper *colList, struct fieldedRow *fr, +static char *calcMergedKey(struct mergeColHelper *colList, char **row, struct dyString *key) /* Calculate merging key for this row. Puts result in key, whose contents will be overwritten */ { dyStringClear(key); struct mergeColHelper *col; -char **row = fr->row; for (col = colList; col != NULL; col = col->next) { if (col->isFacet && !col->isMerged) { if (key->stringSize != 0) dyStringAppendC(key, ' '); dyStringAppend(key, row[col->ix]); } } return key->string; } long summedCountAt(int countIx, struct mergeGroup *merge) /* Return sum of column across merge group */ { long long total = 0; struct slRef *ref; for (ref = merge->rowRefList; ref != NULL; ref = ref->next) { - char **row = ref->val; + struct fieldedRow *fr = ref->val; + char **row = fr->row; int count = sqlUnsigned(row[countIx]); total += count; } return total; } static int scaleColorPart(double colSum, long count) /* Return component scaled for count but lighten it away if * the count is to low. */ { int maxVal = 240; int threshold = 100; if (count == 0) return maxVal; // Which makes us nearly white and invisible double fullColor = colSum/count; int underThreshold = threshold - count; if (underThreshold < 0) return round(fullColor); /* We are under threshold. Blend us with white (aka maxVal) */ double ratioUnderThreshold = (double)underThreshold/(double)threshold; return (1 - ratioUnderThreshold)*fullColor + ratioUnderThreshold*maxVal; } long long weightedAveValAt(int countIx, int valIx, struct mergeGroup *merge) /* Return wieghted average value */ { long totalCount = 0; double weightedSum = 0; struct slRef *ref; for (ref = merge->rowRefList; ref != NULL; ref = ref->next) { - char **row = ref->val; + struct fieldedRow *fr = ref->val; + char **row = fr->row; int count = sqlUnsigned(row[countIx]); double val = sqlDouble(row[valIx]); totalCount += count; weightedSum += val*count; } return weightedSum/totalCount; } long long aveValAt(int valIx, struct mergeGroup *merge) /* Return average value */ { int count = 0; double weightedSum = 0; struct slRef *ref; for (ref = merge->rowRefList; ref != NULL; ref = ref->next) { - char **row = ref->val; + struct fieldedRow *fr = ref->val; + char **row = fr->row; double val = sqlDouble(row[valIx]); count += 1; weightedSum += val; } return weightedSum/count; } -void weightedAveColorAt(int countIx, int colorIx, struct mergeGroup *merge, +void weightedAveColorAt(struct mergeColHelper *countCol, int colorIx, struct mergeGroup *merge, int *retR, int *retG, int *retB) /* Return wieghted average color. Assumes color fields all #RRGGBB hex format */ { long totalCount = 0; double rSum = 0, gSum=0, bSum = 0; struct slRef *ref; for (ref = merge->rowRefList; ref != NULL; ref = ref->next) { - char **row = ref->val; - int count = (countIx >= 0 ? sqlUnsigned(row[countIx]) : 1); + struct fieldedRow *fr = ref->val; + char **row = fr->row; + int count = (countCol != NULL ? sqlUnsigned(row[countCol->ix]) : 1); char *hexColor = row[colorIx]; unsigned htmlColor; // Format will be 8 bits each of RGB if (!htmlColorForCode(hexColor, &htmlColor)) internalErr(); int r,g,b; htmlColorToRGB(htmlColor, &r, &g, &b); rSum += r*count; gSum += g*count; bSum += b*count; totalCount += count; } *retR = scaleColorPart(rSum,totalCount); *retG = scaleColorPart(gSum,totalCount); *retB = scaleColorPart(bSum,totalCount); } - -static struct fieldedTable *groupColumns(struct facetedTable *facTab, struct fieldedTable *tableIn, - struct facetField *selList, struct facetField ***retFfArray) -/* Create a new table that has somewhat fewer fields than the input table based on grouping - * together columns based on isMerged values of selList. Returns a new ffArray to go with new - * table. */ +static struct mergeGroup *findMergeGroups(struct fieldedTable *tableIn, + struct mergeColHelper *colList) +/* Return list of mergeGroups corresponding to colList */ { -struct mergeColHelper *countCol = NULL, *valCol = NULL, *colorCol = NULL; -struct mergeColHelper *inColList = makeColList(facTab, tableIn, selList, - &countCol, &valCol, &colorCol); - -/* Create new empty table and merge hash full of mergeGroups */ struct mergeGroup *mergeList = NULL; struct hash *mergeHash = hashNew(0); - -/* Stream through table calculating key for the row and updating mergeGroup */ -struct fieldedRow *fr; struct dyString *keyBuf = dyStringNew(0); +struct fieldedRow *fr; for (fr = tableIn->rowList; fr != NULL; fr = fr->next) { - char *key = calcMergedKey(inColList, fr, keyBuf); + char *key = calcMergedKey(colList, fr->row, keyBuf); struct mergeGroup *merge = hashFindVal(mergeHash, key); if (merge == NULL) { AllocVar(merge); - hashAddSaveName(mergeHash, key, merge, &merge->key); + merge->key = cloneString(key); merge->aRow = fr->row; slAddHead(&mergeList, merge); + hashAdd(mergeHash, key, merge); } - refAdd(&merge->rowRefList, fr->row); + refAdd(&merge->rowRefList, fr); } slReverse(&mergeList); +/* Clean up and go home. */ +dyStringFree(&keyBuf); +hashFree(&mergeHash); +return mergeList; +} + + +static char **annotateColListAndMakeNewFields(struct mergeColHelper *colList, + struct mergeGroup *mergeList, int *retCount) +{ /* Make a trip through merged table noting whether a each column merges cleanly */ -struct mergeColHelper *inCol; -for (inCol = inColList; inCol != NULL; inCol = inCol->next) +struct mergeColHelper *col; +for (col = colList; col != NULL; col = col->next) { struct mergeGroup *merge; - int colIx = inCol->ix; + int colIx = col->ix; boolean mergeOk = TRUE; for (merge = mergeList; merge != NULL && mergeOk; merge = merge->next) { struct slRef *ref = merge->rowRefList; - char **row = ref->val; + struct fieldedRow *fr = ref->val; + char **row = fr->row; char *first = row[colIx]; for (ref = ref->next; ref != NULL; ref = ref->next) { - row = ref->val; + fr = ref->val; + row = fr->row; if (!sameString(first, row[colIx])) { mergeOk = FALSE; break; } } } - inCol->sameAfterMerger = mergeOk; + col->sameAfterMerger = mergeOk; } /* Figure out list of fields for new table * We'll keep the first field but repurpose it as new index. * We'll copy over the non-merged facet fields * We'll copy over other fields that have the same value for each row in the merge * We'll compute new count and val fields where count is sum and val is weighted average * If there's a color field we'll merge it by weighted average as well */ -char *newFields[tableIn->fieldCount]; /* Might not use all of these but it's big enough */ +char **newFields; +AllocArray(newFields, slCount(colList)); // Big enough int newFieldCount = 0; -for (inCol = inColList; inCol != NULL; inCol = inCol->next) +for (col = colList; col != NULL; col = col->next) { - char *name = inCol->name; - if (inCol->sameAfterMerger || inCol->isKey || inCol->isCount || inCol->isVal || inCol->isColor) + char *name = col->name; + if (col->sameAfterMerger || col->isKey || col->isCount || col->isVal || col->isColor) { newFields[newFieldCount] = name; - inCol->mergedIx = newFieldCount; + col->mergedIx = newFieldCount; newFieldCount += 1; } else - inCol->mergedIx = -1; + col->mergedIx = -1; + } +*retCount = newFieldCount; +return newFields; } +static struct facetField **recomputeFfArray(struct facetField **oldFfArray, + struct mergeColHelper *colList, int newFieldCount) +/* Return freshly computed ffArray with some of bits of oldArray skipped */ +{ +/* Recompute ffArray */ +struct facetField **newFfArray; +AllocArray(newFfArray, newFieldCount); +struct mergeColHelper *col; +for (col = colList; col != NULL; col = col->next) + { + if (col->mergedIx != -1) + newFfArray[col->mergedIx] = oldFfArray[col->ix]; + } +return newFfArray; +} + +static struct fieldedTable *groupColumns(struct facetedTable *facTab, struct fieldedTable *tableIn, + struct facetField *selList, struct facetField ***retFfArray) +/* Create a new table that has somewhat fewer fields than the input table based on grouping + * together columns based on isMerged values of selList. Returns a new ffArray to go with new + * table. */ +{ +struct mergeColHelper *countCol = NULL, *valCol = NULL, *colorCol = NULL; +struct mergeColHelper *colList = makeColList(facTab, tableIn, selList, + &countCol, &valCol, &colorCol); +struct mergeGroup *mergeList = findMergeGroups(tableIn, colList); +int newFieldCount; +char **newFields = annotateColListAndMakeNewFields(colList, mergeList, &newFieldCount); + struct fieldedTable *tableOut = fieldedTableNew("merged", newFields, newFieldCount); char *newRow[newFieldCount]; struct mergeGroup *merge; for (merge = mergeList; merge != NULL; merge = merge->next) { char countBuf[32], valBuf[32], colorBuf[8]; - for (inCol = inColList; inCol != NULL; inCol = inCol->next) + struct mergeColHelper *col; + for (col = colList; col != NULL; col = col->next) { - if (inCol->mergedIx != -1) + if (col->mergedIx != -1) { - if (inCol->isKey) + if (col->isKey) { - newRow[inCol->mergedIx] = merge->key; + newRow[col->mergedIx] = merge->key; } - else if (inCol->isCount) + else if (col->isCount) { - long totalCount = summedCountAt(inCol->ix, merge); + long totalCount = summedCountAt(col->ix, merge); safef(countBuf, sizeof(countBuf), "%ld", totalCount); - newRow[inCol->mergedIx] = countBuf; + newRow[col->mergedIx] = countBuf; } - else if (inCol->isVal) + else if (col->isVal) { double val; if (countCol == NULL) - val = aveValAt(inCol->ix, merge); + val = aveValAt(col->ix, merge); else - val = weightedAveValAt(countCol->ix, inCol->ix, merge); + val = weightedAveValAt(countCol->ix, col->ix, merge); safef(valBuf, sizeof(valBuf), "%g", val); - newRow[inCol->mergedIx] = valBuf; + newRow[col->mergedIx] = valBuf; } - else if (inCol->isColor) + else if (col->isColor) { int r,g,b; - weightedAveColorAt(countCol->ix, inCol->ix, merge, &r, &g, &b); + weightedAveColorAt(countCol, col->ix, merge, &r, &g, &b); safef(colorBuf, sizeof(colorBuf), "#%02X%02X%02X", r, g, b); - newRow[inCol->mergedIx] = colorBuf; + newRow[col->mergedIx] = colorBuf; } - else if (inCol->sameAfterMerger) + else if (col->sameAfterMerger) { - newRow[inCol->mergedIx] = merge->aRow[inCol->ix]; + newRow[col->mergedIx] = merge->aRow[col->ix]; } } } fieldedTableAdd(tableOut, newRow, newFieldCount, tableOut->rowCount); } -/* Recompute ffArray */ -struct facetField **oldFfArray = facTab->ffArray; -struct facetField **newFfArray; -AllocArray(newFfArray, newFieldCount); -for (inCol = inColList; inCol != NULL; inCol = inCol->next) - { - if (inCol->mergedIx != -1) - newFfArray[inCol->mergedIx] = oldFfArray[inCol->ix]; - } - -*retFfArray = newFfArray; +*retFfArray = recomputeFfArray(facTab->ffArray, colList, newFieldCount); return tableOut; } struct fieldedTable *facetedTableSelect(struct facetedTable *facTab, struct cart *cart, struct facetField ***retFfArray) /* Return table containing rows of table that have passed facet selection */ { char *selList = facetedTableSelList(facTab, cart); struct fieldedTable *subtable = facetFieldSelectRows(facTab->table, selList, facTab->ffArray); struct facetField *ffSelList = deLinearizeFacetValString(selList); if (facTab->mergeFacetsOk) { return groupColumns(facTab, subtable, ffSelList, retFfArray); } else { *retFfArray = facTab->ffArray; return subtable; } } struct slInt *facetedTableSelectOffsets(struct facetedTable *facTab, struct cart *cart) /* Return a list of row positions that pass faceting */ { char *selList = facetedTableSelList(facTab, cart); struct fieldedTable *ft = facTab->table; int fieldCount = ft->fieldCount; struct slInt *retList = NULL; facetFieldsFromSqlTableInit(ft->fields, fieldCount, selList, facTab->ffArray); struct fieldedRow *fr; int ix = 0; for (fr = facTab->table->rowList; fr != NULL; fr = fr->next) { if (perRowFacetFields(fieldCount, fr->row, "", facTab->ffArray)) { struct slInt *el = slIntNew(ix); slAddHead(&retList, el); } ++ix; } slReverse(&retList); return retList; } +static int facetedTableMergedOffsetCmp(const void *va, const void *vb) +/* Compare to sort based on name start. */ +{ +const struct facetedTableMergedOffset *a = *((struct facetedTableMergedOffset **)va); +const struct facetedTableMergedOffset *b = *((struct facetedTableMergedOffset **)vb); +return cmpWordsWithEmbeddedNumbers(a->name, b->name); +} + + +struct facetedTableMergedOffset *facetedTableMakeMergedOffsets(struct facetedTable *facTab, + struct cart *cart) +/* Return a structure that will let us relatively rapidly merge together one row */ +{ +/* Figure out user selection of fields to select on and merge as ffSelList */ +char *selList = facetedTableSelList(facTab, cart); +struct facetField *ffSelList = deLinearizeFacetValString(selList); + +/* Get output column list and pointers to key columns */ +struct fieldedTable *tableIn = facTab->table; +struct mergeColHelper *countCol = NULL, *valCol = NULL, *colorCol = NULL; +struct mergeColHelper *colList = makeColList(facTab, tableIn, ffSelList, + &countCol, &valCol, &colorCol); +struct mergeGroup *mergeList = findMergeGroups(tableIn, colList); +int newFieldCount; +annotateColListAndMakeNewFields(colList, mergeList, &newFieldCount); + +fieldedTableResetRowIds(tableIn, 0); // Use the seldom-used ID column as indexes +struct facetedTableMergedOffset *tmoList = NULL; +struct mergeGroup *merge; +struct dyString *keyBuf = dyStringNew(0); +int outIx = 0; +for (merge = mergeList; merge != NULL; merge = merge->next) + { + struct facetedTableMergedOffset *tmo; + AllocVar(tmo); + slAddHead(&tmoList, tmo); + tmo->outIx = outIx++; + dyStringClear(keyBuf); + tmo->name = merge->key; + int r,g,b; + weightedAveColorAt(countCol, colorCol->ix, merge, &r, &g, &b); + safef(tmo->color, sizeof(tmo->color), "#%02X%02X%02X", r, g, b); + + struct slRef *ref; + for (ref = merge->rowRefList; ref != NULL; ref = ref->next) + { + struct fieldedRow *fr = ref->val; + struct facetedTableCountOffset *tco; + AllocVar(tco); + tco->valIx = fr->id; + int count = tco->count = (countCol != NULL ? sqlUnsigned(fr->row[countCol->ix]) : 1); + tmo->totalCount += count; + slAddHead(&tmo->colList, tco); + } + } +slSort(&tmoList, facetedTableMergedOffsetCmp); +return tmoList; +} + +void facetedTableMergeVals(struct facetedTableMergedOffset *tmoList, float *inVals, float *outVals) +/* Populate outVals array with columns of weighted averages derived from applying + * tmoList to inVals array. */ +{ +struct facetedTableMergedOffset *tmo; +for (tmo = tmoList; tmo != NULL; tmo = tmo->next) + { + struct facetedTableCountOffset *colList = tmo->colList; + if (colList->next == NULL) // specail case of one + { + outVals[tmo->outIx] = inVals[colList->valIx]; + } + else + { + double scale = 1.0/tmo->totalCount; + double weightedSum = 0; + struct facetedTableCountOffset *col; + for (col = colList; col != NULL; col = col->next) + { + weightedSum += inVals[col->valIx] * (double)col->count; + } + outVals[tmo->outIx] = weightedSum*scale; + } + } +} + void facetedTableWriteHtml(struct facetedTable *facTab, struct cart *cart, struct fieldedTable *selected, struct facetField **selectedFf, char *displayList, char *returnUrl, int maxLenField, struct hash *tagOutputWrappers, void *wrapperContext, int facetUsualSize) /* Write out the main HTML associated with facet selection and table. */ { facetedTableWebInit(); struct hash *emptyHash = hashNew(0); webFilteredFieldedTable(cart, selected, displayList, returnUrl, facTab->varPrefix, maxLenField, tagOutputWrappers, wrapperContext, FALSE, NULL, selected->rowCount, facetUsualSize, NULL, emptyHash, selectedFf, facTab->facets, NULL, facTab->mergeFacetsOk); hashFree(&emptyHash); }