af3a143571e5aa064eab75c34f9444b35413b562 chmalee Tue Nov 30 15:28:15 2021 -0800 Add snippet support to trix searching. Required changing the wordPos from the first highest matching wordIndex to the wordIndex of the actual span. Have trixContextIndex create a second level index for fast retrieval of line offsets in original text file used by ixIxx. Create a simple UI for navigating hgFind search results. diff --git src/hg/lib/hgFind.c src/hg/lib/hgFind.c index ffed074..f49fa44 100644 --- src/hg/lib/hgFind.c +++ src/hg/lib/hgFind.c @@ -30,30 +30,33 @@ #include "kgAlias.h" #include "kgProtAlias.h" #include "findKGAlias.h" #include "findKGProtAlias.h" #include "tigrCmrGene.h" #include "minGeneInfo.h" #include "pipeline.h" #include "hgConfig.h" #include "trix.h" #include "trackHub.h" #include "udc.h" #include "hubConnect.h" #include "bigBedFind.h" #include "genbank.h" #include "chromAlias.h" +#include "cart.h" +#include "cartTrackDb.h" +#include "jsonParse.h" // Exhaustive searches can lead to timeouts on CGIs (#11626). // However, hgGetAnn requires exhaustive searches (#11665). #define NONEXHAUSTIVE_SEARCH_LIMIT 500 #define EXHAUSTIVE_SEARCH_REQUIRED -1 char *hgAppName = ""; /* alignment tables to check when looking for mrna alignments */ static char *estTables[] = { "intronEst", "all_est", "xenoEst", NULL }; static char *estLabels[] = { "Spliced ESTs", "ESTs", "Other ESTs", NULL }; static char *mrnaTables[] = { "all_mrna", "xenoMrna", NULL }; static char *mrnaLabels[] = { "mRNAs", "Other mRNAs", NULL }; static struct dyString *hgpMatchNames = NULL; @@ -96,30 +99,47 @@ } static void hgPosTableFreeList(struct hgPosTable **pList) /* Free a list of dynamically allocated hgPos's */ { struct hgPosTable *el, *next; for (el = *pList; el != NULL; el = next) { next = el->next; hgPosTableFree(&el); } *pList = NULL; } +void searchCategoryFree(struct searchCategory **el) +{ +struct searchCategory *pEl = *el; +if (pEl != NULL) + { + freeMem(pEl->id); + freeMem(pEl->name); + freeMem(pEl->searchString); + freeMem(pEl->label); + freeMem(pEl->description); + freeMem(pEl->groupName); + trixClose(&pEl->trix); + slNameFreeList(pEl->parents); + slNameFreeList(pEl->errors); + } +} + #define HGPOSRANGESIZE 64 static char *hgPosBrowserRange(struct hgPos *pos, char range[HGPOSRANGESIZE]) /* Convert pos to chrN:123-456 format. If range parameter is NULL it returns * static buffer, otherwise writes and returns range. */ { static char buf[HGPOSRANGESIZE]; if (range == NULL) range = buf; safef(range, HGPOSRANGESIZE, "%s:%d-%d", pos->chrom, pos->chromStart+1, pos->chromEnd); return range; } @@ -446,224 +466,183 @@ result = FALSE; } return result; } struct tsrPos /* Little helper structure tying together search result * and pos, used by addKnownGeneItems */ { struct tsrPos *next; /* Next in list. */ struct trixSearchResult *tsr; /* Basically a gene symbol */ struct hgPos *posList; /* Associated list of positions. */ }; -static boolean isCanonical(struct sqlConnection *conn, char *geneName) -/* Look for the name in knownCannonical, return true if found */ -{ -boolean foundIt = FALSE; -if (sqlTableExists(conn, "knownCanonical")) - { - char query[512]; - sqlSafef(query, sizeof(query), "select transcript from knownCanonical" - " where transcript = '%s'", geneName); - struct sqlResult *sr = sqlGetResult(conn, query); - char **row; - if ((row = sqlNextRow(sr)) != NULL) - { - foundIt = TRUE; - } - sqlFreeResult(&sr); - } -return foundIt; -} - - static int hgPosCmpCanonical(const void *vhg1, const void *vhg2) // Compares two hgPos structs and returns an integer { const struct hgPos *hg1 = *((struct hgPos**)vhg1); const struct hgPos *hg2 = *((struct hgPos**)vhg2); int diff = trixSearchResultCmp(&hg1->tp->tsr, &hg2->tp->tsr); if (diff == 0) { diff = (hg2->canonical - hg1->canonical); if (diff == 0) { // Prioritize things on main chromosomes diff = chrNameCmpWithAltRandom(hg1->chrom, hg2->chrom); } } return diff; } static void addKnownGeneItems(struct hgPosTable *table, - struct trixSearchResult *tsrList, struct sqlConnection *conn, struct sqlConnection *conn2, char *name) + struct trixSearchResult *tsrList, struct sqlConnection *conn, char *name, struct trix *trix, struct hgFindSpec *hfs) /* Convert tsrList to posList, and hang posList off of table. */ { -/* This code works with just two SQL queries no matter how - * big the search result list is. For cases where the search - * result list is big (say 100 or 1000 items) this is noticably - * faster than the simpler-to-code approach that would do two - * queries for each search result. We pay for this speed tweak - * by having to construct a more elaborate query, and by having - * to maintain a hash to connect the query results back to the - * individual positions. */ struct dyString *dy = dyStringNew(0); struct trixSearchResult *tsr; struct hash *hash = hashNew(16); struct hgPos *pos, *posList = NULL; struct tsrPos *tpList = NULL, *tp; struct sqlResult *sr; char **row; int maxToReturn = NONEXHAUSTIVE_SEARCH_LIMIT; char *db = sqlGetDatabase(conn); char *dbName; if (sameString(name, "knownGene")) dbName = db; else dbName = name; if (slCount(tsrList) > maxToReturn) { - warn("Search terms are not very specific, only showing first %d matching UCSC Genes.", - maxToReturn); + //warn("Search terms are not very specific, only showing first %d matching UCSC Genes.", + // maxToReturn); tsr = slElementFromIx(tsrList, maxToReturn-1); tsr->next = NULL; } +char *context = hgFindSpecSetting(hfs, "searchTrixContext"); +if (context && sameString(context, "on")) + addSnippetsToSearchResults(tsrList, trix); /* Make hash of all search results - one for each known gene ID. */ for (tsr = tsrList; tsr != NULL; tsr = tsr->next) { lmAllocVar(hash->lm, tp); tp->tsr = tsr; slAddHead(&tpList, tp); hashAdd(hash, tsr->itemId, tp); } /* Stream through knownGenes table and make up a pos * for each mapping of each gene matching search. */ sqlDyStringPrintf(dy, - "select name,chrom,txStart,txEnd from %s.knownGene where name in (", dbName); + "select kg.name,kg.chrom,kg.txStart,kg.txEnd,geneSymbol,description,kc.transcript from %s.knownGene kg " + "join %s.kgXref on kg.name = %s.kgXref.kgID " + "left join %s.knownCanonical kc on " + "kc.transcript = kg.name and kc.chrom=kg.chrom and kc.chromStart = kg.txStart " + "where name in (", dbName, dbName, dbName, dbName); for (tsr = tsrList; tsr != NULL; tsr = tsr->next) { sqlDyStringPrintf(dy, "'%s'", tsr->itemId); if (tsr->next != NULL) sqlDyStringPrintf(dy, ","); } sqlDyStringPrintf(dy, ")"); sr = sqlGetResult(conn, dy->string); while ((row = sqlNextRow(sr)) != NULL) { tp = hashFindVal(hash, row[0]); + char nameBuf[256]; if (tp == NULL) internalErr(); + else + { AllocVar(pos); pos->chrom = cloneString(row[1]); pos->chromStart = sqlUnsigned(row[2]); pos->chromEnd = sqlUnsigned(row[3]); pos->tp = tp; slAddHead(&tp->posList, pos); - } -sqlFreeResult(&sr); - -/* Stream through kgXref table adding description and geneSymbol */ -dyStringClear(dy); -sqlDyStringPrintf(dy, - "select kgID,geneSymbol,description from %s.kgXref where kgID in (", dbName); -for (tsr = tsrList; tsr != NULL; tsr = tsr->next) - { - sqlDyStringPrintf(dy, "'%s'", tsr->itemId); - if (tsr->next != NULL) - sqlDyStringPrintf(dy, ","); - } -sqlDyStringPrintf(dy, ")"); - -sr = sqlGetResult(conn, dy->string); - -while ((row = sqlNextRow(sr)) != NULL) - { - tp = hashFindVal(hash, row[0]); - if (tp == NULL) - internalErr(); - for (pos = tp->posList; pos != NULL; pos = pos->next) - { - char nameBuf[256]; - safef(nameBuf, sizeof(nameBuf), "%s (%s)", row[1], row[0]); + safef(nameBuf, sizeof(nameBuf), "%s (%s)", row[4], row[0]); pos->name = cloneString(nameBuf); - if (isCanonical(conn2,row[0])) - { - pos->canonical = TRUE; - } - else{ - pos->canonical = FALSE; - } - pos->description = cloneString(row[2]); pos->browserName = cloneString(row[0]); + if (tp->tsr->snippet) + pos->description = tp->tsr->snippet; + else + pos->description = cloneString(row[5]); + pos->canonical = row[6] != NULL; } } sqlFreeResult(&sr); /* Hang all pos onto table. */ for (tp = tpList; tp != NULL; tp = tp->next) { struct hgPos *next; for (pos = tp->posList; pos != NULL; pos = next) { next = pos->next; slAddHead(&posList, pos); } } slSort(&posList, hgPosCmpCanonical); table->posList = posList; hashFree(&hash); dyStringFree(&dy); } -static boolean findKnownGeneFullText(char *db, char *term,struct hgPositions *hgp, char *name, char *path) -/* Look for position in full text. */ +static boolean findKnownGeneFullText(char *db, char *term,struct hgPositions *hgp, char *name, char *path, struct hgFindSpec *hfs, boolean measureTiming) +/* Look for position in full text. TODO: Add snippet support*/ { +long startTime = clock1000(); boolean gotIt = FALSE; struct trix *trix; struct trixSearchResult *tsrList; char *lowered = cloneString(term); char *keyWords[HGFIND_MAX_KEYWORDS]; int keyCount; +struct hgPosTable *table = NULL; trix = trixOpen(path); tolowers(lowered); keyCount = chopLine(lowered, keyWords); tsrList = trixSearch(trix, keyCount, keyWords, tsmExpand); if (tsrList != NULL) { - struct hgPosTable *table = addKnownGeneTable(db, hgp, name); + table = addKnownGeneTable(db, hgp, name); struct sqlConnection *conn = hAllocConn(db); struct sqlConnection *conn2 = hAllocConn(db); - addKnownGeneItems(table, tsrList, conn, conn2, name); + addKnownGeneItems(table, tsrList, conn, name, trix, hfs); hFreeConn(&conn); hFreeConn(&conn2); gotIt = TRUE; } freez(&lowered); trixSearchResultFreeList(&tsrList); trixClose(&trix); +// This is hacky but rely on knownGene table being at head of list +// for timing. TODO: make this more robust +if (measureTiming && table != NULL) + table->searchTime = clock1000() - startTime; return gotIt; } static char *getUiUrl(struct cart *cart) /* Get rest of UI from browser. */ { static struct dyString *dy = NULL; static char *s = NULL; if (dy == NULL) { dy = dyStringNew(64); if (cart != NULL) dyStringPrintf(dy, "%s=%s", cartSessionVarName(), cartSessionId(cart)); s = dy->string; } @@ -1622,33 +1601,34 @@ for (accEl = accList; accEl != NULL; accEl = accEl->next) { sqlSafef(query, sizeof(query), "select * from %s where mrnaAcc = '%s'", refLinkTable, accEl->name); sr = sqlGetResult(conn, query); while ((row = sqlNextRow(sr)) != NULL) { struct refLink *rl = refLinkLoad(row); slAddHead(pList, rl); } sqlFreeResult(&sr); } } static boolean findRefGenes(char *db, struct hgFindSpec *hfs, char *spec, - struct hgPositions *hgp) + struct hgPositions *hgp, boolean measureTiming) /* Look up refSeq genes in table. */ { +long startTime = clock1000(); struct sqlConnection *conn = hAllocConn(db); struct dyString *ds = dyStringNew(256); struct refLink *rlList = NULL, *rl; boolean gotRefLink = sqlTableExists(conn, refLinkTable); boolean found = FALSE; char *specNoVersion = cloneString(spec); // chop off the version number, e.g. "NM_000454.4 ", // but if spec starts with "." like ".stuff" then specNoVersion is entirely empty. (void) chopPrefix(specNoVersion); if (gotRefLink && isNotEmpty(specNoVersion)) { if (startsWith("NM_", specNoVersion) || startsWith("NR_", specNoVersion) || startsWith("XM_", specNoVersion)) { sqlDyStringPrintf(ds, "select * from %s where mrnaAcc = '%s'", refLinkTable, specNoVersion); addRefLinks(conn, ds, &rlList); @@ -1658,44 +1638,44 @@ sqlDyStringPrintf(ds, "select * from %s where protAcc = '%s'", refLinkTable, specNoVersion); addRefLinks(conn, ds, &rlList); } else if (isUnsignedInt(specNoVersion)) { sqlDyStringPrintf(ds, "select * from %s where locusLinkId = '%s'", refLinkTable, specNoVersion); addRefLinks(conn, ds, &rlList); dyStringClear(ds); sqlDyStringPrintf(ds, "select * from %s where omimId = '%s'", refLinkTable,specNoVersion); addRefLinks(conn, ds, &rlList); } else { char *indexFile = getGenbankGrepIndex(db, hfs, refLinkTable, "mrnaAccProduct"); - sqlDyStringPrintf(ds, "select * from %s where name like '%s%%'", - refLinkTable, specNoVersion); + sqlDyStringPrintf(ds, "select * from %s where name like '%s%%' limit %d", + refLinkTable, specNoVersion, NONEXHAUSTIVE_SEARCH_LIMIT); addRefLinks(conn, ds, &rlList); if (indexFile != NULL) { struct slName *accList = doGrepQuery(indexFile, refLinkTable, specNoVersion, NULL); addRefLinkAccs(conn, accList, &rlList); } else { dyStringClear(ds); - sqlDyStringPrintf(ds, "select * from %s where product like '%%%s%%'", - refLinkTable, specNoVersion); + sqlDyStringPrintf(ds, "select * from %s where product like '%%%s%%' limit %d", + refLinkTable, specNoVersion, NONEXHAUSTIVE_SEARCH_LIMIT); addRefLinks(conn, ds, &rlList); } } } if (rlList != NULL) { struct hgPosTable *table = NULL; struct hash *hash = newHash(8); for (rl = rlList; rl != NULL; rl = rl->next) { char where[64]; struct genePredReader *gpr; struct genePred *gp; /* Don't return duplicate mrna accessions */ @@ -1726,30 +1706,32 @@ } slAddHead(&table->posList, pos); pos->name = cloneString(rl->name); pos->browserName = cloneString(rl->mrnaAcc); dyStringClear(ds); dyStringPrintf(ds, "(%s) %s", rl->mrnaAcc, rl->product); pos->description = cloneString(ds->string); pos->chrom = hgOfficialChromName(db, gp->chrom); pos->chromStart = gp->txStart; pos->chromEnd = gp->txEnd; genePredFree(&gp); found = TRUE; } genePredReaderFree(&gpr); } + if (table != NULL && measureTiming) + table->searchTime = clock1000() - startTime; refLinkFreeList(&rlList); freeHash(&hash); } dyStringFree(&ds); hFreeConn(&conn); return(found); } /* Lowe lab additions */ static void addTigrCmrGenes(struct sqlConnection *conn, struct dyString *query, struct tigrCmrGene **pList) /* Query database and add returned tigrCmrGenes to head of list. */ { struct sqlResult *sr = sqlGetResult(conn, query->string); @@ -2113,44 +2095,45 @@ char **retChromName, int *retWinStart, int *retWinEnd, boolean *retIsMultiTerm, struct cart *cart, char *hgAppName, char **retMultiChrom, struct dyString *dyWarn) /* Search for positions that match spec (possibly ;-separated in which case *retIsMultiTerm is set). * Return a container of tracks and positions (if any) that match term. If different components * of a multi-term search land on different chromosomes then *retMultiChrom will be set. */ { struct hgPositions *hgp = NULL; char *chrom = NULL; int start = INT_MAX; int end = 0; char *terms[16]; int termCount = chopByChar(cloneString(spec), ';', terms, ArraySize(terms)); boolean multiTerm = (termCount > 1); +boolean measureTiming = cartUsualBoolean(cart, "measureTiming", FALSE); if (retIsMultiTerm) *retIsMultiTerm = multiTerm; if (retMultiChrom) *retMultiChrom = NULL; int i; for (i = 0; i < termCount; i++) { trimSpaces(terms[i]); if (isEmpty(terms[i])) continue; // Append warning messages to dyWarn, but allow errAborts to continue struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) - hgp = hgPositionsFind(db, terms[i], "", hgAppName, cart, multiTerm); + hgp = hgPositionsFind(db, terms[i], "", hgAppName, cart, multiTerm, measureTiming, NULL); errCatchEnd(errCatch); if (errCatch->gotError) errAbort("%s", errCatch->message->string); else if (isNotEmpty(errCatch->message->string)) dyStringAppend(dyWarn, errCatch->message->string); errCatchFree(&errCatch); if (hgp->singlePos != NULL) { if (retMultiChrom && chrom != NULL && differentString(chrom, hgp->singlePos->chrom)) *retMultiChrom = cloneString(chrom); chrom = hgp->singlePos->chrom; if (hgp->singlePos->chromStart < start) start = hgp->singlePos->chromStart; if (hgp->singlePos->chromEnd > end) end = hgp->singlePos->chromEnd; @@ -2283,66 +2266,67 @@ relStart+1, relEnd, table); } #endif static boolean isBigFileFind(struct hgFindSpec *hfs) /* is this a find on a big* file? */ { return sameString(hfs->searchType, "bigBed") || sameString(hfs->searchType, "bigPsl") || sameString(hfs->searchType, "bigBarChart") || sameString(hfs->searchType, "bigGenePred"); } static boolean findBigBed(struct cart *cart, char *db, struct hgFindSpec *hfs, char *spec, - struct hgPositions *hgp) + struct hgPositions *hgp, boolean measureTiming) /* Look up items in bigBed */ { struct trackDb *tdb = tdbFindOrCreate(db, NULL, hfs->searchTable); -return findBigBedPosInTdbList(cart, db, tdb, spec, hgp, hfs); +return findBigBedPosInTdbList(cart, db, tdb, spec, hgp, hfs, measureTiming); } boolean searchSpecial(struct cart *cart, char *db, struct hgFindSpec *hfs, char *term, int limitResults, struct hgPositions *hgp, boolean relativeFlag, - int relStart, int relEnd, boolean *retFound) + int relStart, int relEnd, boolean *retFound, boolean measureTiming) /* Handle searchTypes for which we have special code. Return true if * we have special code. Set retFind according to whether we find term. */ { boolean isSpecial = TRUE; boolean found = FALSE; char *upcTerm = cloneString(term); touppers(upcTerm); + if (startsWith("knownGene", hfs->searchType)) { char *knownDatabase = hdbDefaultKnownDb(db); char *name = (sameString(knownDatabase, db)) ? "knownGene" : knownDatabase; char *indexPath = hReplaceGbdb(hgFindSpecSetting(hfs, "searchTrix")); if (indexPath == NULL) indexPath = makeIndexPath(db, name); if (gotFullText(db, indexPath)) - found = findKnownGeneFullText(db, term, hgp, name, indexPath); + found = findKnownGeneFullText(db, term, hgp, name, indexPath, hfs, measureTiming); } else if (sameString(hfs->searchType, "refGene")) { - found = findRefGenes(db, hfs, term, hgp); + found = findRefGenes(db, hfs, term, hgp, measureTiming); } else if (isBigFileFind(hfs)) { - found = findBigBed(cart, db, hfs, term, hgp); + found = findBigBed(cart, db, hfs, term, hgp, measureTiming); } else if (sameString(hfs->searchType, "cytoBand")) { char *chrom; int start, end; found = hgFindCytoBand(db, term, &chrom, &start, &end); if (found) singlePos(hgp, hfs->searchDescription, NULL, hfs->searchTable, term, term, chrom, start, end); } else if (sameString(hfs->searchType, "gold")) { char *chrom; int start, end; found = findChromContigPos(db, term, &chrom, &start, &end); @@ -2395,30 +2379,31 @@ // example from human/hg19/trackDb.ra // xrefTable kgXref, ucscRetroInfo5 // xrefQuery select ucscRetroInfo5.name, spDisplayID from %s where spDisplayID like '%s%%' and kgName = kgID // NOTE this also goes into hgFindSpec table as hti fields hfs->xrefTable and hfs->xrefQuery. // hfs->xrefTable is sometimes a comma-separated list of fields // xrefTable = [hgFixed.refLink, ucscRetroInfo8] struct dyString *dy = dyStringNew(256); sqlCkIl(xrefTableSafe, hfs->xrefTable) // Replace the %s with %-s if it has not already been done in the upstream source .ra files // it would be better to do this upstream in .ra and hgFindSpec char *update = replaceChars(hfs->xrefQuery, " from %s ", " from %-s "); // this patches older values that still need it. sqlDyStringPrintf(dy, update, xrefTableSafe, term); +sqlDyStringPrintf(dy, " limit %d", NONEXHAUSTIVE_SEARCH_LIMIT); freeMem(update); sr = sqlGetResult(conn, dy->string); dyStringFree(&dy); while ((row = sqlNextRow(sr)) != NULL) { if (!isFuzzy || keyIsPrefixIgnoreCase(term, row[1])) { xrefPtr = slPairNew(cloneString(row[1]), cloneString(row[0])); slAddHead(&xrefList, xrefPtr); } } sqlFreeResult(&sr); hFreeConn(&conn); slReverse(&xrefList); @@ -2427,47 +2412,48 @@ return(xrefList); } char *addHighlight(char *db, char *chrom, unsigned start, unsigned end) /* Return a string that can be assigned to the cart var addHighlight, to add a yellow highlight * at db.chrom:start+1-end for search results. */ { char *color = "fcfcac"; struct dyString *dy = dyStringCreate("%s.%s:%u-%u#%s", db, chrom, start+1, end, color); return dyStringCannibalize(&dy); } static boolean doQuery(char *db, struct hgFindSpec *hfs, char *xrefTerm, char *term, struct hgPositions *hgp, boolean relativeFlag, int relStart, int relEnd, - boolean multiTerm, int limitResults) + boolean multiTerm, int limitResults, boolean measureTiming) /* Perform a query as specified in hfs, assuming table existence has been * checked and xref'ing has been taken care of. */ { struct slName *tableList = hSplitTableNames(db, hfs->searchTable); struct slName *tPtr = NULL; struct hgPosTable *table = NULL; struct hgPos *pos = NULL; struct sqlConnection *conn = hAllocConn(db); struct sqlResult *sr = NULL; char **row = NULL; char *termPrefix = hgFindSpecSetting(hfs, "termPrefix"); char *paddingStr = hgFindSpecSetting(hfs, "padding"); int padding = isEmpty(paddingStr) ? 0 : atoi(paddingStr); boolean found = FALSE; char *description = NULL; char buf[2048]; +long startTime = clock1000(); if (isNotEmpty(termPrefix) && startsWith(termPrefix, term)) term += strlen(termPrefix); if (isEmpty(term)) return(FALSE); if (isNotEmpty(hfs->searchDescription)) truncatef(buf, sizeof(buf), "%s", hfs->searchDescription); else safef(buf, sizeof(buf), "%s", hfs->searchTable); description = cloneString(buf); if (hgp->tableList != NULL && sameString(hgp->tableList->name, hfs->searchTable) && sameString(hgp->tableList->description, description)) @@ -2522,87 +2508,88 @@ pos->chromEnd += padding; if (pos->chromStart < 0) pos->chromStart = 0; if (pos->chromEnd > chromSize) pos->chromEnd = chromSize; } slAddHead(&table->posList, pos); } } if (table != NULL) slReverse(&table->posList); sqlFreeResult(&sr); hFreeConn(&conn); slFreeList(&tableList); +if (measureTiming && table) + table->searchTime += clock1000() - startTime; return(found); } static boolean hgFindUsingSpec(struct cart *cart, char *db, struct hgFindSpec *hfs, char *term, int limitResults, struct hgPositions *hgp, boolean relativeFlag, - int relStart, int relEnd, boolean multiTerm) + int relStart, int relEnd, boolean multiTerm, boolean measureTiming) /* Perform the search described by hfs on term. If successful, put results * in hgp and return TRUE. (If not, don't modify hgp.) */ { struct slPair *xrefList = NULL, *xrefPtr = NULL; boolean found = FALSE; if (hfs == NULL || term == NULL || hgp == NULL) errAbort("NULL passed to hgFindUsingSpec.\n"); if (strlen(term)<2 && ! (sameString(hfs->searchName, "knownGene") || sameString(hfs->searchName, "flyBaseGeneSymbolOneLetter"))) return FALSE; if (isNotEmpty(hfs->termRegex) && ! regexMatchNoCase(term, hfs->termRegex)) return(FALSE); if ((!(sameString(hfs->searchType, "mrnaKeyword") || sameString(hfs->searchType, "mrnaAcc"))) && !isBigFileFind(hfs)) { if (! hTableOrSplitExists(db, hfs->searchTable)) return(FALSE); } -if (isNotEmpty(hfs->searchType) && searchSpecial(cart, - db, hfs, term, limitResults, hgp, relativeFlag, - relStart, relEnd, &found)) +if (isNotEmpty(hfs->searchType) && searchSpecial(cart, db, hfs, term, limitResults, + hgp, relativeFlag, relStart, relEnd, &found, measureTiming)) return(found); if (isNotEmpty(hfs->xrefTable)) { struct sqlConnection *conn = hAllocConn(db); // NOTE hfs->xrefTable can sometimes contain a comma-separated table list, // rather than just a single table. char *tables = replaceChars(hfs->xrefTable, ",", " "); boolean exists = sqlTablesExist(conn, tables); hFreeConn(&conn); freeMem(tables); if (! exists) return(FALSE); xrefList = getXrefTerms(db, hfs, term); } else xrefList = slPairNew(cloneString(""), cloneString(term)); for (xrefPtr = xrefList; xrefPtr != NULL; xrefPtr = xrefPtr->next) { found |= doQuery(db, hfs, xrefPtr->name, (char *)xrefPtr->val, hgp, - relativeFlag, relStart, relEnd, multiTerm, limitResults); + relativeFlag, relStart, relEnd, multiTerm, limitResults, measureTiming); } slPairFreeValsAndList(&xrefList); return(found); } /* Support these formats for range specifiers. Note the ()'s around chrom, * start and end portions for substring retrieval: */ char *canonicalRangeExp = "^([[:alnum:]._#\\-]+)" "[[:space:]]*:[[:space:]]*" "([-0-9,]+)" "[[:space:]]*[-_][[:space:]]*" "([0-9,]+)$"; char *gbrowserRangeExp = @@ -2702,53 +2689,491 @@ } hFreeConn(&conn); return foundIt; } static struct hgFindSpec *hfsFind(struct hgFindSpec *list, char *name) /* Return first element of list that matches name. */ { struct hgFindSpec *el; for (el = list; el != NULL; el = el->next) if (sameString(name, el->searchName)) return el; return NULL; } +static void myLoadFindSpecs(char *db, struct searchCategory *categories, struct hgFindSpec **quickList, struct hgFindSpec **fullList) +/* Get all find specs where the search table or search name is what we want */ +{ +struct dyString *clause = dyStringNew(0); +struct searchCategory *categ; +sqlDyStringPrintf(clause, "select * from hgFindSpec_chmalee where searchName in ("); +for (categ = categories; categ != NULL; categ = categ->next) + { + sqlDyStringPrintf(clause, "'%s'", categ->id); + if (categ->next) + sqlDyStringPrintf(clause, ","); + } +sqlDyStringPrintf(clause, ") or searchTable in ("); +for (categ = categories; categ != NULL; categ = categ->next) + { + sqlDyStringPrintf(clause, "'%s'", categ->id); + if (categ->next) + sqlDyStringPrintf(clause, ","); + } +sqlDyStringPrintf(clause, ")"); +struct hgFindSpec *shortList = NULL, *longList = NULL; +struct sqlConnection *conn = hAllocConn(db); +struct sqlResult *sr = sqlGetResult(conn, dyStringCannibalize(&clause)); +char **row = NULL; +while ((row = sqlNextRow(sr)) != NULL) + { + struct hgFindSpec *hfs = hgFindSpecLoad(row); + if (hfs->shortCircuit) + slAddHead(&shortList, hfs); + else + slAddHead(&longList, hfs); + } +sqlFreeResult(&sr); +hFreeConn(&conn); + +if (quickList != NULL) + { + slSort(&shortList, hgFindSpecPriCmp); + *quickList = shortList; + } +else + hgFindSpecFreeList(&shortList); +if (fullList != NULL) + { + slSort(&longList, hgFindSpecPriCmp); + *fullList = longList; + } +else + hgFindSpecFreeList(&longList); +} + +static bool subtrackEnabledInTdb(struct trackDb *subTdb) +/* Return TRUE unless the subtrack was declared with "subTrack ... off". */ +{ +bool enabled = TRUE; +char *words[2]; +char *setting; +if ((setting = trackDbLocalSetting(subTdb, "parent")) != NULL) + { + if (chopLine(cloneString(setting), words) >= 2) + if (sameString(words[1], "off")) + enabled = FALSE; + } +else + return subTdb->visibility != tvHide; + +return enabled; +} + +static bool isSubtrackVisible(struct cart *cart, struct trackDb *tdb) +/* Has this subtrack not been deselected in hgTrackUi or declared with + * * "subTrack ... off"? -- assumes composite track is visible. */ +{ +boolean overrideComposite = (NULL != cartOptionalString(cart, tdb->track)); +bool enabledInTdb = subtrackEnabledInTdb(tdb); +char option[1024]; +safef(option, sizeof(option), "%s_sel", tdb->track); +boolean enabled = cartUsualBoolean(cart, option, enabledInTdb); +if (overrideComposite) + enabled = TRUE; +return enabled; +} + +static bool isParentVisible(struct cart *cart, struct trackDb *tdb) +// Are this track's parents visible? +{ +if (tdb->parent == NULL) + return TRUE; + +if (!isParentVisible(cart, tdb->parent)) + return FALSE; + +char *cartVis = cartOptionalString(cart, tdb->parent->track); +boolean vis; +if (cartVis != NULL) + vis = differentString(cartVis, "hide"); +else if (tdbIsSuperTrack(tdb->parent)) + vis = tdb->parent->isShow; +else + vis = tdb->parent->visibility != tvHide; +return vis; +} + +static bool isTrackVisible(struct cart *cart, struct trackDb *tdb) +/* Is a track visible? */ +{ +boolean isVisible = FALSE; +if (tdb->parent == NULL) + { + char *cartVis = cartOptionalString(cart, tdb->track); + if (cartVis == NULL) + isVisible = tdb->visibility != tvHide; + else + isVisible = differentString(cartVis, "hide"); + } +else if (isParentVisible(cart, tdb) && isSubtrackVisible(cart, tdb)) + isVisible = TRUE; +return isVisible; +} + +static struct searchableTrack *getSearchableTracks(struct cart *cart, char *database, struct hash *trackHash) +/* Return the list of all tracks with an hgFindSpec available */ +{ +if (trackHubDatabase(database)) + return NULL; +struct sqlConnection *conn = hAllocConn(database); +char query[1024]; +sqlSafef(query, sizeof(query), "select distinct tableName,shortLabel,longLabel,searchDescription,priority " + "from hgFindSpec_chmalee join trackDb_chmalee on " + "hgFindSpec_chmalee.searchTable=trackDb_chmalee.tableName or " + "hgFindSpec_chmalee.searchName=trackDb_chmalee.tableName where searchTable !='knownGene' and searchName != 'knownGene'" + "order by priority,shortLabel"); +struct sqlResult *sr = sqlGetResult(conn, query); +char **row = NULL; +struct searchableTrack *ret = NULL; +struct trackDb *tdb = NULL; +while ( (row = sqlNextRow(sr)) != NULL) + { + if ( (tdb = hashFindVal(trackHash, row[0])) != NULL) + { + struct searchableTrack *track = NULL; + AllocVar(track); + track->track = cloneString(row[0]); + track->shortLabel = cloneString(row[1]); + track->longLabel = cloneString(row[2]); + track->description = cloneString(row[3]); + track->visibility = isTrackVisible(cart, tdb); + track->priority = sqlDouble(row[4]); + track->grp = tdb->grp; + slAddHead(&ret, track); + } + } +sqlFreeResult(&sr); +hFreeConn(&conn); +slReverse(&ret); +return ret; +} + +//TODO: fix all these +#define hiveSearch "/hive/users/chmalee/search/manticore/" +#define publicHubsTrix "hubSearchTextRows" +#define helpDocsTrix "searchableDocs" + +static struct trackDb *hubCategoriesToTdbList(struct searchCategory *categories) +/* Make a list of trackDbs for the selected tracks */ +{ +struct trackDb *ret = NULL; +struct searchCategory *categ; +for (categ = categories; categ != NULL; categ = categ->next) + { + if (startsWith("hub_", categ->id)) + slAddHead(&ret, categ->tdb); + } +return ret; +} + +static struct searchCategory *searchCategoryFromTdb(struct trackDb *tdb, struct searchableTrack *searchTrack, int visibility) +/* Make a searchCategory from a leaf tdb, use searchCategory settings if possible, as they + * have more accurate visibilities and labels */ +{ +struct searchCategory *category = NULL; +AllocVar(category); +category->tdb = tdb; +category->id = tdb->track; +category->name = searchTrack != NULL ? searchTrack->shortLabel : tdb->shortLabel; +category->visibility = searchTrack != NULL ? searchTrack->visibility: tdb->visibility; +if (visibility > 0) // for when tdb is from a hub track + category->visibility = visibility; +category->priority = searchTrack != NULL ? searchTrack->priority : tdb->priority; +if (slCount(category->errors) == 0) + { + category->label = searchTrack != NULL ? searchTrack->shortLabel: tdb->shortLabel; + category->description = searchTrack != NULL ? searchTrack->description: tdb->longLabel; + category->groupName = searchTrack != NULL ? searchTrack->grp: tdb->grp; + category->parents = NULL; + while (tdb->parent) + { + slNameAddHead(&category->parents, tdb->parent->track); + slNameAddHead(&category->parents, tdb->parent->shortLabel); + tdb = tdb->parent; + } + if (category->parents) + slReverse(&category->parents); + } +return category; +} + +struct searchCategory *makeTrixCategory(char *indexName, char *database) +/* Fill out the fields for a category filter for the UI. */ +{ +struct searchCategory *category = NULL; +AllocVar(category); +struct errCatch *errCatch = errCatchNew(); +if (errCatchStart(errCatch)) + { + if (sameString(indexName, "publicHubs")) + { + category->id = "publicHubs"; + category->name = "publicHubs"; + category->label = "Public Hubs"; + category->description = "Search track names and track descriptions of public hubs"; + category->priority = 3.0; + char trixPath[PATH_LEN]; + safef(trixPath, sizeof(trixPath), "%s%s.ix", hiveSearch, publicHubsTrix); + category->trix = trixOpen(trixPath); + } + else if (sameString(indexName, "helpDocs")) + { + category->id = "helpDocs"; + category->name = "helpDocs"; + category->label = "Help Pages"; + category->description = "Search for matches to help documentation"; + category->visibility = 1; + category->priority = 4.0; + char trixPath[PATH_LEN]; + safef(trixPath, sizeof(trixPath), "%s%s.ix", hiveSearch, helpDocsTrix); + category->trix = trixOpen(trixPath); + } + else if (startsWith("trackDb", indexName)) + { + category->id = "trackDb"; + category->name = "trackDb"; + category->visibility = 1; + category->priority = 2.0; + char trixPath[PATH_LEN]; + safef(trixPath, sizeof(trixPath), "%s Track Labels/Descriptions", database); + category->label = cloneString(trixPath); + category->description = "Search for matches to track names or track descriptions"; + safef(trixPath, sizeof(trixPath), "/gbdb/%s/trackDb.ix", database); + category->trix = trixOpen(trixPath); + } + } +errCatchEnd(errCatch); +if (errCatch->gotError) + slAddHead(&category->errors, slNameNew(errCatch->message->string)); +return category; +} + +static struct searchCategory *makeCategoryForTrack(struct trackDb *tdb, struct searchableTrack *searchTrack) +/* Make a searchCategory from a track. If the track is any type of container, + * we will recurse down all the way to subtracks, as only leaf nodes have searchSpecs */ +{ +struct trackDb *sub; +struct searchCategory *ret = NULL; +if (tdb->subtracks) + { + for (sub = tdb->subtracks; sub != NULL; sub = sub->next) + { + if (sub->subtracks) + { + struct searchCategory *temp = makeCategoryForTrack(sub, searchTrack); + if (temp) + slAddHead(&ret, temp); + } + else + { + struct searchCategory *temp = searchCategoryFromTdb(sub, NULL, 0); + if (temp) + slAddHead(&ret, temp); + } + } + } +else + ret = searchCategoryFromTdb(tdb, searchTrack, 0); +return ret; +} + +struct searchCategory *makeCategory(struct cart *cart, char *categName, struct searchableTrack *searchTrack, char *db, + struct hash *trackHash, struct hash *groupHash) +/* Make a single searchCategory, unless the requested categName is a container + * track or track group (for example all phenotype tracks), in which case we make + * categories for each subtrack */ +{ +struct searchCategory *ret = NULL; + +if (sameString(categName, "helpDocs")) + ret = makeTrixCategory("helpDocs", NULL); +else if (sameString(categName, "publicHubs")) + ret = makeTrixCategory("publicHubs", NULL); +else if (startsWith("trackDb", categName)) + ret = makeTrixCategory("trackDb", db); +else if (hashLookup(groupHash, categName) != NULL) + { + // add all tracks for this track grouping + struct hashEl *hel, *helList = hashElListHash(trackHash); + for (hel = helList; hel != NULL; hel = hel->next) + { + struct trackDb *tdb = hel->val; + if (isTdbSearchable(tdb) && sameString(tdb->grp, categName)) + { + struct searchCategory *temp = makeCategoryForTrack(tdb, searchTrack); + if (temp) + slAddHead(&ret, temp); + } + } + } +else + { + // must be a track, ret will contain subtracks if necessary + struct trackDb *tdb = hashFindVal(trackHash, categName); + if (tdb) + ret = makeCategoryForTrack(tdb, searchTrack); + } +return ret; +} + +struct searchCategory *getCategsForNonDb(struct cart *cart, char *db, struct hash *trackHash, struct hash *groupHash) +/* Return the default categories for all databases */ +{ +struct searchCategory *ret = NULL; +struct searchCategory *kgCategory = makeCategory(cart, "knownGene", NULL, db, trackHash, groupHash); +if (kgCategory) + slAddHead(&ret, kgCategory); +struct searchCategory *helpDocCategory = makeCategory(cart, "helpDocs", NULL, db, trackHash, groupHash); +if (helpDocCategory) + slAddHead(&ret, helpDocCategory); +struct searchCategory *publicHubCategory = makeCategory(cart, "publicHubs", NULL, db, trackHash, groupHash); +if (publicHubCategory) + slAddHead(&ret, publicHubCategory); +char trackDbIndexName[2048]; +safef(trackDbIndexName, sizeof(trackDbIndexName), "trackDb%s", db); +struct searchCategory *tdbCategory = makeCategory(cart, trackDbIndexName, NULL, db, trackHash, groupHash); +if (tdbCategory) + slAddHead(&ret, tdbCategory); +return ret; +} + +struct searchCategory *getCategsForDatabase(struct cart *cart, char *db, struct hash *trackHash, struct hash *groupHash) +/* Get the default categories to search if user has not selected any before. + * By default we search for gene loci (knownGene), track names, and track items */ +{ +struct searchCategory *ret = NULL; + +struct searchableTrack *track = NULL, *searchableTracks = getSearchableTracks(cart, db, trackHash); +for (track = searchableTracks; track != NULL; track = track->next) + { + struct searchCategory *trackCategory = makeCategory(cart, track->track, track, db, trackHash, groupHash); + if (trackCategory) + { + if (ret) + slCat(&ret, trackCategory); + else + ret = trackCategory; + } + } +// add hub tracks to list +struct trackDb *tdb, *hubList = hubCollectTracks(db, NULL); +hubList = getSearchableBigBeds(hubList); +for (tdb = hubList; tdb != NULL; tdb = tdb->next) + { + int visibility = isTrackVisible(cart, tdb); + struct searchCategory *tmp = searchCategoryFromTdb(tdb, NULL, visibility); + if (tmp) + slAddHead(&ret, tmp); + } +return ret; +} + +struct searchCategory *getAllCategories(struct cart *cart, char *db, struct hash *trackHash, struct hash *groupHash) +{ +struct searchCategory *ret = NULL; +struct searchCategory *tdbCategories = getCategsForDatabase(cart, db, trackHash, groupHash); +if (tdbCategories) + ret = tdbCategories; +struct searchCategory *staticCategs = getCategsForNonDb(cart, db, trackHash, groupHash); +if (staticCategs) + { + if (ret) + slCat(&ret, staticCategs); + else + ret = staticCategs; + } +return ret; +} + +static boolean userDefinedSearch(char *db, char *term, int limitResults, struct cart *cart, + struct hgPositions *hgp, struct searchCategory *categories, boolean measureTiming) +/* If a search type(s) is specified in the cart, perform that search. + * If the search is successful, fill in hgp and return TRUE. */ +{ +boolean foundIt = FALSE; +struct hgFindSpec *shortList = NULL, *longList = NULL; +struct hash *foundSpecHash = hashNew(0); +struct hgFindSpec *hfs; +if (!trackHubDatabase(db)) + { + if (categories) + myLoadFindSpecs(db, categories, &shortList, &longList); + else + hgFindSpecGetAllSpecs(db, &shortList, &longList); + } +for (hfs = shortList; hfs != NULL; hfs = hfs->next) + { + boolean foundSpec = hgFindUsingSpec(cart, db, hfs, term, limitResults, hgp, FALSE, 0, 0, FALSE, measureTiming); + if (foundSpec) + hashAdd(foundSpecHash, hfs->searchTable, hfs->searchTable); + foundIt |= foundSpec; + } +for (hfs = longList; hfs != NULL; hfs = hfs->next) + { + if (hashFindVal(foundSpecHash, hfs->searchTable) != NULL) + continue; + foundIt |= hgFindUsingSpec(cart, db, hfs, term, limitResults, hgp, FALSE, 0, 0, FALSE, measureTiming); + } + +// lastly search any included track hubs, or in the case of an assembly hub, any of the tracks +struct trackDb *hubCategoryList = hubCategoriesToTdbList(categories); +if (hubCategoryList) + foundIt |= findBigBedPosInTdbList(cart, db, hubCategoryList, term, hgp, NULL, measureTiming); +if (foundIt) + { + fixSinglePos(hgp); + if (cart && hgp->singlePos && isNotEmpty(hgp->singlePos->highlight)) + cartSetString(cart, "addHighlight", hgp->singlePos->highlight); + slReverse(&hgp->tableList); + } +return foundIt; +} static boolean singleSearch(char *db, char *term, int limitResults, struct cart *cart, - struct hgPositions *hgp) + struct hgPositions *hgp, boolean measureTiming) /* If a search type is specified in the CGI line (not cart), perform that search. * If the search is successful, fill in hgp as a single-pos result and return TRUE. */ { char *search = cgiOptionalString("singleSearch"); if (search == NULL) return FALSE; cartRemove(cart, "singleSearch"); boolean foundIt = FALSE; if (sameString(search, "knownCanonical")) foundIt = searchKnownCanonical(db, term, hgp); else { struct hgFindSpec *shortList = NULL, *longList = NULL; hgFindSpecGetAllSpecs(db, &shortList, &longList); struct hgFindSpec *hfs = hfsFind(shortList, search); if (hfs == NULL) hfs = hfsFind(longList, search); if (hfs != NULL) - foundIt = hgFindUsingSpec(cart, db, hfs, term, limitResults, hgp, FALSE, 0,0, FALSE); + foundIt = hgFindUsingSpec(cart, db, hfs, term, limitResults, hgp, FALSE, 0,0, FALSE, measureTiming); else warn("Unrecognized singleSearch=%s in URL", search); } if (foundIt) { fixSinglePos(hgp); if (cart != NULL) cartSetString(cart, "hgFind.matches", hgp->tableList->posList->browserName); } return foundIt; } // a little data structure for combining multiple transcripts that resolve // to the same hgvs change. This struct can be used to fill out a struct hgPos struct hgvsHelper @@ -2908,31 +3333,31 @@ // highlight the 'mapped' bases to distinguish from padding hgp->tableList->posList->highlight = addHighlight(db, helper->chrom, spanStart, spanEnd); warn("%s", dyStringContents(allWarnings)); warn("Sorry, couldn't locate %s, moving to general location", term); } else warn("%s", dyStringContents(dyWarn)); } dyStringFree(&dyWarn); dyStringFree(&allWarnings); } return foundIt; } struct hgPositions *hgPositionsFind(char *db, char *term, char *extraCgi, - char *hgAppNameIn, struct cart *cart, boolean multiTerm) + char *hgAppNameIn, struct cart *cart, boolean multiTerm, boolean measureTiming, struct searchCategory *categories) /* Return container of tracks and positions (if any) that match term. */ { struct hgPositions *hgp = NULL, *hgpItem = NULL; regmatch_t substrs[4]; boolean canonicalSpec = FALSE; boolean gbrowserSpec = FALSE; boolean lengthSpec = FALSE; boolean singleBaseSpec = FALSE; boolean relativeFlag = FALSE; int relStart = 0, relEnd = 0; hgAppName = hgAppNameIn; // Exhaustive searches can lead to timeouts on CGIs (#11626). // However, hgGetAnn requires exhaustive searches (#11665). @@ -2942,32 +3367,39 @@ if (sameString(hgAppNameIn,"hgGetAnn")) limitResults = EXHAUSTIVE_SEARCH_REQUIRED; AllocVar(hgp); hgp->useAlias = FALSE; term = trimSpaces(term); if(isEmpty(term)) return hgp; hgp->query = cloneString(term); hgp->database = db; if (extraCgi == NULL) extraCgi = ""; hgp->extraCgi = cloneString(extraCgi); -if (singleSearch(db, term, limitResults, cart, hgp)) +if (singleSearch(db, term, limitResults, cart, hgp, measureTiming)) + return hgp; + +if (categories != NULL) + { + userDefinedSearch(db, term, limitResults, cart, hgp, categories, measureTiming); + if (hgp->posCount > 0) return hgp; + } /* Allow any search term to end with a :Start-End range -- also support stuff * pasted in from BED (chrom start end) or SQL query (chrom | start | end). * If found, strip it off and remember the start and end. */ char *originalTerm = term; if ((canonicalSpec = regexMatchSubstrNoCase(term, canonicalRangeExp, substrs, ArraySize(substrs))) || (gbrowserSpec = regexMatchSubstrNoCase(term, gbrowserRangeExp, substrs, ArraySize(substrs))) || (lengthSpec = regexMatchSubstrNoCase(term, lengthRangeExp, substrs, ArraySize(substrs))) || regexMatchSubstrNoCase(term, bedRangeExp, substrs, ArraySize(substrs)) || (singleBaseSpec = regexMatchSubstrNoCase(term, singleBaseExp, substrs, ArraySize(substrs))) || regexMatchSubstrNoCase(term, sqlRangeExp, substrs, ArraySize(substrs))) @@ -3031,54 +3463,54 @@ if (singleBaseSpec) { singleBaseSpec = relativeFlag = FALSE; term = cloneString(originalTerm); // restore original term relStart = relEnd = 0; } if (!trackHubDatabase(db)) hgFindSpecGetAllSpecs(db, &shortList, &longList); if ((cart == NULL) || (cartOptionalString(cart, "noShort") == NULL)) { hgp->shortCircuited = TRUE; for (hfs = shortList; hfs != NULL; hfs = hfs->next) { if (hgFindUsingSpec(cart, db, hfs, term, limitResults, hgp, relativeFlag, relStart, relEnd, - multiTerm)) + multiTerm, measureTiming)) { done = TRUE; if (! hgFindSpecSetting(hfs, "semiShortCircuit")) break; } } } else cartRemove(cart, "noShort"); if (! done) { hgp->shortCircuited = FALSE; for (hfs = longList; hfs != NULL; hfs = hfs->next) { hgFindUsingSpec(cart, db, hfs, term, limitResults, hgp, relativeFlag, relStart, relEnd, - multiTerm); + multiTerm, measureTiming); } /* Lowe lab additions -- would like to replace these with specs, but * will leave in for now. */ if (!trackHubDatabase(db)) findTigrGenes(db, term, hgp); - trackHubFindPos(cart, db, term, hgp); + trackHubFindPos(cart, db, term, hgp, measureTiming); } hgFindSpecFreeList(&shortList); hgFindSpecFreeList(&longList); if (cart != NULL) { if(hgpMatchNames == NULL) hgpMatchNames = dyStringNew(256); dyStringClear(hgpMatchNames); int matchCount = 0; for(hgpItem = hgp; hgpItem != NULL; hgpItem = hgpItem->next) { struct hgPosTable *hpTable = NULL; for(hpTable = hgpItem->tableList; hpTable != NULL; hpTable = hpTable->next) { struct hgPos *pos = NULL;