8715d83d38d8bb18adfdd9ce05771b4cede85407 angie Fri Dec 5 09:52:07 2014 -0800 Moved some hgTables code that had been copied into a couple differentplaces into a new lib module, cartTrackDb. hgGenome/import.c has code that is fairly similar but that also needs to filter out custom tracks that were generated by hgGenome. diff --git src/hg/hgVai/libifyMe.c src/hg/hgVai/libifyMe.c index 8baf084..8db79a6 100644 --- src/hg/hgVai/libifyMe.c +++ src/hg/hgVai/libifyMe.c @@ -1,489 +1,297 @@ /* Stuff lifted from hgTables that should be libified. */ /* Copyright (C) 2014 The Regents of the University of California * See README in this or parent directory for licensing information. */ #include "common.h" #include "cheapcgi.h" #include "customTrack.h" #include "grp.h" #include "hdb.h" #include "hgFind.h" #include "hubConnect.h" #include "hui.h" #include "trackHub.h" #include "wikiTrack.h" #include "annoGratorQuery.h" #include "annoGratorGpVar.h" #include "annoStreamBigBed.h" #include "annoStreamDb.h" #include "annoStreamDbFactorSource.h" #include "annoStreamTab.h" #include "annoStreamVcf.h" #include "annoStreamWig.h" #include "annoGrateWigDb.h" #include "annoFormatTab.h" #include "annoFormatVep.h" #include "pgSnp.h" #include "vcf.h" #include "libifyMe.h" static boolean searchPosition(char *range, struct cart *cart, char *cartVar) /* Try and fill in region via call to hgFind. Return FALSE * if it can't find a single position. */ { struct hgPositions *hgp = NULL; char retAddr[512]; char position[512]; char *chrom = NULL; int start=0, end=0; char *db = cloneString(cartString(cart, "db")); // gets clobbered if position is not found! safef(retAddr, sizeof(retAddr), "%s", cgiScriptName()); hgp = findGenomePosWeb(db, range, &chrom, &start, &end, cart, TRUE, retAddr); if (hgp != NULL && hgp->singlePos != NULL) { safef(position, sizeof(position), "%s:%d-%d", chrom, start+1, end); cartSetString(cart, cartVar, position); return TRUE; } else if (start == 0) /* Confusing way findGenomePosWeb says pos not found. */ { cartSetString(cart, cartVar, hDefaultPos(db)); return FALSE; } else return FALSE; } boolean lookupPosition(struct cart *cart, char *cartVar) /* Look up position if it is not already seq:start-end. Return FALSE if it puts * up multiple positions. */ { char *db = cartString(cart, "db"); char *range = cartUsualString(cart, cartVar, ""); boolean isSingle = TRUE; range = trimSpaces(range); if (range[0] != 0) isSingle = searchPosition(range, cart, cartVar); else cartSetString(cart, cartVar, hDefaultPos(db)); return isSingle; } void nbSpaces(int count) /* Print some non-breaking spaces. */ { int i; for (i=0; i<count; ++i) printf(" "); } -//#*** --------------------- almost verbatim from hgTables/custom.c (+ globals) ------------------------- -static struct customTrack *getCustomTracks(char *database, struct cart *cart) -/* Get custom track list. */ -{ -static struct slName *browserLines = NULL; -static struct customTrack *theCtList = NULL; -if (theCtList == NULL) - theCtList = customTracksParseCart(database, cart, &browserLines, NULL); -return theCtList; -} -//#*** --------------------- end verbatim from hgTables/custom.c ------------------------- - -boolean hasCustomTracks(struct cart *cart) -/* Return TRUE if cart has custom tracks for the current db. */ -{ -char *db = cartString(cart, "db"); -struct customTrack *ctList = getCustomTracks(db, cart); -return (ctList != NULL); -} - -//#*** libify to hg/lib/wikiTrack.c -//#*** should it sort trackDb, or leave that to the caller? -void wikiTrackDb(struct trackDb **list) -/* create a trackDb entry for the wiki track */ -{ -struct trackDb *tdb; - -AllocVar(tdb); -tdb->track = WIKI_TRACK_TABLE; -tdb->table = WIKI_TRACK_TABLE; -tdb->shortLabel = WIKI_TRACK_LABEL; -tdb->longLabel = WIKI_TRACK_LONGLABEL; -tdb->visibility = tvFull; -tdb->priority = WIKI_TRACK_PRIORITY; - -tdb->html = hFileContentsOrWarning(hHelpFile(tdb->track)); -tdb->type = "none"; -tdb->grp = "map"; -tdb->canPack = FALSE; - -slAddHead(list, tdb); -slSort(list, trackDbCmp); -} - -static struct trackDb *getFullTrackList(struct cart *cart, struct hubConnectStatus *hubList, - struct grp **pHubGroups) -/* Get all tracks including custom tracks if any. *pHubGroups is reversed. */ -{ -char *db = cartString(cart, "db"); -struct trackDb *list = hTrackDb(db); -struct customTrack *ctList, *ct; - -/* exclude any track with a 'tableBrowser off' setting */ -//#*** NOTE: this only scans top-level tdb's, and would miss a setting applied -//#*** only to lower levels. So far we have not encountered such a track. -struct trackDb *tdb, *nextTdb, *newList = NULL; -for (tdb = list; tdb != NULL; tdb = nextTdb) - { - nextTdb = tdb->next; - if (tdbIsDownloadsOnly(tdb) || tdb->table == NULL) - { - //freeMem(tdb); // should not free tdb's. - // While hdb.c should and says it does cache the tdbList, it doesn't. - // The most notable reason that the tdbs are not cached is this hgTables CGI !!! - // It needs to be rewritten to make tdbRef structures for the lists it creates here! - continue; - } - - char *tbOff = trackDbSetting(tdb, "tableBrowser"); - if (tbOff == NULL || !startsWithWord("off", tbOff)) - slAddHead(&newList, tdb); - } -slReverse(&newList); -list = newList; - -/* add wikiTrack if enabled */ -if (wikiTrackEnabled(db, NULL)) - wikiTrackDb(&list); - -/* Add hub tracks. */ -struct trackDb *hubTrackList = hubCollectTracks(db, pHubGroups); -if (hubTrackList != NULL) - list = slCat(list, hubTrackList); - -/* Add custom tracks to list */ -ctList = getCustomTracks(db, cart); -for (ct = ctList; ct != NULL; ct = ct->next) - { - slAddHead(&list, ct->tdb); - } - -return list; -} - -static struct grp *makeGroupList(char *db, struct trackDb *trackList, - struct grp **pHubGrpList, boolean allTablesOk) -/* Get list of groups that actually have something in them. */ -{ -struct grp *groupsAll, *groupList = NULL, *group; -struct hash *groupsInTrackList = newHash(0); -struct hash *groupsInDb = newHash(0); -struct trackDb *track; - -/* Stream through track list building up hash of active groups. */ -for (track = trackList; track != NULL; track = track->next) - hashStoreName(groupsInTrackList, track->grp); - -/* Scan through group table, putting in ones where we have data. */ -groupsAll = hLoadGrps(db); -for (group = slPopHead(&groupsAll); group != NULL; group = slPopHead(&groupsAll)) - { - if (hashLookup(groupsInTrackList, group->name)) - { - slAddTail(&groupList, group); - hashAdd(groupsInDb, group->name, group); - } - else - grpFree(&group); - } - -/* if we have custom tracks, we want to add the track hubs - * after that group */ -struct grp *addAfter = NULL; -if (sameString(groupList->name, "user")) - addAfter = groupList; - -/* Add in groups from hubs. *pHubGrpList is reversed, so we add at head to restore order. */ -for (group = slPopHead(pHubGrpList); group != NULL; group = slPopHead(pHubGrpList)) - { - /* check to see if we're inserting hubs rather than - * adding them to the front of the list */ - if (addAfter != NULL) - { - group->next = addAfter->next; - addAfter->next = group; - } - else - slAddHead(&groupList, group); - hashAdd(groupsInDb, group->name, group); - } - -/* Do some error checking for tracks with group names that are - * not in db. Just warn about them. */ -for (track = trackList; track != NULL; track = track->next) - { - if (!hashLookup(groupsInDb, track->grp)) - warn("Track %s has group %s, which isn't in grp table", - track->table, track->grp); - } - -/* Create dummy group for all tracks. */ -AllocVar(group); -group->name = cloneString("allTracks"); -group->label = cloneString("All Tracks"); -slAddTail(&groupList, group); - -/* Create another dummy group for all tables. */ -if (allTablesOk) - { - AllocVar(group); - group->name = cloneString("allTables"); - group->label = cloneString("All Tables"); - slAddTail(&groupList, group); - } - -hashFree(&groupsInTrackList); -hashFree(&groupsInDb); -return groupList; -} - -void initGroupsTracksTables(struct cart *cart, - struct trackDb **retFullTrackList, struct grp **retFullGroupList) -/* Get lists of all tracks and of groups that actually have tracks in them. */ -{ -static boolean inited = FALSE; -static struct trackDb *fullTrackList = NULL; -static struct grp *fullGroupList = NULL; -if (! inited) - { - struct hubConnectStatus *hubList = hubConnectStatusListFromCart(cart); - struct grp *hubGrpList = NULL; - fullTrackList = getFullTrackList(cart, hubList, &hubGrpList); - char *db= cartString(cart, "db"); - fullGroupList = makeGroupList(db, fullTrackList, &hubGrpList, FALSE); - inited = TRUE; - } -if (retFullTrackList != NULL) - *retFullTrackList = fullTrackList; -if (retFullGroupList != NULL) - *retFullGroupList = fullGroupList; -} - //#*** duplicated in hgVarAnnoGrator and annoGratorTester struct annoAssembly *getAnnoAssembly(char *db) /* Make annoAssembly for db. */ { static struct annoAssembly *aa = NULL; if (aa == NULL) { char *nibOrTwoBitDir = hDbDbNibPath(db); if (nibOrTwoBitDir == NULL) errAbort("Can't find .2bit for db '%s'", db); char twoBitPath[HDB_MAX_PATH_STRING]; safef(twoBitPath, sizeof(twoBitPath), "%s/%s.2bit", nibOrTwoBitDir, db); char *path = hReplaceGbdb(twoBitPath); aa = annoAssemblyNew(db, path); freeMem(path); } return aa; } static boolean columnsMatch(struct asObject *asObj, struct sqlFieldInfo *fieldList) /* Return TRUE if asObj's column names match the given SQL fields. */ { if (asObj == NULL) return FALSE; struct sqlFieldInfo *firstRealField = fieldList; if (sameString("bin", fieldList->field) && differentString("bin", asObj->columnList->name)) firstRealField = fieldList->next; boolean columnsMatch = TRUE; struct sqlFieldInfo *field = firstRealField; struct asColumn *asCol = asObj->columnList; for (; field != NULL && asCol != NULL; field = field->next, asCol = asCol->next) { if (!sameString(field->field, asCol->name)) { columnsMatch = FALSE; break; } } if (field != NULL || asCol != NULL) columnsMatch = FALSE; return columnsMatch; } static struct asObject *asObjectFromFields(char *name, struct sqlFieldInfo *fieldList) /* Make autoSql text from SQL fields and pass it to asParse. */ { struct dyString *dy = dyStringCreate("table %s\n" "\"Column names grabbed from mysql\"\n" " (\n", name); struct sqlFieldInfo *field; for (field = fieldList; field != NULL; field = field->next) { char *sqlType = field->type; // hg19.wgEncodeOpenChromSynthGm12878Pk.pValue has sql type "float unsigned", // and I'd rather pretend it's just a float than work unsigned floats into autoSql. if (sameString(sqlType, "float unsigned")) sqlType = "float"; char *asType = asTypeNameFromSqlType(sqlType); if (asType == NULL) errAbort("No asTypeInfo for sql type '%s'!", field->type); dyStringPrintf(dy, " %s %s;\t\"\"\n", asType, field->field); } dyStringAppend(dy, " )\n"); return asParseText(dy->string); } static struct asObject *getAutoSqlForTable(char *db, char *dataDb, char *dbTable, struct trackDb *tdb) /* Get autoSql for dataDb.dbTable from tdb and/or db.tableDescriptions; * if it doesn't match columns, make one up from dataDb.table sql fields. */ //#*** should we just always use sql fields? { struct sqlConnection *connDataDb = hAllocConn(dataDb); struct sqlFieldInfo *fieldList = sqlFieldInfoGet(connDataDb, dbTable); hFreeConn(&connDataDb); struct asObject *asObj = NULL; if (tdb != NULL) { struct sqlConnection *connDb = hAllocConn(db); asObj = asForTdb(connDb, tdb); hFreeConn(&connDb); } if (columnsMatch(asObj, fieldList)) return asObj; else return asObjectFromFields(dbTable, fieldList); } static char *getBigDataFileName(char *db, struct trackDb *tdb, char *selTable, char *chrom) /* Get fileName from bigBed/bigWig/BAM/VCF database table, or bigDataUrl from custom track. */ { struct sqlConnection *conn = hAllocConn(db); char *fileOrUrl = bbiNameFromSettingOrTableChrom(tdb, conn, selTable, chrom); hFreeConn(&conn); return fileOrUrl; } struct annoStreamer *streamerFromTrack(struct annoAssembly *assembly, char *selTable, struct trackDb *tdb, char *chrom, int maxOutRows) /* Figure out the source and type of data and make an annoStreamer. */ { struct annoStreamer *streamer = NULL; char *db = assembly->name, *dataDb = db, *dbTable = selTable; if (chrom == NULL) chrom = hDefaultChrom(db); if (isCustomTrack(selTable)) { dbTable = trackDbSetting(tdb, "dbTableName"); if (dbTable != NULL) // This is really a database table, not a bigDataUrl CT. dataDb = CUSTOM_TRASH; } if (startsWith("wig", tdb->type)) streamer = annoStreamWigDbNew(dataDb, dbTable, assembly, maxOutRows); else if (sameString("vcfTabix", tdb->type)) { char *fileOrUrl = getBigDataFileName(db, tdb, selTable, chrom); streamer = annoStreamVcfNew(fileOrUrl, TRUE, assembly, maxOutRows); } else if (sameString("vcf", tdb->type)) { char *fileOrUrl = getBigDataFileName(dataDb, tdb, dbTable, chrom); streamer = annoStreamVcfNew(fileOrUrl, FALSE, assembly, maxOutRows); } else if (sameString("bam", tdb->type)) { warn("Sorry, BAM is not yet supported"); } else if (startsWith("bigBed", tdb->type)) { char *fileOrUrl = getBigDataFileName(db, tdb, selTable, chrom); streamer = annoStreamBigBedNew(fileOrUrl, assembly, maxOutRows); } else if (sameString("factorSource", tdb->type)) { char *sourceTable = trackDbSetting(tdb, "sourceTable"); char *inputsTable = trackDbSetting(tdb, "inputTrackTable"); streamer = annoStreamDbFactorSourceNew(db, tdb->track, sourceTable, inputsTable, assembly, maxOutRows); } else { struct sqlConnection *conn = hAllocConn(dataDb); char maybeSplitTable[1024]; if (sqlTableExists(conn, dbTable)) safecpy(maybeSplitTable, sizeof(maybeSplitTable), dbTable); else safef(maybeSplitTable, sizeof(maybeSplitTable), "%s_%s", chrom, dbTable); hFreeConn(&conn); struct asObject *asObj = getAutoSqlForTable(db, dataDb, maybeSplitTable, tdb); streamer = annoStreamDbNew(dataDb, maybeSplitTable, assembly, asObj, maxOutRows); } return streamer; } struct annoGrator *gratorFromBigDataFileOrUrl(char *fileOrUrl, struct annoAssembly *assembly, int maxOutRows, enum annoGratorOverlap overlapRule) /* Determine what kind of big data file/url we have and make streamer & grator for it. */ { struct annoStreamer *streamer = NULL; struct annoGrator *grator = NULL; if (endsWith(fileOrUrl, ".bb")) streamer = annoStreamBigBedNew(fileOrUrl, assembly, maxOutRows); else if (endsWith(fileOrUrl, ".vcf.gz")) streamer = annoStreamVcfNew(fileOrUrl, TRUE, assembly, maxOutRows); else if (endsWith(fileOrUrl, ".bw")) grator = annoGrateBigWigNew(fileOrUrl, assembly); else errAbort("Can't tell bigData type of file or url '%s'", fileOrUrl); if (grator == NULL) grator = annoGratorNew(streamer); grator->setOverlapRule(grator, overlapRule); return grator; } struct annoGrator *gratorFromTrackDb(struct annoAssembly *assembly, char *selTable, struct trackDb *tdb, char *chrom, int maxOutRows, struct asObject *primaryAsObj, enum annoGratorOverlap overlapRule) /* Figure out the source and type of data, make an annoStreamer & wrap in annoGrator. * If not NULL, primaryAsObj is used to determine whether we can make an annoGratorGpVar. */ { struct annoGrator *grator = NULL; char *dataDb = assembly->name, *dbTable = selTable; if (isCustomTrack(selTable)) { dataDb = CUSTOM_TRASH; dbTable = trackDbSetting(tdb, "dbTableName"); if (dbTable == NULL) { char *bigDataUrl = trackDbSetting(tdb, "bigDataUrl"); if (bigDataUrl != NULL) grator = gratorFromBigDataFileOrUrl(bigDataUrl, assembly, maxOutRows, overlapRule); else errAbort("Can't find dbTableName or bigDataUrl for custom track %s", selTable); } } if (startsWith("wig", tdb->type)) { grator = annoGrateWigDbNew(dataDb, dbTable, assembly, maxOutRows); } else { struct annoStreamer *streamer = streamerFromTrack(assembly, dbTable, tdb, chrom, maxOutRows); if (primaryAsObj != NULL && (asObjectsMatch(primaryAsObj, pgSnpAsObj()) || asObjectsMatch(primaryAsObj, vcfAsObj())) && asColumnNamesMatchFirstN(streamer->asObj, genePredAsObj(), 10)) grator = annoGratorGpVarNew(streamer); else grator = annoGratorNew(streamer); } grator->setOverlapRule(grator, overlapRule); return grator; }