cdb81647deb6096ff29d18a21b4f9e83f45b4ac9 chmalee Tue Mar 7 14:38:53 2023 -0800 Fix bug where hubApi was allowing ranged requests for tracks with 'tableBrowser off' setting in trackDb. Fix cartTrackDbIsAccessDenied function to recognize correctly when 'tableBrowser off' setting is present on the same host as the CGI is running. Add tests to hubApi system to check more 'tableBrowser' settings and check getting data from more than one track at once diff --git src/hg/cgilib/cartTrackDb.c src/hg/cgilib/cartTrackDb.c index 000d877..95c4879 100644 --- src/hg/cgilib/cartTrackDb.c +++ src/hg/cgilib/cartTrackDb.c @@ -1,501 +1,508 @@ /* cartTrackDb - Combine custom tracks, hub tracks & hTrackDb to get unified groups/tracks/tables */ #include "common.h" #include "cartTrackDb.h" #include "cheapcgi.h" #include "customTrack.h" #include "hash.h" #include "hCommon.h" #include "hdb.h" #include "hgConfig.h" #include "hgMaf.h" #include "hubConnect.h" #include "joiner.h" #include "trackHub.h" #include "wikiTrack.h" /* Static globals */ static boolean useAC = FALSE; static struct slRef *accessControlTrackRefList = NULL; static struct trackDb *getFullTrackList(struct cart *cart, char *db, struct grp **pHubGroups) { struct trackDb *list = hTrackDb(db); /* add wikiTrack if enabled */ if (wikiTrackEnabled(db, NULL)) slAddHead(&list, wikiTrackDb()); slSort(&list, trackDbCmp); // Add hub tracks at head of list struct trackDb *hubTdbList = hubCollectTracks(db, pHubGroups); list = slCat(list, hubTdbList); /* exclude any track with a 'tableBrowser off' setting */ struct trackDb *tdb, *nextTdb, *newList = NULL; for (tdb = list; tdb != NULL; tdb = nextTdb) { nextTdb = tdb->next; if (tdbIsDownloadsOnly(tdb) || tdb->table == NULL) { //freeMem(tdb); // should not free tdb's. // While hdb.c should and says it does cache the tdbList, it doesn't. // The most notable reason that the tdbs are not cached is this hgTables CGI !!! // It needs to be rewritten to make tdbRef structures for the lists it creates here! continue; } char *tbOff = trackDbSetting(tdb, "tableBrowser"); if (useAC && tbOff != NULL && (startsWithWord("off", tbOff) || startsWithWord("noGenome", tbOff) || startsWithWord("tbNoGenome", tbOff))) { slAddHead(&accessControlTrackRefList, slRefNew(tdb)); if (! startsWithWord("off", tbOff)) slAddHead(&newList, tdb); } else slAddHead(&newList, tdb); } slReverse(&newList); list = newList; // Add custom tracks at head of list if (cart) { struct customTrack *ctList, *ct; ctList = customTracksParseCart(db, cart, NULL, NULL); for (ct = ctList; ct != NULL; ct = ct->next) { slAddHead(&list, ct->tdb); } } return list; } static struct grp *makeGroupList(char *db, struct trackDb *trackList, struct grp **pHubGrpList, boolean allTablesOk) /* Get list of groups that actually have something in them. */ { struct grp *groupsAll, *groupList = NULL, *group; struct hash *groupsInTrackList = newHash(0); struct hash *groupsInDatabase = newHash(0); struct trackDb *track; /* Do some error checking for tracks with group names that are not in database. * Warnings at this stage mess up CGIs that may produce text output like hgTables & hgIntegrator, * so don't warn, just put CTs in group user and others in group x. */ groupsAll = hLoadGrps(db); if (!trackHubDatabase(db)) { struct hash *allGroups = hashNew(0); for (group = groupsAll; group != NULL; group = group->next) hashAdd(allGroups, group->name, group); for (track = trackList; track != NULL; track = track->next) { /* If track isn't in a track up, and has a group we don't know about, change it to one we do. */ if (!startsWith("hub_", track->grp) && !hashLookup(allGroups, track->grp)) { fprintf(stderr, "Track %s has group %s, which isn't in grp table\n", track->table, track->grp); if (isCustomTrack(track->track)) track->grp = cloneString("user"); else track->grp = cloneString("x"); } } hashFree(&allGroups); } /* Stream through track list building up hash of active groups. */ for (track = trackList; track != NULL; track = track->next) { if (!hashLookup(groupsInTrackList,track->grp)) hashAdd(groupsInTrackList, track->grp, NULL); } /* Scan through group table, putting in ones where we have data. */ for (group = slPopHead(&groupsAll); group != NULL; group = slPopHead(&groupsAll)) { if (hashLookup(groupsInTrackList, group->name)) { slAddTail(&groupList, group); hashAdd(groupsInDatabase, group->name, group); } else grpFree(&group); } /* if we have custom tracks, we want to add the track hubs * after that group */ struct grp *addAfter = NULL; if ((groupList != NULL) && sameString(groupList->name, "user")) addAfter = groupList; /* Add in groups from hubs. */ for (group = slPopHead(pHubGrpList); group != NULL; group = slPopHead(pHubGrpList)) { // if the group isn't represented in any track, don't add it to list if (!hashLookup(groupsInTrackList,group->name)) continue; /* check to see if we're inserting hubs rather than * adding them to the front of the list */ struct grp *newGrp = grpDup(group); if (addAfter != NULL) { newGrp->next = addAfter->next; addAfter->next = newGrp; } else slAddHead(&groupList, newGrp); hashAdd(groupsInDatabase, newGrp->name, newGrp); } /* Create dummy group for all tracks. */ AllocVar(group); group->name = cloneString("allTracks"); group->label = cloneString("All Tracks"); slAddTail(&groupList, group); /* Create another dummy group for all tables. */ if (allTablesOk) { AllocVar(group); group->name = cloneString("allTables"); group->label = cloneString("All Tables"); slAddTail(&groupList, group); } hashFree(&groupsInTrackList); hashFree(&groupsInDatabase); return groupList; } void cartTrackDbInit(struct cart *cart, struct trackDb **retFullTrackList, struct grp **retFullGroupList, boolean useAccessControl) /* Get lists of all tracks and of groups that actually have tracks in them. * If useAccessControl, exclude tracks with 'tableBrowser off' nor tables listed * in the table tableAccessControl. */ { char *db = cartString(cart, "db"); useAC = useAccessControl; struct grp *hubGrpList = NULL; struct trackDb *fullTrackList = getFullTrackList(cart, db, &hubGrpList); boolean allTablesOk = hAllowAllTables() && !trackHubDatabase(db); struct grp *fullGroupList = makeGroupList(db, fullTrackList, &hubGrpList, allTablesOk); if (retFullTrackList != NULL) *retFullTrackList = fullTrackList; if (retFullGroupList != NULL) *retFullGroupList = fullGroupList; } void cartTrackDbInitForApi(struct cart *cart, char *db, struct trackDb **retFullTrackList, struct grp **retFullGroupList, boolean useAccessControl) /* Similar to cartTrackDbInit, but allow cart to be NULL */ { useAC = useAccessControl; struct grp *hubGrpList = NULL; struct trackDb *fullTrackList = getFullTrackList(cart, db, &hubGrpList); boolean allTablesOk = hAllowAllTables() && !trackHubDatabase(db); struct grp *fullGroupList = makeGroupList(db, fullTrackList, &hubGrpList, allTablesOk); if (retFullTrackList != NULL) *retFullTrackList = fullTrackList; if (retFullGroupList != NULL) *retFullGroupList = fullGroupList; } static char *chopAtFirstDot(char *string) /* Terminate string at first '.' if found. Return string for convenience. */ { char *ptr = strchr(string, '.'); if (ptr != NULL) *ptr = '\0'; return string; } struct accessControl /* Restricted permission settings for a table */ { struct slName *hostList; // List of hosts that are allowed to view this table boolean isNoGenome; // True if it's OK for position range but not genome-wide query + boolean isFullDeny; // True if track/table is completely inaccessible }; void accessControlAddHost(struct accessControl *ac, char *host) /* Alloc an slName for host (unless NULL or empty) and add it to ac->hostList. */ { if (isNotEmpty(host)) slAddHead(&ac->hostList, slNameNew(host)); } -struct accessControl *accessControlNew(char *host, boolean isNoGenome) +struct accessControl *accessControlNew(char *host, boolean isNoGenome, boolean isFullDeny) /* Alloc, init & return accessControl. */ { struct accessControl *ac; AllocVar(ac); accessControlAddHost(ac, host); ac->isNoGenome = isNoGenome; +ac->isFullDeny = isFullDeny; return ac; } -static void acHashAddOneTable(struct hash *acHash, char *table, char *host, boolean isNoGenome) +static void acHashAddOneTable(struct hash *acHash, char *table, char *host, boolean isNoGenome, boolean isFullDeny) /* If table is already in acHash, update its accessControl fields; otherwise store a * new accessControl for table. */ { struct hashEl *hel = hashLookup(acHash, table); if (hel == NULL) { - struct accessControl *ac = accessControlNew(host, isNoGenome); + struct accessControl *ac = accessControlNew(host, isNoGenome, isFullDeny); hashAdd(acHash, table, ac); } else { struct accessControl *ac = hel->val; ac->isNoGenome = isNoGenome; + ac->isFullDeny = isFullDeny; accessControlAddHost(ac, host); } } static struct hash *accessControlInit(char *db) /* Return a hash associating restricted table/track names in the given db/conn * with virtual hosts -- hash is empty if there is no tableAccessControl table and no * accessControlTrackRefList (see getFullTrackList). */ { struct hash *acHash = hashNew(0); if (! trackHubDatabase(db)) { struct sqlConnection *conn = hAllocConn(db); if (sqlTableExists(conn, "tableAccessControl")) { struct sqlResult *sr = NULL; char **row = NULL; acHash = newHash(0); char query[1024]; sqlSafef(query, sizeof query, "select name,host from tableAccessControl"); sr = sqlGetResult(conn, query); while ((row = sqlNextRow(sr)) != NULL) - acHashAddOneTable(acHash, row[0], chopAtFirstDot(row[1]), FALSE); + acHashAddOneTable(acHash, row[0], chopAtFirstDot(row[1]), FALSE, FALSE); sqlFreeResult(&sr); } hFreeConn(&conn); } struct slRef *tdbRef; // init accessControlTrackRefList if (!accessControlTrackRefList) { boolean oldAC = useAC; useAC = TRUE; (void)getFullTrackList(NULL, db, NULL); useAC = oldAC; } for (tdbRef = accessControlTrackRefList; tdbRef != NULL; tdbRef = tdbRef->next) { struct trackDb *tdb = tdbRef->val; char *tbOff = cloneString(trackDbSetting(tdb, "tableBrowser")); if (isEmpty(tbOff)) errAbort("accessControlInit bug: tdb for %s does not have tableBrowser setting", tdb->track); // First word is "off" or "noGenome" or "tbNoGenome": char *type = nextWord(&tbOff); boolean isNoGenome = sameString(type, "noGenome"); + boolean isFullDeny = sameString(type, "off"); if (!isNoGenome) isNoGenome = sameString(type, "off") || sameString(type, "tbNoGenome"); // like 'noGenome' but only in the table browser, not the API // since the API does not use this function // Add track table to acHash: - acHashAddOneTable(acHash, tdb->table, NULL, isNoGenome); + acHashAddOneTable(acHash, tdb->table, NULL, isNoGenome, isFullDeny); // Remaining words are additional table names to add: char *tbl; while ((tbl = nextWord(&tbOff)) != NULL) - acHashAddOneTable(acHash, tbl, NULL, isNoGenome); + acHashAddOneTable(acHash, tbl, NULL, isNoGenome, isFullDeny); // add any subtracks that don't have their own setting overriding us struct trackDb *subTdb; for (subTdb = tdb->subtracks; subTdb != NULL; subTdb = subTdb->next) { char *subTdbOff = cloneString(trackDbSetting(tdb, "tableBrowser")); if (!subTdbOff) - acHashAddOneTable(acHash, subTdb->table, NULL, isNoGenome); + acHashAddOneTable(acHash, subTdb->table, NULL, isNoGenome, isFullDeny); else { char *subTdbType = nextWord(&subTdbOff); boolean subTdbIsNoGenome = sameString(subTdbType, "noGenome"); + boolean subTdbIsDenied = sameString(subTdbType, "off"); if (!subTdbIsNoGenome) subTdbIsNoGenome = sameString(subTdbType, "off") || sameString(subTdbType, "tbNoGenome"); - acHashAddOneTable(acHash, subTdb->table, NULL, subTdbIsNoGenome); + acHashAddOneTable(acHash, subTdb->table, NULL, subTdbIsNoGenome, subTdbIsDenied); } } } return acHash; } static struct hash *getCachedAcHash(char *db) /* Returns a hash that maps table names to accessControl, creating it if necessary. */ { static struct hash *dbToAcHash = NULL; if (dbToAcHash == NULL) dbToAcHash = hashNew(0); struct hash *acHash = hashFindVal(dbToAcHash, db); if (acHash == NULL) { acHash = accessControlInit(db); hashAdd(dbToAcHash, db, acHash); } return acHash; } static char *getCachedCurrentHost() /* Return the current host name chopped at the first '.' or NULL if not in the CGI environment. */ { static char *currentHost = NULL; if (currentHost == NULL) { currentHost = cloneString(cgiServerName()); if (currentHost == NULL) return NULL; else chopAtFirstDot(currentHost); } return currentHost; } boolean cartTrackDbIsAccessDenied(char *db, char *table) /* Return TRUE if useAccessControl=TRUE was passed to cartTrackDbInit and * if access to table is denied (at least on this host) by 'tableBrowser off' * or by the tableAccessControl table. */ { if (!useAC) return FALSE; struct hash *acHash = getCachedAcHash(db); struct accessControl *ac = hashFindVal(acHash, table); if (ac == NULL) return FALSE; +if (ac->isFullDeny) + return TRUE; if (ac->isNoGenome && ac->hostList == NULL) return FALSE; char *currentHost = getCachedCurrentHost(); if (currentHost == NULL) warn("accessControl: unable to determine current host"); return (! slNameInList(ac->hostList, currentHost)); } boolean cartTrackDbIsNoGenome(char *db, char *table) /* Return TRUE if range queries, but not genome-queries, are permitted for this table. */ { struct hash *acHash = getCachedAcHash(db); struct accessControl *ac = hashFindVal(acHash, table); return (ac != NULL && ac->isNoGenome); } static void addTablesAccordingToTrackType(char *db, struct slName **pList, struct hash *uniqHash, struct trackDb *track) /* Parse out track->type and if necessary add some tables from it. */ { struct slName *name; char *trackDupe = cloneString(track->type); if (trackDupe != NULL && trackDupe[0] != 0) { char *s = trackDupe; char *type = nextWord(&s); if (sameString(type, "wigMaf")) { static char *wigMafAssociates[] = {"frames", "summary"}; int i; for (i=0; i<ArraySize(wigMafAssociates); ++i) { char *setting = wigMafAssociates[i]; char *table = trackDbSetting(track, setting); if (table != NULL) { name = slNameNew(table); slAddHead(pList, name); hashAdd(uniqHash, table, NULL); } } /* include conservation wiggle tables */ struct consWiggle *wig, *wiggles = wigMafWiggles(db, track); slReverse(&wiggles); for (wig = wiggles; wig != NULL; wig = wig->next) { name = slNameNew(wig->table); slAddHead(pList, name); hashAdd(uniqHash, wig->table, NULL); } } if (track->subtracks) { struct slName *subList = NULL; struct slRef *tdbRefList = trackDbListGetRefsToDescendantLeaves(track->subtracks); slSort(&tdbRefList, trackDbRefCmp); struct slRef *tdbRef; for (tdbRef = tdbRefList; tdbRef != NULL; tdbRef = tdbRef->next) { struct trackDb *subTdb = tdbRef->val; name = slNameNew(subTdb->table); slAddTail(&subList, name); hashAdd(uniqHash, subTdb->table, NULL); } pList = slCat(pList, subList); } } freez(&trackDupe); } struct slName *cartTrackDbTablesForTrack(char *db, struct trackDb *track, boolean useJoiner) /* Return list of all tables associated with track. If useJoiner, the result can include * non-positional tables that are related to track by all.joiner. */ { static struct joiner *allJoiner = NULL; struct hash *uniqHash = newHash(8); struct slName *name, *nameList = NULL; char *trackTable = track->table; /* suppress for parent tracks -- only the subtracks have tables */ if (track->subtracks == NULL) { name = slNameNew(trackTable); slAddHead(&nameList, name); hashAdd(uniqHash, trackTable, NULL); } addTablesAccordingToTrackType(db, &nameList, uniqHash, track); if (useJoiner) { if (allJoiner == NULL) allJoiner = joinerRead("all.joiner"); struct slName *joinedList = NULL, *t; for (t = nameList; t != NULL; t = t->next) { struct joinerPair *jpList, *jp; jpList = joinerRelate(allJoiner, db, t->name, db); for (jp = jpList; jp != NULL; jp = jp->next) { struct joinerDtf *dtf = jp->b; // If we are checking a non-assembly database like hgFixed, then // we can't have a valid tableBrowser setting to deny the table // anyways (because we have no tracks for that database), so don't // bother checking unless the joined tables have the same database if (sameString(dtf->database, db) && cartTrackDbIsAccessDenied(dtf->database, dtf->table)) continue; char buf[256]; char *s; if (sameString(dtf->database, db)) s = dtf->table; else { safef(buf, sizeof(buf), "%s.%s", dtf->database, dtf->table); s = buf; } if (!hashLookup(uniqHash, s)) { hashAdd(uniqHash, s, NULL); name = slNameNew(s); slAddHead(&joinedList, name); } } } slNameSort(&joinedList); nameList = slCat(nameList, joinedList); } hashFree(&uniqHash); return nameList; }