e6ddf60465deb96e43be6738c5ca6a7a6168cac8
braney
Sat Aug 22 15:08:59 2026 -0700
hg/lib: add an option to send the content policy as an http response header
New cspWriteResponseHeader() in hg/lib/hCommon.c, gated on hg.conf's
cspResponseHeader, which defaults off everywhere. The policy string itself is
still built by the existing code in lib/htmshell.c, which now also knows how to
format it as a response header. Both carry the same nonce, since getNonce() is
one per process, so a page may safely have the header and the meta tag.
Most pages pick it up from addHttpHeaders() in cart.c, the existing hook for
extra response headers, which every cart based CGI already passes through. Six
places build their own http header block and so call it directly: the two "too
many requests" pages, the captcha and its error page, the hubApi help redirect,
and the hgSearch redirect to hgTracks.
Inline scripts on three of those pages now carry the nonce, and the policy
allows the Cloudflare script the bot check loads, so the option works when it
is turned on. The Cloudflare entry is the only part of this that takes effect
with the option off.
diff --git src/hg/hubApi/hubApi.c src/hg/hubApi/hubApi.c
index 979e8fa8725..ad680bc4da0 100644
--- src/hg/hubApi/hubApi.c
+++ src/hg/hubApi/hubApi.c
@@ -1,1760 +1,1763 @@
/* hubApi - access mechanism to hub data resources. */
#include "dataApi.h"
#include "botDelay.h"
+#include "hCommon.h"
#include "jsHelper.h"
#include "srcVersion.h"
#include "asmAlias.h"
#include "hubSpaceKeys.h"
/* can not include bamFile.h with the liftOver business, there
* is a conflict in a definition of the enum 'bed'
*/
#include "bamFile.h"
/*
+------------------+------------------+------+-----+---------+-------+
| Field | Type | Null | Key | Default | Extra |
+------------------+------------------+------+-----+---------+-------+
| hubUrl | longblob | NO | PRI | NULL | |
| shortLabel | varchar(255) | NO | | NULL | |
| longLabel | varchar(255) | NO | | NULL | |
| registrationTime | varchar(255) | NO | | NULL | |
| dbCount | int(10) unsigned | NO | | NULL | |
| dbList | blob | YES | | NULL | |
| descriptionUrl | longblob | YES | | NULL | |
+------------------+------------------+------+-----+---------+-------+
*/
/* Global Variables for all modules */
static int maxItemLimit = 1000000; /* maximum of 1,000,000 items returned */
int maxItemsOutput = 1000000; /* can be set in URL maxItemsOutput=N */
boolean reachedMaxItems = FALSE; /* during getData, signal to return */
long long itemsReturned = 0; /* for getData functions, number of items returned */
/* for debugging purpose, current bot delay value */
int botDelay = 0;
boolean debug = FALSE; /* can be set in URL debug=1, to turn off: debug=0 */
#define delayFraction 0.03
/* default is to list all trackDb entries, composite containers too.
* This option will limit to only the actual track entries with data
*/
boolean trackLeavesOnly = FALSE; /* set by CGI parameter 'trackLeavesOnly' */
/* this selects output type 'arrays', where the default type is: objects */
boolean jsonOutputArrays = FALSE; /* set by CGI parameter 'jsonOutputArrays' */
boolean measureTiming = FALSE; /* set by CGI parameters */
/* downloadUrl for use in error exits when reachedMaxItems */
struct dyString *downloadUrl = NULL;
/* valid argument listings to verify extraneous arguments */
char *argListPublicHubs[] = { NULL };
char *argListUcscGenomes[] = { argGenome, NULL };
char *argListGenarkGenomes[] = { argMaxItemsOutput, argGenome, NULL };
char *argListHubGenomes[] = { argHubUrl, NULL };
char *argListTracks[] = { argGenome, argHubUrl, argTrackLeavesOnly, NULL };
char *argListChromosomes[] = { argGenome, argHubUrl, argTrack, NULL };
char *argListSchema[] = { argGenome, argHubUrl, argTrack, NULL };
char *argListFiles[] = { argGenome, argMaxItemsOutput, argFormat, argSkipContext, argFileType, NULL };
char *argGetDataTrack[] = { argGenome, argHubUrl, argTrack, argChrom, argStart, argEnd, argMaxItemsOutput, argJsonOutputArrays, NULL };
char *argGetDataSequence[] = { argGenome, argHubUrl, argTrack, argChrom, argStart, argEnd, argRevComp, NULL };
char *argSearch[] = {argSearchTerm, argGenome, argHubUrl, argCategories, NULL};
char *argFindGenome[] = {argQ, argMaxItemsOutput, argJsonOutputArrays, argStatsOnly, argBrowser, argYear, argCategory, argStatus, argLevel, argLiftable, NULL};
char *argLiftOver[] = {argFromGenome, argToGenome, argChrom, argStart, argEnd, argFilter, argMaxItemsOutput, argReturnTo, NULL};
char *argLiftRequest[] = {argFromGenome, argToGenome, argEmail, argComment, NULL};
char *argAssemblyRequest[] = {argAsmId, argName, argEmail, argBetterName, argComment, NULL};
char *argBlat[] = {argGenome, argHubUrl, argUserSeq, argFormat, argMaxItemsOutput, argJsonOutputArrays, argApiKey, NULL};
/* Global only to this one source file */
static struct cart *cart; /* CGI and other variables */
static struct hash *oldVars = NULL;
static struct hash *trackCounter = NULL;
static long totalTracks = 0;
static boolean allTrackSettings = FALSE; /* checkbox setting */
static char **shortLabels = NULL; /* public hub short labels in array */
static int publicHubCount = 0;
static char *defaultHub = "Synonymous Constraint";
static char *defaultDb = "ce11";
long enteredMainTime = 0; /* will become = clock1000() on entry */
/* to allow calculation of when to bail out, taking too long */
static long timeOutSeconds = 100;
static boolean timedOut = FALSE;
static char *urlPrefix = ""; /* initalized to support self references */
/* supportedTypes will be initialized to a known supported set */
struct slName *supportedTypes = NULL;
static void initSupportedTypes()
/* initalize the list of supported track types */
{
struct slName *el = newSlName("bed");
slAddHead(&supportedTypes, el);
el = newSlName("wig");
slAddHead(&supportedTypes, el);
el = newSlName("wigMaf");
slAddHead(&supportedTypes, el);
el = newSlName("broadPeak");
slAddHead(&supportedTypes, el);
el = newSlName("narrowPeak");
slAddHead(&supportedTypes, el);
el = newSlName("bigBed");
slAddHead(&supportedTypes, el);
el = newSlName("bigWig");
slAddHead(&supportedTypes, el);
el = newSlName("bigLolly");
slAddHead(&supportedTypes, el);
el = newSlName("bigNarrowPeak");
slAddHead(&supportedTypes, el);
el = newSlName("bigGenePred");
slAddHead(&supportedTypes, el);
el = newSlName("genePred");
slAddHead(&supportedTypes, el);
el = newSlName("psl");
slAddHead(&supportedTypes, el);
el = newSlName("rmsk");
slAddHead(&supportedTypes, el);
el = newSlName("bigRmsk");
slAddHead(&supportedTypes, el);
el = newSlName("bigPsl");
slAddHead(&supportedTypes, el);
el = newSlName("altGraphX");
slAddHead(&supportedTypes, el);
el = newSlName("barChart");
slAddHead(&supportedTypes, el);
el = newSlName("chain");
slAddHead(&supportedTypes, el);
el = newSlName("ctgPos");
slAddHead(&supportedTypes, el);
el = newSlName("expRatio");
slAddHead(&supportedTypes, el);
el = newSlName("factorSource");
slAddHead(&supportedTypes, el);
el = newSlName("gvf");
slAddHead(&supportedTypes, el);
el = newSlName("interact");
slAddHead(&supportedTypes, el);
el = newSlName("netAlign");
slAddHead(&supportedTypes, el);
el = newSlName("peptideMapping");
slAddHead(&supportedTypes, el);
el = newSlName("pgSnp");
slAddHead(&supportedTypes, el);
el = newSlName("bigBarChart");
slAddHead(&supportedTypes, el);
el = newSlName("bigInteract");
slAddHead(&supportedTypes, el);
el = newSlName("clonePos");
slAddHead(&supportedTypes, el);
el = newSlName("bigDbSnp");
slAddHead(&supportedTypes, el);
el = newSlName("bigMaf");
slAddHead(&supportedTypes, el);
el = newSlName("bigChain");
slAddHead(&supportedTypes, el);
slNameSort(&supportedTypes);
} /* static void initSupportedTypes() */
static int publicHubCmpCase(const void *va, const void *vb)
/* Compare two shortLabels, ignore case. */
{
const struct hubPublic *a = *((struct hubPublic **)va);
const struct hubPublic *b = *((struct hubPublic **)vb);
return strcasecmp(a->shortLabel, b->shortLabel);
}
static void publicHubSortCase(struct hubPublic **pList)
/* Sort slName list, ignore case. */
{
slSort(pList, publicHubCmpCase);
}
struct hubPublic *hubPublicDbLoadAll()
/* read entire hubPublic table in hgcentral and return resulting list */
{
char query[1024];
struct hubPublic *list = NULL;
struct sqlConnection *conn = hConnectCentral();
sqlSafef(query, sizeof(query), "select * from %s", hubPublicTableName());
struct sqlResult *sr = sqlGetResult(conn, query);
char **row;
while ((row = sqlNextRow(sr)) != NULL)
{
struct hubPublic *el = hubPublicLoad(row);
slAddHead(&list, el);
}
sqlFreeResult(&sr);
hDisconnectCentral(&conn);
publicHubSortCase(&list);
int listSize = slCount(list);
AllocArray(shortLabels, listSize);
struct hubPublic *el = list;
int i = 0;
for ( ; el != NULL; el = el->next )
{
shortLabels[i++] = el->shortLabel;
++publicHubCount;
}
return list;
}
static boolean timeOutReached()
/* see if the timeout has been reached to determine if an exit
* is appropriate at this time
*/
{
long nowTime = clock1000();
timedOut = FALSE;
if ((nowTime - enteredMainTime) > (1000 * timeOutSeconds))
timedOut= TRUE;
return timedOut;
}
static void hashCountTrack(struct trackDb *tdb, struct hash *countTracks)
/* this is counting up track types into the hash countTracks */
{
char *stripType = cloneString(tdb->type);
if (startsWith("chain ", tdb->type))
stripType = cloneString("chain");
else if (startsWith("netAlign ", tdb->type))
stripType = cloneString("netAlign");
else if (startsWith("genePred ", tdb->type))
stripType = cloneString("genePred");
else if (startsWith("bigWig ", tdb->type))
stripType = cloneString("bigWig");
else if (startsWith("wigMaf ", tdb->type))
stripType = cloneString("wigMaf");
else if (startsWith("wig ", tdb->type))
stripType = cloneString("wig");
else
stripType = cloneString(tdb->type);
boolean compositeContainer = tdbIsComposite(tdb);
boolean compositeView = tdbIsCompositeView(tdb);
boolean superChild = tdbIsSuperTrackChild(tdb);
if (compositeContainer)
hashIncInt(countTracks, "composite container");
else if (compositeView)
hashIncInt(countTracks, "composite view");
else if (superChild)
{
hashIncInt(countTracks, "superTrack child");
hashIncInt(countTracks, stripType);
hashIncInt(countTracks, "track count");
}
else if (isEmpty(tdb->type))
hashIncInt(countTracks, "no type specified");
else
{
hashIncInt(countTracks, stripType);
hashIncInt(countTracks, "track count");
}
freeMem(stripType);
}
static void sampleUrl(struct trackHub *hub, char *db, struct trackDb *tdb, char *errorString)
/* print out a sample getData URL */
{
char errorPrint[2048];
errorPrint[0] = 0;
if (isNotEmpty(errorString))
{
safef(errorPrint, sizeof(errorPrint), " ERROR: %s", errorString);
}
boolean superChild = tdbIsSuperTrackChild(tdb);
char *genome = NULL;
if (hub)
genome = hub->genomeList->name;
// tdb->track (and tdb->parent->track) carry the hub__ decoration;
// display and outgoing URLs should show/use the name as it actually
// appears in trackDb.txt. protectedTrack() below still needs the
// decorated tdb->track, untouched, to recognize a hub track.
char *trackName = trackHubSkipHubName(tdb->track);
char *parentName = tdb->parent ? trackHubSkipHubName(tdb->parent->track) : NULL;
struct dyString *extraDyFlags = dyStringNew(128);
if (debug)
dyStringAppend(extraDyFlags, ";debug=1");
if (jsonOutputArrays)
dyStringAppend(extraDyFlags, ";jsonOutputArrays=1");
char *extraFlags = dyStringCannibalize(&extraDyFlags);
if (protectedTrack(db, tdb, tdb->track))
hPrintf("%s : %s <protected data>\n", trackName, tdb->type);
else if (db)
{
if (hub)
{
char urlReference[2048];
safef(urlReference, sizeof(urlReference), " (sample data)%s\n", urlPrefix, hub->url, genome, trackName, extraFlags, errorPrint);
if (tdb->parent)
hPrintf("%s: %s subtrack of parent: %s%s\n", trackName, tdb->type, parentName, urlReference);
else
hPrintf("%s: %s%s\n", trackName, tdb->type, urlReference);
}
else
{
char urlReference[2048];
safef(urlReference, sizeof(urlReference), " (sample data)%s\n", urlPrefix, db, trackName, extraFlags, errorPrint);
if (superChild)
hPrintf("%s: %s superTrack child of parent: %s%s\n", trackName, tdb->type, parentName, urlReference);
else if (tdb->parent)
hPrintf("%s: %s subtrack of parent: %s%s\n", trackName, tdb->type, parentName, urlReference);
else
hPrintf("%s: %s%s\n", trackName, tdb->type, urlReference );
}
}
else if (hub)
{
char urlReference[2048];
safef(urlReference, sizeof(urlReference), " (sample data)%s\n", urlPrefix, hub->url, genome, trackName, extraFlags, errorPrint);
if (tdb->parent)
hPrintf("%s: %s subtrack of parent: %s%s\n", trackName, tdb->type, parentName, urlReference);
else
hPrintf("%s: %s%s\n", trackName, tdb->type, urlReference);
}
else
hPrintf("%s : %s not db hub track ?\n", trackName, tdb->type);
}
static void hubSampleUrl(struct trackHub *hub, char *db, struct trackDb *tdb,
long chromCount, long itemCount, char *genome, char *errorString)
{
struct dyString *extraDyFlags = dyStringNew(128);
if (debug)
dyStringAppend(extraDyFlags, ";debug=1");
if (jsonOutputArrays)
dyStringAppend(extraDyFlags, ";jsonOutputArrays=1");
char *extraFlags = dyStringCannibalize(&extraDyFlags);
char errorPrint[2048];
errorPrint[0] = 0;
if (isNotEmpty(errorString))
{
safef(errorPrint, sizeof(errorPrint), " : ERROR: %s", errorString);
}
char countsMessage[512];
countsMessage[0] = 0;
if (chromCount > 0 || itemCount > 0)
{
if (allowedBigBedType(tdb->type))
safef(countsMessage, sizeof(countsMessage), " : %ld chroms : %ld item count ", chromCount, itemCount);
else if (startsWithWord("bigWig", tdb->type))
safef(countsMessage, sizeof(countsMessage), " : %ld chroms : %ld bases covered ", chromCount, itemCount);
else
safef(countsMessage, sizeof(countsMessage), " : %ld chroms : %ld count ", chromCount, itemCount);
}
// display and outgoing URLs get the name as it appears in trackDb.txt;
// protectedTrack() below still needs the decorated tdb->track, untouched.
char *trackName = trackHubSkipHubName(tdb->track);
if (protectedTrack(db, tdb, tdb->track))
hPrintf(" %s: %s protected data\n", trackName, tdb->type);
else if (isSupportedType(tdb->type))
{
char urlReference[2048];
safef(urlReference, sizeof(urlReference), "(sample data)%s\n", urlPrefix, hub->url, genome, trackName, extraFlags, errorPrint);
if (allowedBigBedType(tdb->type))
hPrintf(" %s: %s%s%s\n", trackName, tdb->type, countsMessage, urlReference);
else if (startsWithWord("bigWig", tdb->type))
hPrintf(" %s: %s%s%s\n", trackName, tdb->type, countsMessage, urlReference);
else
hPrintf(" %s: %s%s%s\n", trackName, tdb->type, countsMessage, urlReference);
}
else
{
if (allowedBigBedType(tdb->type))
hPrintf(" %s: %s%s\n", trackName, tdb->type, countsMessage);
else if (startsWithWord("bigWig", tdb->type))
hPrintf(" %s: %s%s\n", trackName, tdb->type, countsMessage);
else
hPrintf(" %s: %s%s\n", trackName, tdb->type, countsMessage);
}
} /* static void hubSampleUrl(struct trackHub *hub, struct trackDb *tdb,
* long chromCount, long itemCount, char *genome)
*/
static void bbiLargestChrom(struct bbiChromInfo *chromList, char **chromName,
unsigned *chromSize)
/* find largest chromosome name and size in the chromList */
{
if (chromName && chromSize)
{
*chromSize = 0;
char *returnName = NULL;
struct bbiChromInfo *el;
for (el = chromList; el; el = el->next)
{
if (el->size > *chromSize)
{
*chromSize = el->size;
returnName = el->name;
}
}
if (chromSize > 0)
*chromName = cloneString(returnName);
}
}
static int bbiBriefMeasure(char *type, char *bigDataUrl, char *bigDataIndex, long *chromCount, long *itemCount, struct dyString *errors, char **chromName, unsigned *chromSize)
/* check a bigDataUrl to find chrom count and item count, return
* name of largest chrom and its size
*/
{
int retVal = 0;
*chromCount = 0;
*itemCount = 0;
if (isEmpty(bigDataUrl))
{
// a container/parent-level tdb (superTrack parent, view, etc. not
// already filtered out by the caller) has no bigDataUrl of its own --
// nothing to measure. Every bigFileOpen()/udcFileOpen() below assumes
// a non-NULL URL string and segfaults on NULL rather than erring out
// through the errCatch below, so this must be checked before any of it.
dyStringPrintf(errors, "no bigDataUrl to measure for type %s", type);
return 1;
}
struct errCatch *errCatch = errCatchNew();
if (errCatchStart(errCatch))
{
if (startsWithWord("bigNarrowPeak", type)
|| startsWithWord("bigBed", type)
|| startsWithWord("bigGenePred", type)
|| startsWithWord("bigPsl", type)
|| startsWithWord("bigDbSnp", type)
|| startsWithWord("bigMaf", type)
|| startsWithWord("bigChain", type)
|| startsWithWord("bigRmsk", type)
|| startsWithWord("bigBarChart", type)
|| startsWithWord("bigInteract", type))
{
struct bbiFile *bbi = NULL;
bbi = bigBedFileOpen(bigDataUrl);
struct bbiChromInfo *chromList = bbiChromList(bbi);
*chromCount = slCount(chromList);
*itemCount = bigBedItemCount(bbi);
bbiLargestChrom(chromList, chromName, chromSize);
bbiChromInfoFreeList(&chromList);
bbiFileClose(&bbi);
}
else if (startsWithWord("bigWig", type))
{
struct bbiFile *bwf = bigWigFileOpen(bigDataUrl);
struct bbiChromInfo *chromList = bbiChromList(bwf);
struct bbiSummaryElement sum = bbiTotalSummary(bwf);
*chromCount = slCount(chromList);
*itemCount = sum.validCount;
bbiLargestChrom(chromList, chromName, chromSize);
bbiChromInfoFreeList(&chromList);
bbiFileClose(&bwf);
}
else if (startsWithWord("vcfTabix", type))
{
struct vcfFile *vcf = vcfTabixFileAndIndexMayOpen(bigDataUrl, bigDataIndex, NULL, 0, 0, 1, 1);
if (vcf == NULL)
{
dyStringPrintf(errors, "Could not open %s and/or its tabix index (.tbi) file. See http://genome.ucsc.edu/goldenPath/help/vcf.html", bigDataUrl);
retVal = 1;
}
else
vcfFileFree(&vcf);
}
else if (startsWithWord("bam", type))
{
bamFileAndIndexMustExist(bigDataUrl, bigDataIndex);
}
else if (startsWithWord("longTabix", type))
{
struct bedTabixFile *btf = bedTabixFileMayOpen(bigDataUrl, NULL, 0, 0);
if (btf == NULL)
{
dyStringPrintf(errors, "Couldn't open %s and/or its tabix index (.tbi) file.", bigDataUrl);
retVal = 1;
}
else
bedTabixFileClose(&btf);
}
#ifdef USE_HAL
else if (startsWithWord("halSnake", type))
{
char *errString;
int handle = halOpenLOD(bigDataUrl, &errString);
if (handle < 0)
{
dyStringPrintf(errors, "HAL open error: %s", errString);
retVal = 1;
}
if (halClose(handle, &errString) < 0)
{
dyStringPrintf(errors, "HAL close error: %s", errString);
retVal = 1;
}
}
#endif
else
{
dyStringPrintf(errors, "unrecognized type %s", type);
retVal = 1;
}
}
errCatchEnd(errCatch);
if (errCatch->gotError)
{
retVal = 1;
dyStringPrintf(errors, "%s", errCatch->message->string);
}
errCatchFree(&errCatch);
return retVal;
} /* static int bbiBriefMeasure() */
static void hubSubTracks(struct trackHub *hub, char *db, struct trackDb *tdb,
struct hash *countTracks, long chromCount, long itemCount,
char *chromName, unsigned chromSize, char *genome, char *errorString)
/* tdb has subtracks, show only subTracks, no details, this is RECURSIVE */
{
hPrintf(" \n");
if (debug)
{
hPrintf(" - subtracks for '%s' db: '%s'
\n", trackHubSkipHubName(tdb->track), db);
hPrintf(" - chrom: '%s' size: %u
\n", chromName, chromSize);
}
if (tdb->subtracks)
{
struct trackDb *tdbEl = NULL;
for (tdbEl = tdb->subtracks; tdbEl; tdbEl = tdbEl->next)
{
// display name as it appears in trackDb.txt; tdbEl->track itself
// stays decorated for hashCountTrack()/recursive calls below
char *tdbElName = trackHubSkipHubName(tdbEl->track);
char *tdbElParentName = tdbEl->parent ? trackHubSkipHubName(tdbEl->parent->track) : NULL;
// trackHasData() also excludes tdbIsContainer() (e.g. 'container
// multiWig'), which has no bigDataUrl of its own -- its data comes
// from its children.
if (trackHasData(tdbEl))
{
char *bigDataIndex = NULL;
char *relIdxUrl = trackDbSetting(tdbEl, "bigDataIndex");
if (relIdxUrl != NULL)
bigDataIndex = trackHubRelativeUrl(hub->genomeList->trackDbFile, relIdxUrl);
char *bigDataUrl = trackDbSetting(tdbEl, "bigDataUrl");
char *longName = NULL;
unsigned longSize = 0;
struct dyString *errors = dyStringNew(1024);
(void) bbiBriefMeasure(tdbEl->type, bigDataUrl, bigDataIndex, &chromCount, &itemCount, errors, &longName, &longSize);
chromSize = longSize;
chromName = longName;
}
if (tdbIsCompositeView(tdbEl))
hPrintf("- %s: %s : composite view of parent: %s
\n", tdbElName, tdbEl->type, tdbElParentName);
else
{
if (isSupportedType(tdbEl->type))
hubSampleUrl(hub, db, tdbEl, chromCount, itemCount, genome, errorString);
else
hPrintf("- %s: %s : subtrack of parent: %s
\n", tdbElName, tdbEl->type, tdbElParentName);
}
hashCountTrack(tdbEl, countTracks);
if (tdbEl->subtracks)
hubSubTracks(hub, db, tdbEl, countTracks, chromCount, itemCount, chromName, chromSize, genome, errorString);
}
}
hPrintf("
\n");
} /* hubSubTracks() */
static void showSubTracks(struct trackHub *hub, char *db, struct trackDb *tdb, struct hash *countTracks,
char *chromName, unsigned chromSize, char *errorString)
/* tdb has subtracks, show only subTracks, no details */
{
hPrintf(" \n");
if (debug)
hPrintf(" - subtracks for '%s' db: '%s'
\n", tdb->track, db);
if (tdb->subtracks)
{
struct dyString *extraDyFlags = dyStringNew(128);
if (debug)
dyStringAppend(extraDyFlags, ";debug=1");
if (jsonOutputArrays)
dyStringAppend(extraDyFlags, ";jsonOutputArrays=1");
char *extraFlags = dyStringCannibalize(&extraDyFlags);
struct trackDb *tdbEl = NULL;
for (tdbEl = tdb->subtracks; tdbEl; tdbEl = tdbEl->next)
{
if (tdbIsCompositeView(tdbEl))
hPrintf("- %s: %s : composite view of parent: %s
\n", tdbEl->track, tdbEl->type, tdbEl->parent->track);
else
{
if (isSupportedType(tdbEl->type))
sampleUrl(hub, db, tdbEl, errorString);
else
if (hub && hub->url)
hPrintf("- %s: %s : subtrack of parent: %s, genome %s, url %s
\n", tdbEl->track, tdbEl->type, tdbEl->parent->track, db, hub->url);
else
{
char urlReference[2048];
safef(urlReference, sizeof(urlReference), " (sample data)\n", urlPrefix, db, tdbEl->track, extraFlags);
hPrintf("- %s: %s : subtrack of parent: %s%s
\n", tdbEl->track, tdbEl->type, tdbEl->parent->track, urlReference);
}
}
hashCountTrack(tdbEl, countTracks);
if (tdbEl->subtracks)
showSubTracks(hub, db, tdbEl, countTracks, chromName, chromSize, errorString);
}
}
hPrintf("
\n");
}
static void trackSettings(char *db, struct trackDb *tdb, struct hash *countTracks)
/* process the settingsHash for a trackDb, recursive when subtracks */
{
hPrintf(" \n");
boolean protectedData = protectedTrack(db, tdb, tdb->track);
struct hashEl *hel;
struct hashCookie hc = hashFirst(tdb->settingsHash);
while ((hel = hashNext(&hc)) != NULL)
{
if (sameWord("track", hel->name))
continue; // already output in header
if (sameWord("tableBrowser", hel->name)
&& startsWithWord("off", (char*)hel->val))
hPrintf(" - protectedData: 'true'
\n");
else if (protectedData && sameWord("bigDataUrl", hel->name))
hPrintf(" - bigDataUrl: <protected data>
\n");
else if (isEmpty((char *)hel->val))
hPrintf(" - %s: <empty>
\n", hel->name);
else
hPrintf(" - %s: '%s'
\n", hel->name, (char *)hel->val);
}
if (tdb->subtracks)
{
struct trackDb *tdbEl = NULL;
if (debug)
hPrintf(" - has %d subtrack(s)
\n", slCount(tdb->subtracks));
for (tdbEl = tdb->subtracks; tdbEl; tdbEl = tdbEl->next)
{
hPrintf("- subtrack: %s of parent: %s : type: '%s' (TBD: sample data)
\n", trackHubSkipHubName(tdbEl->track), trackHubSkipHubName(tdbEl->parent->track), tdbEl->type);
hashCountTrack(tdbEl, countTracks);
trackSettings(db, tdbEl, countTracks);
}
}
hPrintf("
\n");
}
static void hubCountOneTdb(struct trackHub *hub, char *db, struct trackDb *tdb,
char *bigDataIndex, struct hash *countTracks, char *chromName,
unsigned chromSize, char *genome)
{
char *bigDataUrl = trackDbSetting(tdb, "bigDataUrl");
boolean compositeContainer = tdbIsComposite(tdb);
boolean compositeView = tdbIsCompositeView(tdb);
boolean superChild = tdbIsSuperTrackChild(tdb);
boolean depthSearch = cartUsualBoolean(cart, "depthSearch", FALSE);
hashCountTrack(tdb, countTracks);
long chromCount = 0;
long itemCount = 0;
struct dyString *errors = dyStringNew(1024);
/* if given a chromSize, it belongs to a UCSC db and this is *not* an
* assembly hub, otherwise, look up a chrom and size in the bbi file
* trackHasData() also excludes tdbIsContainer() (e.g. 'container multiWig'),
* which has no bigDataUrl of its own -- its data comes from its children.
*/
if (trackHasData(tdb))
{
if (chromSize < 1 || depthSearch)
{
char *longName = NULL;
unsigned longSize = 0;
(void) bbiBriefMeasure(tdb->type, bigDataUrl, bigDataIndex, &chromCount, &itemCount, errors, &longName, &longSize);
chromSize = longSize;
chromName = longName;
}
}
// display name as it appears in trackDb.txt; tdb->track itself stays
// decorated for hashCountTrack()/protectedTrack()/recursive calls
char *trackName = trackHubSkipHubName(tdb->track);
char *parentName = tdb->parent ? trackHubSkipHubName(tdb->parent->track) : NULL;
if (depthSearch && bigDataUrl)
{
if (isSupportedType(tdb->type))
hubSampleUrl(hub, db, tdb, chromCount, itemCount, genome, errors->string);
}
else
{
if (compositeContainer)
hPrintf(" %s: %s : composite track container has %d subtracks\n", trackName, tdb->type, slCount(tdb->subtracks));
else if (compositeView)
hPrintf(" %s: %s : composite view of parent: %s\n", trackName, tdb->type, parentName);
else if (superChild)
{
if (isSupportedType(tdb->type))
hubSampleUrl(hub, db, tdb, chromCount, itemCount, genome, errors->string);
else
hPrintf(" %s: %s : superTrack child of parent: %s\n", trackName, tdb->type, parentName);
}
else if (! depthSearch && bigDataUrl)
{
if (isSupportedType(tdb->type))
{
hubSampleUrl(hub, db, tdb, chromCount, itemCount, genome, errors->string);
}
}
else
{
if (isSupportedType(tdb->type))
{
hubSampleUrl(hub, db, tdb, chromCount, itemCount, genome, errors->string);
}
else
hPrintf(" %s: %s (what is this)\n", trackName, tdb->type);
}
}
if (allTrackSettings)
{
hPrintf(" \n");
trackSettings(db, tdb, countTracks); /* show all settings */
hPrintf("
\n");
}
else if (tdb->subtracks)
{
hubSubTracks(hub, db, tdb, countTracks, chromCount, itemCount, chromName, chromSize, genome, errors->string);
}
return;
} /* static void hubCountOneTdb(char *db, struct trackDb *tdb,
* char *bigDataIndex, struct hash *countTracks,
* char *chromName, unsigned chromSize)
*/
static void countOneTdb(char *db, struct trackDb *tdb,
struct hash *countTracks, char *chromName, unsigned chromSize,
char *errorString)
/* for this tdb in this db, count it up and provide a sample */
{
char *bigDataUrl = trackDbSetting(tdb, "bigDataUrl");
boolean compositeContainer = tdbIsComposite(tdb);
boolean compositeView = tdbIsCompositeView(tdb);
boolean superChild = tdbIsSuperTrackChild(tdb);
boolean depthSearch = cartUsualBoolean(cart, "depthSearch", FALSE);
boolean protectedData = protectedTrack(db, tdb, tdb->track);
hashCountTrack(tdb, countTracks);
if (compositeContainer)
hPrintf(" %s: %s : composite track container has %d subtracks\n", tdb->track, tdb->type, slCount(tdb->subtracks));
else if (compositeView)
hPrintf(" %s: %s : composite view of parent: %s\n", tdb->track, tdb->type, tdb->parent->track);
else if (superChild)
{
if (isSupportedType(tdb->type))
sampleUrl(NULL, db, tdb, errorString);
else
hPrintf(" %s: %s : superTrack child of parent: %s\n", tdb->track, tdb->type, tdb->parent->track);
}
else if (! depthSearch && bigDataUrl)
{
if (protectedData)
hPrintf(" %s: %s : <protected data>\n", tdb->track, tdb->type);
else
hPrintf(" %s: %s : %s\n", tdb->track, tdb->type, bigDataUrl);
}
else
{
if (isSupportedType(tdb->type))
sampleUrl(NULL, db, tdb, errorString);
else
hPrintf(" %s: %s\n", tdb->track, tdb->type);
}
if (allTrackSettings)
{
hPrintf(" \n");
trackSettings(db, tdb, countTracks); /* show all settings */
hPrintf("
\n");
}
else if (tdb->subtracks)
{
showSubTracks(NULL, db, tdb, countTracks, chromName, chromSize, NULL);
}
return;
} /* static void countOneTdb(char *db, struct trackDb *tdb,
* struct hash *countTracks, char *chromName,
* unsigned chromSize)
*/
static unsigned largestChrom(char *db, char **nameReturn, int *chromCount)
/* return the length and get the chrom name for the largest chrom
* from chromInfo table. For use is sample getData URLs
*/
{
char query[1024];
struct sqlConnection *conn = hAllocConn(db);
sqlSafef(query, sizeof(query), "select chrom,size from chromInfo order by size desc limit 1");
struct sqlResult *sr = sqlGetResult(conn, query);
char **row = sqlNextRow(sr);
unsigned length = 0;
if (row)
{
*nameReturn = cloneString(row[0]);
length = sqlLongLong(row[1]);
}
sqlFreeResult(&sr);
if (chromCount)
{
sqlSafef(query, sizeof(query), "select count(*) from chromInfo");
*chromCount = sqlQuickNum(conn, query);
}
hFreeConn(&conn);
return length;
}
static void hubTrackList(struct trackHub *hub, struct trackDb *topTrackDb, struct trackHubGenome *genome)
/* process the track list in a hub to show all tracks */
{
hPrintf(" \n");
if (topTrackDb)
{
struct hash *countTracks = hashNew(0);
struct trackDb *tdb = NULL;
for ( tdb = topTrackDb; tdb; tdb = tdb->next )
{
char *bigDataIndex = NULL;
char *relIdxUrl = trackDbSetting(topTrackDb, "bigDataIndex");
if (relIdxUrl != NULL)
bigDataIndex = trackHubRelativeUrl(genome->trackDbFile, relIdxUrl);
char *defaultGenome = NULL;
if (isNotEmpty(genome->name))
// genome->name is hub__ decorated for assembly hub genomes
// (anything with twoBitPath); strip it here so defaultGenome
// matches what every other consumer (list.c) passes around --
// otherwise protectedTrack() re-decorates an already-decorated
// name to hub__hub__ and hAllocConn() on that
// aborts with "Unknown database"
defaultGenome = trackHubSkipHubName(genome->name);
char *chromName = NULL;
unsigned chromSize = 0;
int chromCount = 0;
if (isEmpty(genome->twoBitPath))
chromSize = largestChrom(defaultGenome, &chromName, &chromCount);
hubCountOneTdb(hub, defaultGenome, tdb, bigDataIndex, countTracks, chromName, chromSize, defaultGenome);
if (timeOutReached())
break;
} /* for ( tdb = topTrackDb; tdb; tdb = tdb->next ) */
hPrintf(" - %d different track types
\n",countTracks->elCount - 1);
/* add this single genome count to the overall multi-genome counts */
if (countTracks->elCount)
{
hPrintf(" \n");
struct hashEl *hel, *helList = hashElListHash(countTracks);
slSort(&helList, hashElCmpIntValDesc);
for (hel = helList; hel; hel = hel->next)
{
if (sameOk("track count", hel->name))
continue;
int prevCount = ptToInt(hashFindVal(trackCounter, hel->name));
if (differentStringNullOk("track count", hel->name))
totalTracks += ptToInt(hel->val);
hashReplace(trackCounter, hel->name, intToPt(prevCount + ptToInt(hel->val)));
if (isSupportedType(hel->name))
hPrintf(" - %d - %s - supported
\n", ptToInt(hel->val), hel->name);
else
hPrintf(" - %d - %s - not supported
\n", ptToInt(hel->val), hel->name);
}
hPrintf("
\n");
}
}
else
hPrintf(" - no trackTopDb ?
\n");
hPrintf("
\n");
} /* static struct trackDb *hubTrackList() */
static void hubAssemblySettings(struct trackHub *hub, struct trackHubGenome *genome)
/* display all the assembly 'settingsHash' */
{
struct trackDb *tdb = obtainTdb(genome, NULL);
hPrintf(" \n");
struct hashEl *hel;
struct hashCookie hc = hashFirst(genome->settingsHash);
while ((hel = hashNext(&hc)) != NULL)
{
if (sameWord("description", hel->name) ||
sameWord("defaultPos", hel->name) ||
sameWord("organism", hel->name) ||
sameWord("groups", hel->name) ||
sameWord("twoBitPath", hel->name) ||
sameWord("genome", hel->name)
)
continue; // already output in header
if (sameWord("trackDb", hel->name)) /* examine the trackDb structure */
hubTrackList(hub, tdb, genome);
else
hPrintf(" - %s: %s
\n", hel->name, (char *)hel->val);
if (timeOutReached())
break;
}
hPrintf("
\n");
}
static unsigned largestChromInfo(struct chromInfo *ci, char **chromName)
/* find largest chrom in this chromInfo, return name and size */
{
unsigned size = 0;
char *name = NULL;
struct chromInfo *el;
for (el = ci; el; el = el->next)
{
if (el->size > size)
{
size = el->size;
name = el->chrom;
}
}
if (chromName)
*chromName = name;
return size;
}
static void hubInfo(char *tag, char *val)
/* print one list element with the given tag and value, show
* if not value present
*/
{
if (isNotEmpty(val))
hPrintf("%s: '%s'\n", tag, val);
else
hPrintf("%s: <empty>\n", tag);
}
static void genomeList(struct trackHub *hubTop)
/* follow the pointers from the trackHub to trackHubGenome and around
* in a circle from one to the other to find all hub resources
*/
{
long totalAssemblyCount = 0;
struct trackHubGenome *genome = hubTop->genomeList;
hPrintf("\n", urlPrefix, hubTop->url);
if (NULL == genome)
{
hPrintf("odd error, can not find a genomeList ? at url: '%s'
\n", hubTop->url);
return;
}
hPrintf("\n");
long lastTime = clock1000();
for ( ; genome; genome = genome->next )
{
++totalAssemblyCount;
char urlReference[2048];
// genome->name and genome->organism carry the hub__ decoration for
// assembly hub genomes (trackHub.c addHubName()); display and outgoing
// links should show/use the name as it actually appears in genomes.txt.
// trackHubAllChromInfo()/trackHubGetGenome() lookups still need the
// decorated genome->name, untouched, to find the genome in hubAssemblyHash.
char *displayGenome = trackHubSkipHubName(genome->name);
char *displayOrganism = trackHubSkipHubName(genome->organism);
if (isNotEmpty(genome->twoBitPath))
{
hPrintf("- Assembly genome '%s' twoBitPath: '%s'
\n", displayGenome, genome->twoBitPath);
char *chromName = NULL;
struct chromInfo *ci = trackHubAllChromInfo(genome->name);
unsigned chromSize = largestChromInfo(ci, &chromName);
char sizeString[64];
sprintLongWithCommas(sizeString, chromSize);
hPrintf("- Sequence count %d, largest: %s at %s bases
\n", slCount(ci), chromName, sizeString);
safef(urlReference, sizeof(urlReference), " JSON example sequence output: %s:%u-%u", urlPrefix, hubTop->url, displayGenome, chromName, chromSize/4, (chromSize/4)+128, chromName, chromSize/4, (chromSize/4)+128);
hPrintf("- %s
\n", urlReference);
}
safef(urlReference, sizeof(urlReference), " JSON example list tracks output", urlPrefix, hubTop->url, displayGenome, trackLeavesOnly ? ";trackLeavesOnly=1" : "");
hPrintf("- %s
\n", urlReference);
hubInfo("organism", displayOrganism);
hubInfo("name", displayGenome);
hubInfo("description", genome->description);
hubInfo("groups", genome->groups);
hubInfo("defaultPos", genome->defaultPos);
hubInfo("trackDbFile", genome->trackDbFile);
hubAssemblySettings(hubTop, genome);
struct trackDb *tdbList = obtainTdb(genome, NULL);
hubTrackList(hubTop, tdbList, genome);
if (measureTiming)
{
long thisTime = clock1000();
hPrintf("- processing time %s: %ld millis
\n", genome->name, thisTime - lastTime);
hPrintf("
\n");
}
if (timeOutReached())
break;
}
if (trackCounter->elCount)
{
hPrintf(" - total genome assembly count: %ld
\n", totalAssemblyCount);
hPrintf(" - %ld total tracks counted, %d different track types:
\n", totalTracks, trackCounter->elCount);
hPrintf(" \n");
struct hashEl *hel, *helList = hashElListHash(trackCounter);
slSort(&helList, hashElCmpIntValDesc);
for (hel = helList; hel; hel = hel->next)
{
hPrintf(" - %d - %s - total
\n", ptToInt(hel->val), hel->name);
}
hPrintf("
\n");
}
hPrintf("
\n");
} /* static void genomeList (hubTop) */
static char *urlFromShortLabel(char *shortLabel)
/* this is not a fair way to get the URL since shortLabel's are not
* necessarily unique. This is temporary. TBD: need to always use URL
* and then get the shortLabel
*/
{
char hubUrl[1024];
char query[1024];
struct sqlConnection *conn = hConnectCentral();
// Build a query to select the hubUrl for the given shortLabel
sqlSafef(query, sizeof(query), "select hubUrl from %s where shortLabel='%s'",
hubPublicTableName(), shortLabel);
if (! sqlQuickQuery(conn, query, hubUrl, sizeof(hubUrl)))
hubUrl[0] = 0;
hDisconnectCentral(&conn);
return cloneString(hubUrl);
}
static struct hash *apiFunctionHash = NULL;
static void setupFunctionHash()
/* initialize the apiFunctionHash */
{
if (apiFunctionHash)
return; /* already done */
apiFunctionHash = hashNew(0);
hashAdd(apiFunctionHash, "list", &apiList);
hashAdd(apiFunctionHash, "getData", &apiGetData);
hashAdd(apiFunctionHash, "search", &apiSearch);
hashAdd(apiFunctionHash, "findGenome", &apiFindGenome);
hashAdd(apiFunctionHash, "liftOver", &apiLiftOver);
hashAdd(apiFunctionHash, "liftRequest", &apiLiftRequest);
hashAdd(apiFunctionHash, "assemblyRequest", &apiAssemblyRequest);
hashAdd(apiFunctionHash, "submitOttoRequest", &apiSubmitOttoRequest);
hashAdd(apiFunctionHash, "blat", &apiBlat);
}
static struct hashEl *parsePathInfo(char *pathInfo, char *words[MAX_PATH_INFO])
/* given a pathInfo string: /command/subCommand/etc...
* parse that and return a function pointer and the parsed words
* Returns NULL if not recognized
*/
{
char *tmp = cloneString(pathInfo);
/* skip the first leading slash to simplify chopByChar parsing */
tmp += 1;
int wordCount = chopByChar(tmp, '/', words, MAX_PATH_INFO);
if (wordCount < 1 || wordCount > 2)
return NULL; /* only 2 words allowed */
struct hashEl *hel = hashLookup(apiFunctionHash, words[0]);
return hel;
}
static void tracksForUcscDb(char *db)
/* scan the specified database for all tracks */
{
struct hash *countTracks = hashNew(0);
char *chromName = NULL;
int chromCount = 0;
unsigned chromSize = largestChrom(db, &chromName, &chromCount);
char countString[64];
sprintLongWithCommas(countString, chromCount);
char sizeString[64];
sprintLongWithCommas(sizeString, chromSize);
hPrintf("Tracks in UCSC genome: '%s', chrom count: %s, longest chrom: %s : %s
\n", db, countString, chromName, sizeString);
char urlReference[2048];
safef(urlReference, sizeof(urlReference), " JSON output: list tracks", urlPrefix, db, trackLeavesOnly ? ";trackLeavesOnly=1" : "");
hPrintf("%s
\n", urlReference);
struct trackDb *tdbList = obtainTdb(NULL, db);
struct trackDb *tdb;
hPrintf("\n");
for (tdb = tdbList; tdb != NULL; tdb = tdb->next )
{
countOneTdb(db, tdb, countTracks, chromName, chromSize, NULL);
if (timeOutReached())
break;
}
int trackCount = ptToInt(hashFindVal(countTracks, "track count"));
/* elCount - 1 since the 'track count' element isn't a track */
hPrintf(" - %d total tracks counted, %d different track types
\n", trackCount, countTracks->elCount - 1);
if (countTracks->elCount)
{
hPrintf(" \n");
struct hashEl *hel, *helList = hashElListHash(countTracks);
slSort(&helList, hashElCmpIntValDesc);
for (hel = helList; hel; hel = hel->next)
{
if (sameOk("track count", hel->name))
continue;
if (isSupportedType(hel->name))
hPrintf(" - %d - %s - supported
\n", ptToInt(hel->val), hel->name);
else
hPrintf(" - %d - %s - not supported
\n", ptToInt(hel->val), hel->name);
}
hPrintf("
\n");
}
hPrintf("
\n");
hPrintf("
\n");
} // static void tracksForUcscDb(char * db)
static void initUrlPrefix()
/* set up urlPrefix for self referenes */
{
char *httpHost = getenv("HTTP_HOST");
if (isEmpty(httpHost))
urlPrefix = "";
else
{
if (! startsWith("hgwdev-api", httpHost))
{
if (startsWith("hgwdev",httpHost) || startsWith("genome-test", httpHost))
{
urlPrefix = "../cgi-bin/hubApi";
}
}
}
}
static void showCartDump()
/* for information purposes only during development, will become obsolete */
{
hPrintf("cart dump
");
hPrintf("\n");
cartDump(cart);
hPrintf("\n");
}
static void sendJsonHogMessage(char *hogHost)
{
apiErrAbort(err429, err429Msg, "Your host, %s, has been sending too many requests lately and is "
"unfairly loading our site, impacting performance for other users. "
"Please contact genome-www@soe.ucsc.edu to ask that your site "
"be reenabled. Also, please consider downloading sequence and/or "
"annotations in bulk -- see http://genome.ucsc.edu/downloads.html.",
hogHost);
}
static void sendHogMessage(char *hogHost)
{
+cspWriteResponseHeader();
puts("Content-Type:text/html");
hPrintf("Status: %d %s\n", err429, err429Msg);
puts("Retry-After: 30");
puts("\n");
hPrintf("\n");
hPrintf("\n");
hPrintf("\n");
hPrintf("\n");
hPrintf("Status %d %s\n", err429, err429Msg);
hPrintf("Status %d %s
\n", err429, err429Msg);
hPrintf("Your host, %s, has been sending too many requests lately and is "
"unfairly loading our site, impacting performance for other users. "
"Please contact genome-www@soe.ucsc.edu to ask that your site "
"be reenabled. Also, please consider downloading sequence and/or "
"annotations in bulk -- see http://genome.ucsc.edu/downloads.html.",
hogHost);
hPrintf("
\n");
cgiExitTime("hubApi hogExit", enteredMainTime);
exit(0);
}
static void hogExit()
/* bottleneck server requests exit */
{
char *hogHost = getenv("REMOTE_ADDR");
char *pathInfo = getenv("PATH_INFO");
/* nothing on incoming path, then display the WEB page instead */
if (sameOk("/",pathInfo))
pathInfo = NULL;
if (isNotEmpty(pathInfo))
{
sendJsonHogMessage(hogHost);
}
else
{
sendHogMessage(hogHost);
}
} /* static void hogExit() */
/* name of radio button group */
#define RADIO_GROUP "selectRadio"
/* button functions */
#define RADIO_PUBHUB "pubHub"
#define RADIO_OTHERHUB "otherHub"
#define RADIO_UCSCDB "ucscDb"
static void selectionForm()
/* setup the selection pull-downs for source */
{
char *hubDropDown = cartUsualString(cart, "publicHubs", defaultHub);
char *urlDropDown = urlFromShortLabel(hubDropDown);
char *otherHubUrl = cartUsualString(cart, "urlHub", "");
char *ucscDb = cartUsualString(cart, "ucscGenome", defaultDb);
if (isEmpty(otherHubUrl))
otherHubUrl = urlDropDown;
char *radioOn = cartUsualString(cart, RADIO_GROUP, RADIO_PUBHUB);
/* create border around table, but not inside the table with the data */
hPrintf("\n");
hPrintf("\n");
hPrintf("| Select one of these three sources, and display options: | \n");
int maxDbNameWidth = 0;
struct dbDb *dbList = ucscDbDb();
char **ucscDbList = NULL;
int listSize = slCount(dbList);
AllocArray(ucscDbList, listSize);
struct dbDb *el = dbList;
int ucscDataBaseCount = 0;
for ( ; el != NULL; el = el->next )
{
ucscDbList[ucscDataBaseCount++] = el->name;
if (strlen(el->name) > maxDbNameWidth)
maxDbNameWidth = strlen(el->name);
}
maxDbNameWidth += 1;
hPrintf("\n");
hPrintf("| (example JSON list output: Public hubs, and UCSC database genomes) | \n");
hPrintf(" \n");
hPrintf(" |
\n");
/* how does debug carry forward ? */
// if (debug)
// cgiMakeHiddenVar("debug", "1");
}
static void apiRequest(char *pathInfo)
{
hPrintDisable();
/*expect no more than MAX_PATH_INFO number of words*/
char *words[MAX_PATH_INFO];
/* can immediately verify valid parameters right here right now */
char *start = cgiOptionalString("start");
char *end = cgiOptionalString("end");
char *db = cgiOptionalString("genome");
char *hubUrl = cgiOptionalString("hubUrl");
struct dyString *errorMsg = dyStringNew(128);
// first check for curated hubs
if (isEmpty(hubUrl) && isNotEmpty(db))
{
char *newHubUrl;
if (hubConnectGetCuratedUrl(db, &newHubUrl))
{
hubUrl = newHubUrl; // use curated hub hubUrl
cgiVarSet("hubUrl", hubUrl); // subsequent code grabs hubUrl from env
}
if (startsWith("hub_", db))
{
unsigned hubId = hubIdFromTrackName(db);
struct hubConnectStatus *hub = hubFromId(hubId);
if (hub)
{
// change up the arguments so it appears an assembly hub had been requested correctly
db = trackHubSkipHubName(db);
cgiVarSet("genome", db);
hubUrl = hub->hubUrl;
cgiVarSet("hubUrl", hubUrl);
}
}
}
if (isEmpty(hubUrl) && isNotEmpty(db))
{
if ( ! isGenArk(db) )
{
struct sqlConnection *conn = hAllocConnMaybe(db);
if (NULL == conn)
{
/* not found directly, try alias lookup before failing */
char *origDb = db;
char *aliasDb = asmAliasFind(db);
if (differentString(aliasDb, db))
{
db = aliasDb;
cgiVarSet("genome", db);
cgiVarSet("genomeInput", origDb); /* preserve original for error messages */
/* re-validate with the resolved name */
if ( ! isGenArk(db) )
{
conn = hAllocConnMaybe(db);
if (NULL == conn)
dyStringPrintf(errorMsg, "can not find genome='%s' for endpoint '%s'", origDb, pathInfo);
else
hFreeConn(&conn);
}
}
else
dyStringPrintf(errorMsg, "can not find genome='%s' for endpoint '%s'", db, pathInfo);
}
else
hFreeConn(&conn);
}
}
if (isNotEmpty(start) || isNotEmpty(end))
{
long long llStart = -1;
long long llEnd = -1;
struct errCatch *errCatch = errCatchNew();
if (errCatchStart(errCatch))
{
if (isNotEmpty(start))
llStart = sqlLongLong(start);
if (isNotEmpty(end))
llEnd = sqlLongLong(end);
}
errCatchEnd(errCatch);
if (errCatch->gotError)
{
if (isNotEmpty(errorMsg->string))
dyStringPrintf(errorMsg, ", ");
dyStringPrintf(errorMsg, "%s", errCatch->message->string);
if (isNotEmpty(start) && (-1 == llStart))
dyStringPrintf(errorMsg, ", can not recognize start coordinate: '%s'", start);
if (isNotEmpty(end) && (-1 == llEnd))
dyStringPrintf(errorMsg, ", can not recognize end coordinate: '%s'", end);
}
else
{
if ( (llStart < 0) || (llEnd < 0) || (llEnd <= llStart) )
{
if (isNotEmpty(errorMsg->string))
dyStringPrintf(errorMsg, ", ");
dyStringPrintf(errorMsg, "illegal start,end coordinates given: %s,%s, 'end' must be greater than 'start', and start greater than or equal to zero", start, end);
}
}
errCatchFree(&errCatch);
}
if (isNotEmpty(errorMsg->string))
apiErrAbort(err400, err400Msg, "%s", errorMsg->string);
setupFunctionHash();
struct hashEl *hel = parsePathInfo(pathInfo, words);
/* verify valid API command */
if (hel) /* have valid command */
{
hPrintDisable();
void (*apiFunction)(char **) = hel->val;
(*apiFunction)(words);
return;
}
else
apiErrAbort(err400, err400Msg, "no such command: '/%s", pathInfo);
/* due to Apache rewrite rules, will never be called with this error */
} /* static void apiRequest(char *pathInfo) */
static void doMiddle(struct cart *theCart)
/* Set up globals and make web page */
{
cart = theCart;
// measureTiming = isNotEmpty(cartOptionalString(cart, "measureTiming"));
char *database = NULL;
char *genome = NULL;
if (measureTiming)
startProcessTiming();
cgiVarSet("ignoreCookie", "1");
getDbAndGenome(cart, &database, &genome, oldVars);
initGenbankTableNames(database);
initUrlPrefix();
trackLeavesOnly = cartUsualBoolean(cart, "trackLeavesOnly", trackLeavesOnly);
jsonOutputArrays = cartUsualBoolean(cart, "jsonOutputArrays", jsonOutputArrays);
char *pathInfo = getenv("PATH_INFO");
/* nothing on incoming path, then display the WEB page instead */
if (sameOk("/",pathInfo))
pathInfo = NULL;
(void) hubPublicDbLoadAll();
webStartJWest(cart, database, "Genome Browser API");
// webStartGbNoBanner(cart, database, "UCSC JSON API interface");
// webStartGbOptionalBanner(cart, database, "UCSC JSON API interface", TRUE, FALSE);
hPrintf("\n");
/* these style mentions need to go into custom css file */
hPrintf("
\n");
if (debug)
{
hPrintf("
\n");
hPrintf("- hgBotDelay: %d
\n", botDelay);
char *envVar = getenv("BROWSER_HOST");
hPrintf("- BROWSER_HOST:%s
\n", envVar);
envVar = getenv("CONTEXT_DOCUMENT_ROOT");
hPrintf("- CONTEXT_DOCUMENT_ROOT:%s
\n", envVar);
envVar = getenv("CONTEXT_PREFIX");
hPrintf("- CONTEXT_PREFIX:%s
\n", envVar);
envVar = getenv("DOCUMENT_ROOT");
hPrintf("- DOCUMENT_ROOT:%s
\n", envVar);
envVar = getenv("HTTP_HOST");
hPrintf("- HTTP_HOST:%s
\n", envVar);
envVar = getenv("REQUEST_URI");
hPrintf("- REQUEST_URI:%s
\n", envVar);
envVar = getenv("SCRIPT_FILENAME");
hPrintf("- SCRIPT_FILENAME:%s
\n", envVar);
envVar = getenv("SCRIPT_NAME");
hPrintf("- SCRIPT_NAME:%s
\n", envVar);
envVar = getenv("SCRIPT_URI");
hPrintf("- SCRIPT_URI:%s
\n", envVar);
envVar = getenv("SCRIPT_URL");
hPrintf("- SCRIPT_URL:%s
\n", envVar);
envVar = getenv("SERVER_NAME");
hPrintf("- SERVER_NAME:%s
\n", envVar);
envVar = getenv("PATH_INFO");
if (isNotEmpty(envVar))
hPrintf("- PATH_INFO:'%s'
\n", envVar);
else
hPrintf("- PATH_INFO:<empty>
\n");
hPrintf("
\n");
}
char *otherHubUrl = cartUsualString(cart, "urlHub", "");
char *hubDropDown = cartUsualString(cart, "publicHubs", defaultHub);
char *urlDropDown = urlFromShortLabel(hubDropDown);
char *ucscDb = cartUsualString(cart, "ucscGenome", defaultDb);
char *selectRadio = cartUsualString(cart, RADIO_GROUP, RADIO_PUBHUB);
char *urlInput = urlDropDown; /* assume public hub */
if (debug)
{
hPrintf("
\n");
hPrintf("- otherHubUrl: '%s'
\n", otherHubUrl);
hPrintf("- hubDropDown: '%s'
\n", hubDropDown);
hPrintf("- urlDropDown: '%s'
\n", urlDropDown);
hPrintf("- ucscDb: '%s'
\n", ucscDb);
hPrintf("- urlInput: '%s'
\n", urlInput);
hPrintf("- trackLeavesOnly: '%s'
\n", trackLeavesOnly ? "TRUE" : "FALSE");
hPrintf("- jsonOutputArrays: '%s'
\n", jsonOutputArrays ? "TRUE" : "FALSE");
hPrintf("
\n");
}
if (isEmpty(otherHubUrl))
otherHubUrl = urlInput;
if (sameWord(RADIO_OTHERHUB, selectRadio)) /* requested other hub URL */
urlInput = otherHubUrl;
long lastTime = clock1000();
struct trackHub *hub = errCatchTrackHubOpen(urlInput);
if (measureTiming)
{
long thisTime = clock1000();
hPrintf("
hub open time: %ld millis\n", thisTime - lastTime);
}
hPrintf("
\n");
if (debug)
showCartDump();
hPrintf("
Explore hub or database assemblies and tracks (v%s)
\n", SRC_VERSION);
selectionForm();
/* these style mentions need to go into custom css file */
hPrintf("
\n");
if (sameWord(RADIO_UCSCDB, selectRadio)) /* requested UCSC db track list */
{
tracksForUcscDb(ucscDb);
}
else
{
hPrintf("
%s url: %s
\n", sameWord(RADIO_PUBHUB,selectRadio) ? "Public hub" : "Other hub", urlInput);
hPrintf("
\n");
hubInfo("hub name", hub->name);
hubInfo("short label", hub->shortLabel);
hubInfo("long label", hub->longLabel);
hubInfo("genomes file", hub->genomesFile);
hubInfo("default db", hub->defaultDb);
hubInfo("description url", hub->descriptionUrl);
hubInfo("email", hub->email);
if (debug)
{
hubInfo("version", hub->version); /* UCSC internal info */
hubInfo("level", hub->level); /* UCSC internal info */
}
hPrintf("
\n");
genomeList(hub);
}
if (timedOut)
hPrintf("
Reached time out %ld seconds
", timeOutSeconds);
if (measureTiming)
hPrintf("
Overall total time: %ld millis\n", clock1000() - enteredMainTime);
hPrintf("
\n");
hPrintf("
\n");
hPrintf("
\n");
webIncludeFile("inc/jWestFooter.html");
webEndJWest();
// cartWebEnd();
} /* void doMiddle(struct cart *theCart) */
static void setGlobalCgiVars()
/* check for legal CGI variables and set global flags */
{
/* count the arguments to see if any occur more than once */
struct hash *varCounter = hashNew(0);
struct cgiVar *varList = cgiVarList();
struct cgiVar *el = varList;
for ( ; el; el = el->next)
{
hashIncInt(varCounter, el->name);
}
struct hashCookie cookie = hashFirst(varCounter);
struct hashEl *hel = NULL;
for ( hel = hashNext(&cookie); hel; hel = hashNext(&cookie))
{
if (ptToInt(hel->val) > 1)
apiErrAbort(err400, err400Msg, "parameter '%s' found %d times, only one instance allowed", hel->name, ptToInt(hel->val));
}
char *trackLeaves = cgiOptionalString("trackLeavesOnly");
if (isNotEmpty(trackLeaves))
{
if (SETTING_IS_ON(trackLeaves))
trackLeavesOnly = TRUE;
else if (sameString("0", trackLeaves))
trackLeavesOnly = FALSE;
else
apiErrAbort(err400, err400Msg, "unrecognized 'trackLeavesOnly=%s' argument, can only be =1 or =0", trackLeaves);
}
char *jsonArray = cgiOptionalString("jsonOutputArrays");
if (isNotEmpty(jsonArray))
{
if (SETTING_IS_ON(jsonArray))
jsonOutputArrays = TRUE;
else if (sameString("0", jsonArray))
jsonOutputArrays = FALSE;
else
apiErrAbort(err400, err400Msg, "unrecognized 'jsonOutputArrays=%s' argument, can only be =1 or =0", jsonArray);
}
int maybeDebug = cgiOptionalInt("debug", 0);
if (1 == maybeDebug)
debug = TRUE;
char *measTime = cgiOptionalString("measureTiming");
if (isNotEmpty(measTime) && sameWord("1", measTime))
measureTiming = TRUE;
char *maxOut = cgiOptionalString("maxItemsOutput");
if (isNotEmpty(maxOut))
{
long long n = -2;
struct errCatch *errCatch = errCatchNew();
if (errCatchStart(errCatch))
{
n = sqlLongLong(maxOut);
}
errCatchEnd(errCatch);
if (errCatch->gotError)
apiErrAbort(err400, err400Msg, "can not recognize maxItemsOutput '%s' as a number", maxOut);
else
{
if (n == -1) /* can use -1 to indicate as much as allowed */
maxItemsOutput = maxItemLimit;
else if (n > maxItemLimit) /* safety check */
apiErrAbort(err400, err400Msg, "requested maxItemsOutput '%s' greater than maximum limit allowed: %d", maxOut, maxItemLimit);
else if (n < 1)
apiErrAbort(err400, err400Msg, "requested maxItemsOutput '%s' can not be less than one", maxOut, maxItemLimit);
else
maxItemsOutput = n;
}
}
} /* static void setGlobalCgiVars() */
static void redirectToHelp()
/* redirect to the help page */
{
+cspWriteResponseHeader();
puts("Content-Type:text/html");
hPrintf("Status: %d %s\n", err301, err301Msg);
hPrintf("Location: /goldenPath/help/api.html\n");
puts("\n");
hPrintf("\n");
hPrintf("\n");
hPrintf("\n");
hPrintf("\n");
hPrintf("\n");
}
/* Null terminated list of CGI Variables we don't want to save
* permanently. */
static char *excludeVars[] = {"Submit", "submit", "sourceSelected", "selectRadio", "ucscGenome", "publicHubs", "clade", NULL,};
int main(int argc, char *argv[])
/* Process command line. */
{
enteredMainTime = clock1000();
cgiSpoof(&argc, argv);
verboseTimeInit();
/* Pre-validate apiKey before the global bot-check. Without this,
* hgBotDelayTimeFrac->getBotCheckString in hg/lib/botDelay.c would
* hUserAbort() on an invalid key, which apache renders as a 500 because
* it emits plain HTML rather than the JSON the API contract promises. */
char *earlyApiKey = cgiOptionalString(argApiKey);
if (isNotEmpty(earlyApiKey))
{
struct sqlConnection *centralConn = hConnectCentralNoCache();
char *userName = hubSpaceUserNameForApiKey(centralConn, earlyApiKey);
sqlDisconnect(¢ralConn);
if (isEmpty(userName))
apiErrAbort(err403, err403Msg,
"invalid '%s' provided. Make sure the apiKey is valid, or contact us.",
argApiKey);
}
/* similar delay system as in DAS server */
struct errCatch *bnErrCatch = errCatchNew();
if (errCatchStart(bnErrCatch))
botDelay = hgBotDelayTimeFrac(delayFraction);
errCatchEnd(bnErrCatch);
if (bnErrCatch->gotError)
apiErrAbort(err500, err500Msg, "bottleneck server unavailable: %s",
bnErrCatch->message->string);
errCatchFree(&bnErrCatch);
if (botDelay > 0)
{
if (botDelay > 2000)
{
sleep1000(botDelay);
hogExit();
return 0;
}
sleep1000(botDelay);
}
setGlobalCgiVars();
#ifdef NOTUSED // the argument processing doesn't allow udcTimeout
int timeout = cgiOptionalInt("udcTimeout", 300);
if (udcCacheTimeout() < timeout)
udcSetCacheTimeout(timeout);
#endif
setUdcCacheDir();
initSupportedTypes();
char *pathInfo = getenv("PATH_INFO");
char *cmd;
if (isNotEmpty(pathInfo)) /* can get to this immediately, no cart needed */
apiRequest(pathInfo);
else if ((cmd = cgiOptionalString("cmd")) != NULL)
apiRequest(cmd);
else
{
char *allowApiHtml = cfgOptionDefault("hubApi.allowHtml", "off");
if (sameWord("on", allowApiHtml))
{
trackCounter = hashNew(0);
cartEmptyShellNoContent(doMiddle, hUserCookie(), excludeVars, oldVars);
}
else
redirectToHelp();
}
cgiExitTime("hubApi", enteredMainTime);
return 0;
}