97bf8f3cb8cbc74dcbe44625654ddf4af6b6e976 hiram Tue Apr 2 13:18:05 2019 -0700 calling bottleneck earlier in the game refs 18869 diff --git src/hg/hubApi/hubApi.c src/hg/hubApi/hubApi.c index 849ede9..ac9a8c0 100644 --- src/hg/hubApi/hubApi.c +++ src/hg/hubApi/hubApi.c @@ -138,45 +138,30 @@ return list; } static boolean timeOutReached() /* see if the timeout has been reached to determine if an exit * is appropriate at this time */ { long nowTime = clock1000(); timedOut = FALSE; if ((nowTime - enteredMainTime) > (1000 * timeOutSeconds)) timedOut= TRUE; return timedOut; } -#ifdef NOT -static void showCounts(struct hash *countTracks) -{ -if (countTracks->elCount) - { - hPrintf(" <li><ul>\n"); - struct hashEl *hel; - struct hashCookie hc = hashFirst(countTracks); - while ((hel = hashNext(&hc)) != NULL) - hPrintf(" <li>%d - %s</li>\n", ptToInt(hel->val), hel->name); - hPrintf(" </ul></li>\n"); - } -} -#endif - static void hashCountTrack(struct trackDb *tdb, struct hash *countTracks) /* this is counting up track types into the hash countTracks */ { char *stripType = cloneString(tdb->type); if (startsWith("chain ", tdb->type)) stripType = cloneString("chain"); else if (startsWith("netAlign ", tdb->type)) stripType = cloneString("netAlign"); else if (startsWith("genePred ", tdb->type)) stripType = cloneString("genePred"); else if (startsWith("bigWig ", tdb->type)) stripType = cloneString("bigWig"); else if (startsWith("wigMaf ", tdb->type)) stripType = cloneString("wigMaf"); else if (startsWith("wig ", tdb->type)) @@ -194,31 +179,30 @@ hashIncInt(countTracks, "composite view"); else if (superChild) { hashIncInt(countTracks, "superTrack child"); hashIncInt(countTracks, stripType); hashIncInt(countTracks, "track count"); } else if (isEmpty(tdb->type)) hashIncInt(countTracks, "no type specified"); else { hashIncInt(countTracks, stripType); hashIncInt(countTracks, "track count"); } freeMem(stripType); -// showCounts(countTracks); } static void sampleUrl(struct trackHub *hub, char *db, struct trackDb *tdb, char *chrom, unsigned chromSize, char *errorString) /* print out a sample getData URL */ { char errorPrint[2048]; errorPrint[0] = 0; if (isNotEmpty(errorString)) { safef(errorPrint, sizeof(errorPrint), " <font color='red'>ERROR: %s</font>", errorString); } unsigned start = chromSize / 4; unsigned end = start + 10000; @@ -988,103 +972,94 @@ if (isEmpty(httpHost)) urlPrefix = ""; else { if (! startsWith("hgwdev-api", httpHost)) { if (startsWith("hgwdev",httpHost) || startsWith("genome-test", httpHost)) { urlPrefix = "../cgi-bin/hubApi"; } } } } -#ifdef NOT -static void showExamples(char *url, struct trackHubGenome *hubGenome, char *ucscDb) -{ -hPrintf("<h2>Example URLs to return json data structures:</h2>\n"); - -hPrintf("<h3>listing functions</h3>\n"); -hPrintf("<ol>\n"); -hPrintf("<li><a href='%s/list/publicHubs' target=_blank>list public hubs</a> <em>%s/list/publicHubs</em></li>\n", urlPrefix, urlPrefix); -hPrintf("<li><a href='%s/list/ucscGenomes' target=_blank>list database genomes</a> <em>%s/list/ucscGenomes</em></li>\n", urlPrefix, urlPrefix); -hPrintf("<li><a href='%s/list/hubGenomes?hubUrl=%s' target=_blank>list genomes from specified hub</a> <em>%s/list/hubGenomes?hubUrl=%s</em></li>\n", urlPrefix, url, urlPrefix, url); -hPrintf("<li><a href='%s/list/tracks?hubUrl=%s&genome=%s' target=_blank>list tracks from specified hub and genome</a> <em>%s/list/tracks?hubUrl=%s&genome=%s</em></li>\n", urlPrefix, url, hubGenome->name, urlPrefix, url, hubGenome->name); -hPrintf("<li><a href='%s/list/tracks?db=%s' target=_blank>list tracks from specified UCSC database</a> <em>%s/list/tracks?db=%s</em></li>\n", urlPrefix, ucscDb, urlPrefix, ucscDb); -hPrintf("<li><a href='%s/list/chromosomes?db=%s' target=_blank>list chromosomes from specified UCSC database</a> <em>%s/list/chromosomes?db=%s</em></li>\n", urlPrefix, ucscDb, urlPrefix, ucscDb); -hPrintf("<li><a href='%s/list/chromosomes?db=%s&track=gold' target=_blank>list chromosomes from specified track from UCSC databaset</a> <em>%s/list/chromosomes?db=%s&track=gold</em></li>\n", urlPrefix, ucscDb, urlPrefix, ucscDb); -hPrintf("</ol>\n"); - -hPrintf("<h3>getData functions</h3>\n"); -hPrintf("<ol>\n"); -hPrintf("<li><a href='%s/getData/sequence?db=%s&chrom=chrM' target=_blank>get sequence from specified database and chromosome</a> <em>%s/getData/sequence?db=%s&chrom=chrM</em></li>\n", urlPrefix, ucscDb, urlPrefix, ucscDb); -hPrintf("<li><a href='%s/getData/sequence?db=%s&chrom=chrM&start=0&end=128' target=_blank>get sequence from specified database, chromosome with start,end coordinates</a> <em>%s/getData/sequence?db=%s&chrom=chrM&start=0&end=128</em></li>\n", urlPrefix, ucscDb, urlPrefix, ucscDb); -hPrintf("<li><a href='%s/getData/track?db=%s&track=gold' target=_blank>get entire track data from specified database and track name (gold == Assembly)</a> <em>%s/getData/track?db=%s&track=gold</em></li>\n", urlPrefix, ucscDb, urlPrefix, ucscDb); -hPrintf("<li><a href='%s/getData/track?db=%s&chrom=chrM&track=gold' target=_blank>get track data from specified database, chromosome and track name (gold == Assembly)</a> <em>%s/getData/track?db=%s&chrom=chrM&track=gold</em></li>\n", urlPrefix, ucscDb, urlPrefix, ucscDb); -hPrintf("<li><a href='%s/getData/track?db=%s&chrom=chrI&track=gold&start=107680&end=186148' target=_blank>get track data from specified database, chromosome, track name, start and end coordinates</a> <em>%s/getData/track?db=%s&chrom=chrI&track=gold&start=107680&end=186148</em></li>\n", urlPrefix, defaultDb, urlPrefix, defaultDb); -hPrintf("<li><a href='%s/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/GillBejerano/hub.txt&genome=hg19&track=ultraConserved' target=_blank>get entire track data from specified hub and track name</a> <em>%s/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/GillBejerano/hub.txt&genome=hg19&track=ultraConserved</em></li>\n", urlPrefix, urlPrefix); -hPrintf("<li><a href='%s/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chrCp&track=assembly_' target=_blank>get track data from specified hub, chromosome and track name (full chromosome)</a> <em>%s/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chrCp&track=assembly_</em></li>\n", urlPrefix, urlPrefix); -hPrintf("<li><a href='%s/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chr1&track=assembly_&start=0&end=14309681' target=_blank>get track data from specified hub, chromosome, track name, start and end coordinates</a> <em>%s/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chr1&track=assembly_&start=0&end=14309681</em></li>\n", urlPrefix, urlPrefix); -hPrintf("<li><a href='%s/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&track=gc5Base_' target=_blank>get all track data from specified hub and track name</a> <em>%s/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&track=gc5Base</em></li>\n", urlPrefix, urlPrefix); -hPrintf("<li><a href='%s/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chrMt&track=gc5Base_&start=143600&end=143685' target=_blank>get track data from specified hub, chromosome, track name, start and end coordinates</a> <em>%s/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chrMt&track=gc5Base&start=143600&end=143685</em></li>\n", urlPrefix, urlPrefix); -hPrintf("<li><a href='%s/getData/track?db=%s&chrom=chrI&track=gc5BaseBw&start=107680&end=186148' target=_blank>get bigWig track data from specified database, chromosome, track name, start and end coordinates</a> <em>%s/getData/track?db=%s&chrom=chrI&track=gc5BaseBw&start=107680&end=186148</em></li>\n", urlPrefix, defaultDb, urlPrefix, defaultDb); -hPrintf("<li><a href='%s/getData/track?db=%s&chrom=chrII&track=ncbiRefSeqOther&start=14334626&end=14979625' target=_blank>get bigBed track data from specified database, chromosome, track name, start and end coordinates</a> <em>%s/getData/track?db=%s&chrom=chrII&track=ncbiRefSeqOther&start=14334626&end=14979625</em></li>\n", urlPrefix, defaultDb, urlPrefix, defaultDb); -hPrintf("<li><a href='%s/getData/track?db=%s&chrom=chr1&track=wgEncodeAwgDnaseDuke8988tUniPk&start=14334626&end=14979625' target=_blank>get narrowPeak track data from specified database, chromosome, track name, start and end coordinates</a> <em>%s/getData/track?db=%s&chrom=chr1&track=wgEncodeAwgDnaseDuke8988tUniPk&start=14334626&end=14979625</em></li>\n", urlPrefix, "hg19", urlPrefix, "hg19"); -hPrintf("<li><a href='%s/getData/track?db=%s&chrom=chr1&track=wgEncodeBroadHistoneOsteoP300kat3bPk&start=14334626&end=14979625' target=_blank>get broadPeak track data from specified database, chromosome, track name, start and end coordinates</a> <em>%s/getData/track?db=%s&chrom=chr1&track=wgEncodeBroadHistoneOsteoP300kat3bPk&start=14334626&end=14979625</em></li>\n", urlPrefix, "hg19", urlPrefix, "hg19"); - -hPrintf("</ol>\n"); - -hPrintf("<h2>Example URLs to generate errors:</h2>\n"); -hPrintf("<ol>\n"); -hPrintf("<li><a href='%s/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chrI&track=assembly_&start=0&end=14309681' target=_blank>get track data from specified hub, chromosome, track name, start and end coordinates</a> <em>%s/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chrI&track=assembly_&start=0&end=14309681</em></li>\n", urlPrefix, urlPrefix); -hPrintf("</ol>\n"); -} /* static void showExamples() */ - -#endif - static void showCartDump() /* for information purposes only during development, will become obsolete */ { hPrintf("<h4>cart dump</h4>"); hPrintf("<pre>\n"); cartDump(cart); hPrintf("</pre>\n"); } static void sendJsonHogMessage(char *hogHost) { apiErrAbort("Your host, %s, has been sending too many requests lately and is " "unfairly loading our site, impacting performance for other users. " "Please contact genome@soe.ucsc.edu to ask that your site " "be reenabled. Also, please consider downloading sequence and/or " "annotations in bulk -- see http://genome.ucsc.edu/downloads.html.", hogHost); } static void sendHogMessage(char *hogHost) { +hPrintf("<!DOCTYPE HTML>\n"); +hPrintf("<html lang='en'>\n"); +hPrintf("<head>\n"); +hPrintf("<meta charset=\"utf-8\">\n"); +hPrintf("<title>Status 459 Too Many Requests</title></head>\n"); + +hPrintf("<body><h1>Status 459 Too many Requests</h1><p>\n"); hPrintf("Your host, %s, has been sending too many requests lately and is " "unfairly loading our site, impacting performance for other users. " "Please contact genome@soe.ucsc.edu to ask that your site " "be reenabled. Also, please consider downloading sequence and/or " "annotations in bulk -- see http://genome.ucsc.edu/downloads.html.", hogHost); +hPrintf("</p></body></html>\n"); exit(0); } +static void hogExit() +/* bottleneck server requests exit */ +{ +char *hogHost = getenv("REMOTE_ADDR"); +char *pathInfo = getenv("PATH_INFO"); +/* nothing on incoming path, then display the WEB page instead */ +if (sameOk("/",pathInfo)) + pathInfo = NULL; +if (isNotEmpty(pathInfo)) + { + puts("Content-Type:application/json"); + puts("Status: 459 Too Many Requests"); + /* maybe a Retry-After: 3600 statement here ? */ + puts("\n"); + sendJsonHogMessage(hogHost); + } +else + { + puts("Content-Type:text/html"); + puts("Status: 459 Too Many Requests"); + /* maybe a Retry-After: 3600 statement here ? */ + puts("\n"); + sendHogMessage(hogHost); + } +} + static void doMiddle(struct cart *theCart) /* Set up globals and make web page */ { cart = theCart; measureTiming = hPrintStatus() && isNotEmpty(cartOptionalString(cart, "measureTiming")); measureTiming = TRUE; char *database = NULL; char *genome = NULL; cgiVarSet("ignoreCookie", "1"); getDbAndGenome(cart, &database, &genome, oldVars); initGenbankTableNames(database); initSupportedTypes(); initUrlPrefix(); @@ -1108,68 +1083,45 @@ pathInfo = NULL; boolean commandError = FALSE; /*expect no more than MAX_PATH_INFO number of words*/ char *words[MAX_PATH_INFO]; if (isNotEmpty(pathInfo)) { setupFunctionHash(); struct hashEl *hel = parsePathInfo(pathInfo, words); /* verify valid API command */ if (hel) /* have valid command */ { hPrintDisable(); + /* could check botDelay here to see if 459 status is advised */ puts("Content-Type:application/json"); puts("\n"); - /* similar delay system as in DAS server */ - botDelay = hgBotDelayTimeFrac(delayFraction); - if (botDelay > 0) - { - if (botDelay > 2000) - { - char *hogHost = getenv("REMOTE_ADDR"); - sendJsonHogMessage(hogHost); - return; - } - sleep1000(botDelay); - } - void (*apiFunction)(char **) = hel->val; (*apiFunction)(words); return; } else commandError = TRUE; } +/* could check botDelay here to see if 459 status is advised */ puts("Content-Type:text/html"); puts("\n"); -/* similar delay system as in DAS server */ -botDelay = hgBotDelayTimeFrac(delayFraction); -if (botDelay > 0) - { - if (botDelay > 2000) - { - char *hogHost = getenv("REMOTE_ADDR"); - sendHogMessage(hogHost); - } - sleep1000(botDelay); - } - (void) hubPublicDbLoadAll(); struct dbDb *dbList = ucscDbDb(); char **ucscDbList = NULL; int listSize = slCount(dbList); AllocArray(ucscDbList, listSize); struct dbDb *el = dbList; int ucscDataBaseCount = 0; int maxDbNameWidth = 0; for ( ; el != NULL; el = el->next ) { ucscDbList[ucscDataBaseCount++] = el->name; if (strlen(el->name) > maxDbNameWidth) maxDbNameWidth = strlen(el->name); } @@ -1313,19 +1265,31 @@ cartWebEnd(); } /* void doMiddle(struct cart *theCart) */ /* Null terminated list of CGI Variables we don't want to save * permanently. */ static char *excludeVars[] = {"Submit", "submit", "goOtherHub", "goPublicHub", "goUcscDb", "ucscGenomes", "publicHubs", NULL,}; int main(int argc, char *argv[]) /* Process command line. */ { enteredMainTime = clock1000(); cgiSpoof(&argc, argv); measureTiming = TRUE; verboseTimeInit(); +/* similar delay system as in DAS server */ +botDelay = hgBotDelayTimeFrac(delayFraction); +if (botDelay > 0) + { + if (botDelay > 2000) + { + hogExit(); + return 0; + } + sleep1000(botDelay); + } + trackCounter = hashNew(0); cartEmptyShellNoContent(doMiddle, hUserCookie(), excludeVars, oldVars); return 0; }