97bf8f3cb8cbc74dcbe44625654ddf4af6b6e976 hiram Tue Apr 2 13:18:05 2019 -0700 calling bottleneck earlier in the game refs 18869 diff --git src/hg/hubApi/hubApi.c src/hg/hubApi/hubApi.c index 849ede9..ac9a8c0 100644 --- src/hg/hubApi/hubApi.c +++ src/hg/hubApi/hubApi.c @@ -138,45 +138,30 @@ return list; } static boolean timeOutReached() /* see if the timeout has been reached to determine if an exit * is appropriate at this time */ { long nowTime = clock1000(); timedOut = FALSE; if ((nowTime - enteredMainTime) > (1000 * timeOutSeconds)) timedOut= TRUE; return timedOut; } -#ifdef NOT -static void showCounts(struct hash *countTracks) -{ -if (countTracks->elCount) - { - hPrintf("
    • \n"); - } -} -#endif - static void hashCountTrack(struct trackDb *tdb, struct hash *countTracks) /* this is counting up track types into the hash countTracks */ { char *stripType = cloneString(tdb->type); if (startsWith("chain ", tdb->type)) stripType = cloneString("chain"); else if (startsWith("netAlign ", tdb->type)) stripType = cloneString("netAlign"); else if (startsWith("genePred ", tdb->type)) stripType = cloneString("genePred"); else if (startsWith("bigWig ", tdb->type)) stripType = cloneString("bigWig"); else if (startsWith("wigMaf ", tdb->type)) stripType = cloneString("wigMaf"); else if (startsWith("wig ", tdb->type)) @@ -194,31 +179,30 @@ hashIncInt(countTracks, "composite view"); else if (superChild) { hashIncInt(countTracks, "superTrack child"); hashIncInt(countTracks, stripType); hashIncInt(countTracks, "track count"); } else if (isEmpty(tdb->type)) hashIncInt(countTracks, "no type specified"); else { hashIncInt(countTracks, stripType); hashIncInt(countTracks, "track count"); } freeMem(stripType); -// showCounts(countTracks); } static void sampleUrl(struct trackHub *hub, char *db, struct trackDb *tdb, char *chrom, unsigned chromSize, char *errorString) /* print out a sample getData URL */ { char errorPrint[2048]; errorPrint[0] = 0; if (isNotEmpty(errorString)) { safef(errorPrint, sizeof(errorPrint), " ERROR: %s", errorString); } unsigned start = chromSize / 4; unsigned end = start + 10000; @@ -988,103 +972,94 @@ if (isEmpty(httpHost)) urlPrefix = ""; else { if (! startsWith("hgwdev-api", httpHost)) { if (startsWith("hgwdev",httpHost) || startsWith("genome-test", httpHost)) { urlPrefix = "../cgi-bin/hubApi"; } } } } -#ifdef NOT -static void showExamples(char *url, struct trackHubGenome *hubGenome, char *ucscDb) -{ -hPrintf("

    Example URLs to return json data structures:

    \n"); - -hPrintf("

    listing functions

    \n"); -hPrintf("
      \n"); -hPrintf("
    1. list public hubs %s/list/publicHubs
      2. \n", urlPrefix, urlPrefix); -hPrintf("
    2. list database genomes %s/list/ucscGenomes
      3. \n", urlPrefix, urlPrefix); -hPrintf("
    3. list genomes from specified hub %s/list/hubGenomes?hubUrl=%s
      4. \n", urlPrefix, url, urlPrefix, url); -hPrintf("
    4. list tracks from specified hub and genome %s/list/tracks?hubUrl=%s&genome=%s
      5. \n", urlPrefix, url, hubGenome->name, urlPrefix, url, hubGenome->name); -hPrintf("
    5. list tracks from specified UCSC database %s/list/tracks?db=%s
      6. \n", urlPrefix, ucscDb, urlPrefix, ucscDb); -hPrintf("
    6. list chromosomes from specified UCSC database %s/list/chromosomes?db=%s
      7. \n", urlPrefix, ucscDb, urlPrefix, ucscDb); -hPrintf("
    7. list chromosomes from specified track from UCSC databaset %s/list/chromosomes?db=%s&track=gold
      8. \n", urlPrefix, ucscDb, urlPrefix, ucscDb); -hPrintf("
    \n"); - -hPrintf("

    getData functions

    \n"); -hPrintf("
      \n"); -hPrintf("
    1. get sequence from specified database and chromosome %s/getData/sequence?db=%s&chrom=chrM
      2. \n", urlPrefix, ucscDb, urlPrefix, ucscDb); -hPrintf("
    2. get sequence from specified database, chromosome with start,end coordinates %s/getData/sequence?db=%s&chrom=chrM&start=0&end=128
      3. \n", urlPrefix, ucscDb, urlPrefix, ucscDb); -hPrintf("
    3. get entire track data from specified database and track name (gold == Assembly) %s/getData/track?db=%s&track=gold
      4. \n", urlPrefix, ucscDb, urlPrefix, ucscDb); -hPrintf("
    4. get track data from specified database, chromosome and track name (gold == Assembly) %s/getData/track?db=%s&chrom=chrM&track=gold
      5. \n", urlPrefix, ucscDb, urlPrefix, ucscDb); -hPrintf("
    5. get track data from specified database, chromosome, track name, start and end coordinates %s/getData/track?db=%s&chrom=chrI&track=gold&start=107680&end=186148
      6. \n", urlPrefix, defaultDb, urlPrefix, defaultDb); -hPrintf("
    6. get entire track data from specified hub and track name %s/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/GillBejerano/hub.txt&genome=hg19&track=ultraConserved
      7. \n", urlPrefix, urlPrefix); -hPrintf("
    7. get track data from specified hub, chromosome and track name (full chromosome) %s/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chrCp&track=assembly_
      8. \n", urlPrefix, urlPrefix); -hPrintf("
    8. get track data from specified hub, chromosome, track name, start and end coordinates %s/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chr1&track=assembly_&start=0&end=14309681
      9. \n", urlPrefix, urlPrefix); -hPrintf("
    9. get all track data from specified hub and track name %s/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&track=gc5Base
      10. \n", urlPrefix, urlPrefix); -hPrintf("
    10. get track data from specified hub, chromosome, track name, start and end coordinates %s/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chrMt&track=gc5Base&start=143600&end=143685
      11. \n", urlPrefix, urlPrefix); -hPrintf("
    11. get bigWig track data from specified database, chromosome, track name, start and end coordinates %s/getData/track?db=%s&chrom=chrI&track=gc5BaseBw&start=107680&end=186148
      12. \n", urlPrefix, defaultDb, urlPrefix, defaultDb); -hPrintf("
    12. get bigBed track data from specified database, chromosome, track name, start and end coordinates %s/getData/track?db=%s&chrom=chrII&track=ncbiRefSeqOther&start=14334626&end=14979625
      13. \n", urlPrefix, defaultDb, urlPrefix, defaultDb); -hPrintf("
    13. get narrowPeak track data from specified database, chromosome, track name, start and end coordinates %s/getData/track?db=%s&chrom=chr1&track=wgEncodeAwgDnaseDuke8988tUniPk&start=14334626&end=14979625
      14. \n", urlPrefix, "hg19", urlPrefix, "hg19"); -hPrintf("
    14. get broadPeak track data from specified database, chromosome, track name, start and end coordinates %s/getData/track?db=%s&chrom=chr1&track=wgEncodeBroadHistoneOsteoP300kat3bPk&start=14334626&end=14979625
      15. \n", urlPrefix, "hg19", urlPrefix, "hg19"); - -hPrintf("
    \n"); - -hPrintf("

    Example URLs to generate errors:

    \n"); -hPrintf("
      \n"); -hPrintf("
    1. get track data from specified hub, chromosome, track name, start and end coordinates %s/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chrI&track=assembly_&start=0&end=14309681
      2. \n", urlPrefix, urlPrefix); -hPrintf("
    \n"); -} /* static void showExamples() */ - -#endif - static void showCartDump() /* for information purposes only during development, will become obsolete */ { hPrintf("

    cart dump

    "); hPrintf("
    \n");
 cartDump(cart);
 hPrintf("
    \n"); } static void sendJsonHogMessage(char *hogHost) { apiErrAbort("Your host, %s, has been sending too many requests lately and is " "unfairly loading our site, impacting performance for other users. " "Please contact genome@soe.ucsc.edu to ask that your site " "be reenabled. Also, please consider downloading sequence and/or " "annotations in bulk -- see http://genome.ucsc.edu/downloads.html.", hogHost); } static void sendHogMessage(char *hogHost) { +hPrintf("\n"); +hPrintf("\n"); +hPrintf("\n"); +hPrintf("\n"); +hPrintf("Status 459 Too Many Requests\n"); + +hPrintf("

    Status 459 Too many Requests

    \n"); hPrintf("Your host, %s, has been sending too many requests lately and is " "unfairly loading our site, impacting performance for other users. " "Please contact genome@soe.ucsc.edu to ask that your site " "be reenabled. Also, please consider downloading sequence and/or " "annotations in bulk -- see http://genome.ucsc.edu/downloads.html.", hogHost); +hPrintf("

    \n"); exit(0); } +static void hogExit() +/* bottleneck server requests exit */ +{ +char *hogHost = getenv("REMOTE_ADDR"); +char *pathInfo = getenv("PATH_INFO"); +/* nothing on incoming path, then display the WEB page instead */ +if (sameOk("/",pathInfo)) + pathInfo = NULL; +if (isNotEmpty(pathInfo)) + { + puts("Content-Type:application/json"); + puts("Status: 459 Too Many Requests"); + /* maybe a Retry-After: 3600 statement here ? */ + puts("\n"); + sendJsonHogMessage(hogHost); + } +else + { + puts("Content-Type:text/html"); + puts("Status: 459 Too Many Requests"); + /* maybe a Retry-After: 3600 statement here ? */ + puts("\n"); + sendHogMessage(hogHost); + } +} + static void doMiddle(struct cart *theCart) /* Set up globals and make web page */ { cart = theCart; measureTiming = hPrintStatus() && isNotEmpty(cartOptionalString(cart, "measureTiming")); measureTiming = TRUE; char *database = NULL; char *genome = NULL; cgiVarSet("ignoreCookie", "1"); getDbAndGenome(cart, &database, &genome, oldVars); initGenbankTableNames(database); initSupportedTypes(); initUrlPrefix(); @@ -1108,68 +1083,45 @@ pathInfo = NULL; boolean commandError = FALSE; /*expect no more than MAX_PATH_INFO number of words*/ char *words[MAX_PATH_INFO]; if (isNotEmpty(pathInfo)) { setupFunctionHash(); struct hashEl *hel = parsePathInfo(pathInfo, words); /* verify valid API command */ if (hel) /* have valid command */ { hPrintDisable(); + /* could check botDelay here to see if 459 status is advised */ puts("Content-Type:application/json"); puts("\n"); - /* similar delay system as in DAS server */ - botDelay = hgBotDelayTimeFrac(delayFraction); - if (botDelay > 0) - { - if (botDelay > 2000) - { - char *hogHost = getenv("REMOTE_ADDR"); - sendJsonHogMessage(hogHost); - return; - } - sleep1000(botDelay); - } - void (*apiFunction)(char **) = hel->val; (*apiFunction)(words); return; } else commandError = TRUE; } +/* could check botDelay here to see if 459 status is advised */ puts("Content-Type:text/html"); puts("\n"); -/* similar delay system as in DAS server */ -botDelay = hgBotDelayTimeFrac(delayFraction); -if (botDelay > 0) - { - if (botDelay > 2000) - { - char *hogHost = getenv("REMOTE_ADDR"); - sendHogMessage(hogHost); - } - sleep1000(botDelay); - } - (void) hubPublicDbLoadAll(); struct dbDb *dbList = ucscDbDb(); char **ucscDbList = NULL; int listSize = slCount(dbList); AllocArray(ucscDbList, listSize); struct dbDb *el = dbList; int ucscDataBaseCount = 0; int maxDbNameWidth = 0; for ( ; el != NULL; el = el->next ) { ucscDbList[ucscDataBaseCount++] = el->name; if (strlen(el->name) > maxDbNameWidth) maxDbNameWidth = strlen(el->name); } @@ -1313,19 +1265,31 @@ cartWebEnd(); } /* void doMiddle(struct cart *theCart) */ /* Null terminated list of CGI Variables we don't want to save * permanently. */ static char *excludeVars[] = {"Submit", "submit", "goOtherHub", "goPublicHub", "goUcscDb", "ucscGenomes", "publicHubs", NULL,}; int main(int argc, char *argv[]) /* Process command line. */ { enteredMainTime = clock1000(); cgiSpoof(&argc, argv); measureTiming = TRUE; verboseTimeInit(); +/* similar delay system as in DAS server */ +botDelay = hgBotDelayTimeFrac(delayFraction); +if (botDelay > 0) + { + if (botDelay > 2000) + { + hogExit(); + return 0; + } + sleep1000(botDelay); + } + trackCounter = hashNew(0); cartEmptyShellNoContent(doMiddle, hUserCookie(), excludeVars, oldVars); return 0; }