69053e55a7502bb79acb350f002a277ce330cad9 max Tue Sep 10 03:26:43 2019 -0700 adding the retry-after header to hgLib's bottleneck when blocking, pending Jonathan's/Hiram's feedback, refs #24094 diff --git src/hg/pyLib/hgLib.py src/hg/pyLib/hgLib.py index ee13bb9..2cbca3e 100644 --- src/hg/pyLib/hgLib.py +++ src/hg/pyLib/hgLib.py @@ -36,44 +36,46 @@ import cgitb cgitb.enable() # debug level: a number. the higher, the more debug info is printed # to see most debug messages, set to 1 # another way to change this variable is by setting the URL variable "debug" to 1 verboseLevel = 0 cgiArgs = None # like in the kent tree, we keep track of whether we have already output the content-type line contentLineDone = False # show the bot delay warning message before other printing is done? doWarnBot = False +# current bot delay in milliseconds +botDelayMsecs = 0 # two global variables: the first is the botDelay limit after which the page is slowed down and a warning is shown # the second is the limit after which the page is not shown anymore botDelayWarn = 1000 botDelayBlock = 5000 jksqlTrace = False def warn(format, *args): print (format % args) -def errAbort(msg, status=None): +def errAbort(msg, status=None, headers = None): " show msg and abort. Like errAbort.c " - printContentType(status=status) + printContentType(status=status, headers=headers) print msg exit(0) def debug(level, msg): " output debug message with a given verbosity level " if verboseLevel >= level: printContentType() print(msg+"<br>") sys.stdout.flush() def parseConf(fname): " parse a hg.conf style file, return as dict key -> value (all strings) " conf = {} for line in open(fname): line = line.strip() @@ -327,51 +329,55 @@ </head> """ def webStartGbNoBanner(prefix, title): " output the <head> part, largely copied from web.c / webStartGbNoBanner " print (getGbHeader() % (prefix, title, getNonce(), getNonce())) def runCmd(cmd, mustRun=True): " wrapper around system() that prints error messages. cmd preferably a list, not just a string. " import subprocess ret = subprocess.call(cmd) if ret!=0 and mustRun: errAbort("Could not run command %s" % cmd) return ret -def printContentType(contType="text/html", status=None, fname=None): +def printContentType(contType="text/html", status=None, fname=None, headers=None): """ print the HTTP Content-type header line with an optional file name for downloads. Also optionally prints the bot delay note. The argument 'status' must be an int. """ global contentLineDone if not contentLineDone: contentLineDone = True print("Content-type: %s; charset=utf-8" % contType) if status: if status==400: print("Status: 400 Bad Request") elif status==429: print("Status: 429 Too Many Requests") else: raise Exception("Unknown status code, please update hgLib.py") if fname is not None: print("Content-Disposition: attachment; filename=%s" % fname) + if headers: + for key, val in headers.items(): + print("%s: %s" % (key, val)) + print # this newline is essential, it means: end of header lines if doWarnBot: print ("<div style='background-color:yellow; border:2px solid black'>") print ("We have a suspicion that you are an automated web bot software, not a real user. ") print ("To keep our site fast for other users, we have slowed down this page. ") print ("The slowdown will gradually disappear. ") print ("If you think this is a mistake, please contact us at genome-www@soe.ucsc.edu. ") print ("Also note that all data for hgGeneGraph can be obtained through our public MySQL server and") print ("all our software source code is available and can be installed locally onto your own computer. ") print ("If you are unsure how to use these resources, do not hesitate to contact us.") print ("</div>") def queryBottleneck(host, port, ip): @@ -396,42 +402,44 @@ break return int("".join(buf)) def hgBotDelay(): """ Implement bottleneck delay, get bottleneck server from hg.conf. This behaves similar to the function src/hg/lib/botDelay.c:hgBotDelay It does not use the hgsid, currently it always uses the IP address. Using the hgsid makes little sense. It is more lenient than the C version. """ import time if "DOCUMENT_ROOT" not in os.environ: # skip if not called from Apache return global hgConf global doWarnBot + global botDelayMsecs hgConf = parseHgConf() if "bottleneck.host" not in hgConf: return ip = os.environ["REMOTE_ADDR"] delay = queryBottleneck(hgConf["bottleneck.host"], hgConf["bottleneck.port"], ip) debug(1, "Bottleneck delay: %d msecs" % delay) + botDelayMsecs = delay if delay>botDelayBlock: errAbort("Too many HTTP requests and not enough delay between them. " "Your IP has been blocked to keep this website responsive for other users. " "Please contact genome-www@soe.ucsc.edu to unblock your IP address. We can also help you obtain the data you need without " - "web crawling. ", status=429) + "web crawling. ", status=429, headers = {"Retry-after" : str(botDelayMsecs / 1000)}) sys.exit(0) if delay>botDelayWarn: time.sleep(delay/1000.0) doWarnBot = True # = show warning message later in printContentType() def parseRa(text): " Parse ra-style string and return as dict name -> value " import string lines = text.split("\n") data = dict() for l in lines: if len(l)==0: continue key, val = string.split(l, " ", maxsplit=1)