cb99f0b11bdeee5dfa76064d38b6410da0f4a709 max Thu Sep 10 00:55:21 2026 -0700 Centralize CGI Content-Type printing in one cgiPrintContentType() helper Around 90 places across the tree hand-rolled the CGI response header, each with its own spelling: "Content-Type:" or "Content-type:", \n or \r\n, and the terminating blank line written as part of the same string, as a separate puts("\n") (which emits two newlines, so a stray blank line led the body) or as printf("\r\n\r\n") (two blank lines). A handful forgot the terminator entirely and relied on a following header to supply it. cgiPrintContentType() in lib/cheapcgi.c now writes the Content-Type line and the blank line that ends the header. Header lines are not ordered, so the callers that also send Status, Set-Cookie, Content-Disposition, Content-Length or X-Sendfile write those first and call this last to close the header; that keeps it to a single helper rather than a print-the-line / end-the-header pair that a caller can half-use. cart.c's existing httpHeaders list already worked this way. Only the CGI response path is touched. The dyStringPrintf("Content-type: ...") calls that build outgoing HTTP *requests* (genomeSpace, oauthLogin, eapMetaSync, edwWebAuthLogin, ga4ghToBed) are unrelated and left alone. Also fills out the apiKey error message in botDelay.c to say where to create a key and that keys are server-specific. No behavior change on the wire beyond dropping those stray blank lines and adding the missing newline after Retry-After. diff --git src/hg/lib/botDelay.c src/hg/lib/botDelay.c index cd8be6d9680..9ea795e8523 100644 --- src/hg/lib/botDelay.c +++ src/hg/lib/botDelay.c @@ -220,31 +220,37 @@ // apiKey and cookieId if they are valid, check hgsid if the string looks OK. char *apiKey = cgiOptionalString("apiKey"); if (apiKey) { // Here we do a mysql query before the bottleneck is complete. // And this is better than handling the request without bottleneck // The connection is closed right away, so if the bottleneck leads to a long sleep, it won't tie up // the MariaDB server. The cost of opening a connection is less than 1msec. struct sqlConnection *conn = hConnectCentralNoCache(); char *userName = hubSpaceUserNameForApiKey(conn, apiKey); sqlDisconnect(&conn); if (userName) safef(botCheckString, 256, "apiKey%s %f", apiKey, fraction); else - hUserAbort("Invalid apiKey provided on URL. Make sure that the apiKey is valid. Or contact us."); + hUserAbort("Invalid apiKey provided on URL. " + "Make sure that the apiKey is valid, " + "check https://genome-euro.ucsc.edu/cgi-bin/hgHubConnect#hubDeveloper to create one " + "or check this key. Note that an apiKey for genome-euro must be created on " + "https://genome-euro.ucsc.edu/cgi-bin/hgHubConnect and the same for genome-asia or " + "other mirrors, apiKeys are server-specific. If you have problems with the apiKey, " + "contact us."); } else { if (isValidHguid(cookieUserId)) safef(botCheckString, 256, "uid%s %f", cookieUserId, fraction); else { // The following happens very rarely on sites like our RR that use the cloudflare captcha, // as all requests (except hgLogin, hgRenderTracks) should come in with a cookie user ID char *hgsid = cgiOptionalString("hgsid"); // For now, we do not check the hgsid against the MariaDb table, only check if the string looks OK if (hgsid && isValidHgsidForEarlyBotCheck(hgsid)) safef(botCheckString, 256, "sid%s %f", hgsid, fraction); else { @@ -337,35 +343,34 @@ char *botCheckString = getBotCheckString(ip, fraction); delay = botDelayTime(host, atoi(port), botCheckString); freeMem(botCheckString); } return delay; } #define err429 429 #define err429Msg "Too Many Requests" int botDelayMillis = 0; static void jsonHogExit(char *cgiExitName, long enteredMainTime, char *hogHost, int retryAfterSeconds) /* err429 Too Many Requests to be returned as JSON data */ { -puts("Content-Type:application/json"); printf("Status: %d %s\n", err429, err429Msg); if (retryAfterSeconds > 0) - printf("Retry-After: %d", retryAfterSeconds); -puts("\n"); /* blank line between header and body */ + printf("Retry-After: %d\n", retryAfterSeconds); +cgiPrintContentType("application/json"); struct jsonWrite *jw = jsonWriteNew(); jsonWriteObjectStart(jw, NULL); jsonWriteString(jw, "error", err429Msg); jsonWriteNumber(jw, "statusCode", err429); char msg[1024]; safef(msg, sizeof(msg), "Your host, %s, has been sending too many requests " "lately and is unfairly loading our site, impacting performance for " "other users. Please contact genome-www@soe.ucsc.edu to ask that your site " "be reenabled. Also, please consider downloading sequence and/or " "annotations in bulk -- see http://genome.ucsc.edu/downloads.html.", hogHost); @@ -382,35 +387,34 @@ int retryAfterSeconds) /* earlyBotCheck requests exit before CGI has done any output or * setups of any kind. HTML output has not yet started. */ { char *hogHost = getenv("REMOTE_ADDR"); char cgiExitName[1024]; safef(cgiExitName, ArraySize(cgiExitName), "%s hogExit", cgiName); if (sameOk("json", exitType)) jsonHogExit(cgiExitName, enteredMainTime, hogHost, retryAfterSeconds); else { cspWriteResponseHeader(); - puts("Content-Type:text/html"); printf("Status: %d %s\n", err429, err429Msg); if (retryAfterSeconds > 0) - printf("Retry-After: %d", retryAfterSeconds); - puts("\n"); /* blank line between header and body */ + printf("Retry-After: %d\n", retryAfterSeconds); + cgiPrintContentType("text/html"); puts("\n"); puts(""); puts(""); puts(""); printf("Status %d: %s\n", err429, err429Msg); printf("

Status %d: %s

\n", err429, err429Msg); time_t now = time(NULL); printf("There is an exceedingly high volume of traffic coming from your " "site (IP address %s) as of %s (California time). It looks like " "a web robot is launching queries quickly, and not even waiting for " "the results of one query to finish before launching another query. " "We cannot service requests from your IP address under these " "conditions. (code %d) "