9927343536614f63516f107aa859b495abf402d1 jcasper Mon May 18 09:58:41 2026 -0700 Reducing bot abuse by restricting the number of IP addresses we'll accept the same hguid from in a short timeframe before requiring a captcha re-up (if captcha is enabled). refs #37494 diff --git src/hg/lib/botDelay.c src/hg/lib/botDelay.c index fb4dd5d4087..0b0f7fa85f2 100644 --- src/hg/lib/botDelay.c +++ src/hg/lib/botDelay.c @@ -4,30 +4,31 @@ /* Copyright (C) 2014 The Regents of the University of California * See kent/LICENSE or http://genome.ucsc.edu/license/ for licensing information. */ #include "common.h" #include "net.h" #include "portable.h" #include "hgConfig.h" #include "cheapcgi.h" #include "hui.h" #include "hCommon.h" #include "botDelay.h" #include "jsonWrite.h" #include "regexHelper.h" #include "hubSpaceKeys.h" +#include "cartDb.h" #define defaultDelayFrac 1.0 /* standard penalty for most CGIs */ #define defaultWarnMs 10000 /* warning at 10 to 20 second delay */ #define defaultExitMs 20000 /* error 429 Too Many Requests after 20+ second delay */ int botDelayWarnMs = 0; /* global so the previously used value can be retrieved */ void abortAndExplainConnectFail() /* Write out a short 500 response explaining that the connection to the * bottleneck server couldn't be established. Then exit. */ { puts("Content-Type:text/html"); printf("Status: 500 Interal Server Error\n"); puts("\n"); /* blank line between header and body */ @@ -113,92 +114,158 @@ , ip, asctime(localtime(&now)), millis); } static char *getCookieUser() /* get the ID string stored in the hguid cookie, it looks like our hgsid session strings on the URL */ { char *user = NULL; char *centralCookie = hUserCookie(); if (centralCookie) user = findCookieData(centralCookie); return user; } +boolean isValidHguid(char *cookieUserId) +/* Check if a particular hguid is valid, i.e. well-formatted, has matching id and secure string, + * and isn't corrupted. */ +{ +if (isEmpty(cookieUserId)) + return FALSE; +boolean isValid = FALSE; +struct sqlConnection *conn = hConnectCentralNoCache(); +struct cartDb *cdb = cartDbLoadFromId(conn, userDbTable(), cookieUserId); +if (cdb) + { + isValid = TRUE; + cartDbFree(&cdb); + } +sqlDisconnect(&conn); +return isValid; +} + +static void recordHguidIpAndMaybeForceCaptcha() +/* When hguidIpTracking is enabled in hg.conf, upsert this request's + * (hguid, REMOTE_ADDR) into the hgcentral tracking table. If a single + * hguid has been seen from more than hguidIpTracking.maxIps distinct IPs + * within the last hguidIpTracking.windowSeconds, set the "captcha" CGI + * var so forceUserIdOrCaptcha() in cart.c forces the user through the + * Cloudflare captcha (the bypass at cart.c:1618 honors this override). */ +{ +if (!cfgOptionBooleanDefault("hguidIpTracking.enabled", FALSE)) + return; + +char *cookieUserId = getCookieUser(); +char *clientIp = getenv("REMOTE_ADDR"); +if (isEmpty(cookieUserId) || isEmpty(clientIp)) + return; + +if (!isValidHguid(cookieUserId)) + return; + +unsigned int userIdNum = cartDbParseId(cookieUserId, NULL); + +int maxIps = atoi(cfgOptionDefault("hguidIpTracking.maxIps", "10")); +int windowSeconds = atoi(cfgOptionDefault("hguidIpTracking.windowSeconds", "600")); +char *table = cfgOptionDefault("hguidIpTracking.table", "hguidIpAccess"); + +struct sqlConnection *conn = hConnectCentralNoCache(); +char query[512]; + +sqlSafef(query, sizeof(query), + "INSERT INTO %s (userId, ip, lastSeen) VALUES (%u, '%s', NOW()) " + "ON DUPLICATE KEY UPDATE lastSeen=NOW()", + table, userIdNum, clientIp); +sqlUpdate(conn, query); + +sqlSafef(query, sizeof(query), + "SELECT COUNT(DISTINCT ip) FROM %s WHERE userId=%u " + "AND lastSeen > NOW() - INTERVAL %d SECOND", + table, userIdNum, windowSeconds); +int distinctIps = sqlQuickNum(conn, query); + +sqlDisconnect(&conn); + +if (distinctIps > maxIps) + { + cgiVarSet("captcha", "1"); + } +} boolean isValidHgsidForEarlyBotCheck(char *raw_hgsid) /* We want to use the hgsid from the CGI parameters, but sometimes requests come in with bogus strings that * need to be ignored. We don't want to run this against the database just yet, but we can at least check * the format. */ { char hgsid[1024]; // Just in case it's egregiously large, we only need the first part to decide if it's valid. safencpy(hgsid, sizeof(hgsid), raw_hgsid, 50); if (regexMatch(hgsid, "^[0-9][0-9]*_[a-zA-Z0-9]{28}$")) return TRUE; return FALSE; } - char *getBotCheckString(char *ip, double fraction) /* compose "user.ip fraction" string for bot check */ { char *cookieUserId = getCookieUser(); char *botCheckString = needMem(256); boolean useNew = cfgOptionBooleanDefault("newBotDelay", TRUE); if (useNew) { // the new strategy is: bottleneck on apiKey, then cookie-userId, then // hgsid, and only if none of these is available, on IP address. Also, check // apiKey and cookieId if they are valid, check hgsid if the string looks OK. char *apiKey = cgiOptionalString("apiKey"); if (apiKey) { // Here we do a mysql query before the bottleneck is complete. // And this is better than handling the request without bottleneck // The connection is closed right away, so if the bottleneck leads to a long sleep, it won't tie up // the MariaDB server. The cost of opening a connection is less than 1msec. struct sqlConnection *conn = hConnectCentralNoCache(); char *userName = hubSpaceUserNameForApiKey(conn, apiKey); sqlDisconnect(&conn); if (userName) safef(botCheckString, 256, "apiKey%s %f", apiKey, fraction); else hUserAbort("Invalid apiKey provided on URL. Make sure that the apiKey is valid. Or contact us."); } else - if (cookieUserId) + { + if (isValidHguid(cookieUserId)) safef(botCheckString, 256, "uid%s %f", cookieUserId, fraction); else { // The following happens very rarely on sites like our RR that use the cloudflare captcha, // as all requests (except hgLogin, hgRenderTracks) should come in with a cookie user ID char *hgsid = cgiOptionalString("hgsid"); // For now, we do not check the hgsid against the MariaDb table, only check if the string looks OK if (hgsid && isValidHgsidForEarlyBotCheck(hgsid)) safef(botCheckString, 256, "sid%s %f", hgsid, fraction); else { if (hgsid) // We were given an invalid hgsid - penalize this source in case of abuse fraction *= 5; safef(botCheckString, 256, "%s %f", ip, fraction); } } } + } else // our old system - only relevant on mirrors: bottleneck on cookie or IP address { if (cookieUserId) safef(botCheckString, 256, "%s.%s %f", cookieUserId, ip, fraction); else safef(botCheckString, 256, "%s %f", ip, fraction); } return botCheckString; } boolean botException() /* check if the remote ip address is on the exceptions list */ { char *exceptIps = cfgOption("bottleneck.except"); @@ -338,30 +405,32 @@ boolean earlyBotCheck(long enteredMainTime, char *cgiName, double delayFrac, int warnMs, int exitMs, char *exitType) /* replaces the former botDelayCgi now in use before the CGI has started any * output or setup the cart of done any MySQL operations. The boolean * return is used later in the CGI after it has done all its setups and * started output so it can issue the warning. Pass in delayFrac 0.0 * to use the default 1.0, pass in 0 for warnMs and exitMs to use defaults, * and exitType is either 'html' or 'json' to do that type of exit output in * the case of hogExit(); */ { boolean issueWarning = FALSE; if (botException()) /* don't do this if caller is on the exception list */ return issueWarning; +recordHguidIpAndMaybeForceCaptcha(); + if (delayFrac < 0.000001) /* passed in zero, use default */ delayFrac = defaultDelayFrac; botDelayWarnMs = warnMs; if (botDelayWarnMs < 1) /* passed in zero, use default */ botDelayWarnMs = defaultWarnMs; if (exitMs < 1) /* passed in zero, use default */ exitMs = defaultExitMs; botDelayMillis = hgBotDelayTimeFrac(delayFrac); if (botDelayMillis > 0) { if (botDelayMillis > botDelayWarnMs) {