ce1164c7b5a0ae2e6acf9c25a411df4b4c36244e jcasper Wed Mar 15 14:24:23 2017 -0700 Bottleneck server can now impose fractional penalties; hgTracks, hgTrackUi, and hgBlat take advantage, refs #19019, #18461 diff --git src/weblet/bottleneck/bottleneck.c src/weblet/bottleneck/bottleneck.c index 3891122..69c0004 100644 --- src/weblet/bottleneck/bottleneck.c +++ src/weblet/bottleneck/bottleneck.c @@ -1,59 +1,59 @@ /* bottleneck - A server that helps slow down hyperactive web robots. */ #include "common.h" #include <sys/wait.h> #include "localmem.h" #include "linefile.h" #include "hash.h" #include "options.h" #include "portable.h" #include "internet.h" #include "net.h" int port = 17776; /* Default bottleneck port. */ char *host = "localhost"; /* Default host. */ -int penalty = 150; /* Penalty in milliseconds per access. */ +int penalty = 75; /* Penalty in milliseconds per access. */ int recovery = 10; /* Recovery in milliseconds per second. */ char *subnet = NULL; /* Subnet as dotted quads. */ void usage() /* Explain usage and exit. */ { errAbort( "bottleneck v2 - A server that helps slow down hyperactive web robots\n" "usage:\n" " bottleneck start\n" "Start up bottleneck server\n" - " bottleneck query ip-address [count]\n" - "Ask bottleneck server how long to wait to service ip-address\n" + " bottleneck query ip-address [count] [fraction]\n" + "Ask bottleneck server how long to wait to service ip-address after imposing\n" + "the specified fraction of the access penalty (default of 1.0)\n" " bottleneck list\n" "List accessing sites\n" " bottleneck set ip-address milliseconds\n" "Set current delay for given ip-address\n" "options:\n" " -port=XXXX - Use specific tcp/ip port. Default %d.\n" " -host=XXXXX - Use specific host. Default %s.\n" " -subnet=WWW.XXX.YYY.ZZZ Restrict access to subnet (example 192.168.255.255)\n" " -penalty=N - Penalty (in milliseconds) for each access, default %d\n" " -recovery=N - Amount to recover (in milliseconds) for each second\n" " between accesses. Default %d\n" - "Note penalty and recovery if moved from defaults should balance\n" - "At the default settings an equilibrium will be achieved when queries\n" - "are spaced 15 seconds apart. The maximum delay should thus be 15 seconds\n" - "It will take 25 minutes of idleness for the maximum delay to decay back to\n" - "zero.\n" + "Note: penalty and recovery if moved from defaults should balance\n" + "At the default settings, an equilibrium will be achieved when queries\n" + "are spaced 7.5 seconds apart. At the default decay value, an accumulated\n" + "delay of 15 seconds will take 25 minutes of idleness to decay back to zero.\n" , port, host, penalty, recovery ); } static struct optionSpec options[] = { {"port", OPTION_INT}, {"host", OPTION_STRING}, {"subnet", OPTION_STRING}, {"penalty", OPTION_INT}, {"recovery", OPTION_INT}, {NULL, 0}, }; struct tracker /* Keep track of IP addresses. */ @@ -74,57 +74,57 @@ /* Find tracker for given IP address. Make it up if it * doesn't already exist. */ { struct tracker *tracker = hashFindVal(trackerHash, ip); if (tracker == NULL) { lmAllocVar(trackerHash->lm, tracker); hashAddSaveName(trackerHash, ip, tracker, &tracker->name); tracker->lastAccess = now; tracker->curDelay = -penalty; slAddHead(&trackerList, tracker); } return tracker; } -int calcDelay(struct tracker *tracker) +int calcDelay(struct tracker *tracker, double fraction) /* Figure out recommended delay. */ { int timeDiff = now - tracker->lastAccess; int delay; if (timeDiff < 0) timeDiff = 0; -delay = tracker->curDelay + penalty - timeDiff*recovery; +delay = tracker->curDelay + (int)(fraction*penalty) - timeDiff*recovery; if (delay < 0) delay = 0; return delay; } void returnList(int socket) /* Send list of all things tracked down socket. */ { struct tracker *tracker; char buf[256]; int timeDiff; safef(buf, sizeof(buf), "#IP_ADDRESS\thits\ttime\tmax\tcurrent"); netSendString(socket, buf); for (tracker = trackerList; tracker != NULL; tracker = tracker->next) { timeDiff = now - tracker->lastAccess; safef(buf, sizeof(buf), "%s\t%d\t%d\t%d\t%d", tracker->name, tracker->accessCount, timeDiff, - tracker->maxDelay, calcDelay(tracker)); + tracker->maxDelay, calcDelay(tracker, 1.0)); if (!netSendString(socket, buf)) break; } netSendString(socket, ""); } void clearZombies() /* Clear any zombie processes */ { int stat; for (;;) { if (waitpid(-1, &stat, WNOHANG) <= 0) break; } @@ -177,88 +177,94 @@ if (sameString(command, "?set")) { char *ip = nextWord(&s); char *millis = nextWord(&s); if (millis != NULL) { tracker = trackerForIp(ip); tracker->curDelay = atoi(millis) - penalty; tracker->lastAccess = now; } } } } else { - tracker = trackerForIp(s); + char *ip = nextWord(&s); + char *fraction = nextWord(&s); + tracker = trackerForIp(ip); tracker->accessCount += 1; - tracker->curDelay = calcDelay(tracker); + if (fraction != NULL) + tracker->curDelay = calcDelay(tracker, atof(fraction)); + else + tracker->curDelay = calcDelay(tracker, 1.0); if (tracker->maxDelay < tracker->curDelay) tracker->maxDelay = tracker->curDelay; tracker->lastAccess = now; safef(buf, sizeof(buf), "%d", tracker->curDelay); netSendString(socket, buf); } } close(socket); } } void forkOffServer() /* Close standard file handles and fork to be a better daemon. */ { /* Close standard file handles. */ close(0); close(1); close(2); if (fork() == 0) { /* Execute daemon. */ startServer(); } } -void queryServer(char *ip, int count) +void queryServer(char *ip, int count, double fraction) /* Query bottleneck server - just for testing. * Main query is over ip port. */ { int i; +char sendString[256]; +safef(sendString, sizeof(sendString), "%s %f", ip, fraction); for (i=0; i<count; ++i) { int socket = netMustConnect(host, port); char buf[256], *s; - netSendString(socket, ip); + netSendString(socket, sendString); s = netGetString(socket, buf); if (s == NULL) errAbort("Shut out by bottleneck server %s:%d", host, port); printf("%s millisecond delay recommended\n", s); close(socket); } } void setUser(char *ip, char *millis) /* Set user current delay. */ { int socket = netMustConnect(host, port); char buf[256]; safef(buf, sizeof(buf), "?set %s %s", ip, millis); netSendString(socket, buf); close(socket); } - void listAll() /* Ask server for info on all IP addresses. */ { int socket = netMustConnect(host, port); char buf[256], *s; netSendString(socket, "?"); for (;;) { s = netGetString(socket, buf); if (s == NULL || s[0] == 0) break; printf("%s\n", buf); } close(socket); } @@ -275,31 +281,34 @@ penalty = optionInt("penalty", penalty); recovery = optionInt("recovery", recovery); subnet = optionVal("subnet", subnet); command = argv[1]; if (sameString(command, "start")) { if (argc != 2) usage(); forkOffServer(); } else if (sameString(command, "query")) { int count = 1; if (argc > 3) count = atoi(argv[3]); - queryServer(argv[2], count); + double fraction = 1.0; + if (argc > 4) + fraction = atof(argv[4]); + queryServer(argv[2], count, fraction); } else if (sameString(command, "list")) { listAll(); } else if (sameString(command, "set")) { if (argc != 4) usage(); setUser(argv[2], argv[3]); } else { usage(); }