6b5d81231f6dcb4895d9522b80b2552711a4e87e max Tue Aug 6 03:22:14 2024 -0700 adding resolvPrefix to our hg.conf statements for the udc resolver, refs #34240 diff --git src/lib/udc.c src/lib/udc.c index df93499..7ef70c0 100644 --- src/lib/udc.c +++ src/lib/udc.c @@ -159,50 +159,61 @@ bits64 localUpdate; /* Time we last fetched new data into cache. */ bits64 localAccess; /* Time we last accessed data. */ boolean isSwapped; /* If true need to swap all bytes on read. */ int fd; /* File descriptor for file with current block. */ }; static char *bitmapName = "bitmap"; static char *sparseDataName = "sparseData"; static char *redirName = "redir"; static char *resolvedName = "resolv"; #define udcBitmapHeaderSize (64) static int cacheTimeout = 0; #define MAX_SKIP_TO_SAVE_RECONNECT (udcMaxBytesPerRemoteFetch / 2) -/* pseudo-URLs with this prefix (e.g. "s3://" get run through a command to get resolved to real HTTPS URLs) */ +/* pseudo-URLs with this protocol (e.g. "s3://" get run through a command to get resolved to real HTTPS URLs) */ struct slName *resolvProts = NULL; +/* pseudo-URLs that start with this prefix (e.g. "https://myserver.okta.com" get run through a command to get resolved to real HTTPS URLs) */ +static char *resolvPrefix = NULL; +/* this command will get run if either the prefix or the protocol matches */ static char *resolvCmd = NULL; bool udcIsResolvable(char *url) /* check if third-party protocol resolving (e.g. for "s3://") is enabled and if the url starts with a protocol handled by the resolver */ { -if (!resolvProts || !resolvCmd) +if (!resolvCmd) return FALSE; char *colon = strchr(url, ':'); if (!colon) return FALSE; +bool isFound = FALSE; + +if (resolvPrefix && startsWithNoCase(resolvPrefix, url)) + isFound = TRUE; +else + { int colonPos = colon - url; char *protocol = cloneStringZ(url, colonPos); -bool isFound = (slNameFind(resolvProts, protocol) != NULL); + isFound = (protocol && resolvProts && slNameFind(resolvProts, protocol) != NULL); + freez(&protocol); + } + if (isFound) verbose(4, "Check: URL %s has special protocol://, will need resolving\n", url); -freez(&protocol); return isFound; } static off_t ourMustLseek(struct ioStats *ioStats, int fd, off_t offset, int whence) { ioStats->numSeeks++; return mustLseek(fd, offset, whence); } static void ourMustWrite(struct ioStats *ioStats, int fd, void *buf, size_t size) { ioStats->numWrites++; ioStats->bytesWritten += size; mustWriteFd(fd, buf, size); @@ -450,34 +461,35 @@ sleep1000(500); struct stat status; int ret = stat(fileName, &status); if (ret < 0) return FALSE; retInfo->updateTime = status.st_mtime; retInfo->size = status.st_size; return TRUE; } /********* Section for http protocol **********/ static char *defaultDir = "/tmp/udcCache"; static bool udcInitialized = FALSE; -void udcSetResolver(char *prots, char *cmd) +void udcSetResolver(char *prots, char *prefix, char *cmd) /* Set protocols and local wrapper program to resolve s3:// and similar URLs to HTTPS */ { resolvProts = slNameListFromString(cloneString(prots), ','); + resolvPrefix = trimSpaces(cloneString(prefix)); resolvCmd = trimSpaces(cloneString(cmd)); } static void initializeUdc() /* Use the $TMPDIR environment variable, if set, to amend the default location * of the cache */ { if (udcInitialized) return; char *tmpDir = getenv("TMPDIR"); if (isNotEmpty(tmpDir)) { char buffer[2048]; safef(buffer, sizeof(buffer), "%s/udcCache", tmpDir); udcSetDefaultDir(buffer);