eb076410a62ea6a94b71358424080af261ba2c25
jcasper
  Thu Mar 26 03:20:14 2026 -0700
Moved hgFetch into hgTrackUi; added etags, Cache-Control directives, and GET use instead of POST
when udcTimeout isn't set, all to support browser caching of remote files; bumped the DataTables version for facetedComposite
to facilitate making all the checkboxes have a consistent look; refs #36320

diff --git src/hg/hgTrackUi/hgTrackUi.c src/hg/hgTrackUi/hgTrackUi.c
index 35ab2b31f77..3b0e375e388 100644
--- src/hg/hgTrackUi/hgTrackUi.c
+++ src/hg/hgTrackUi/hgTrackUi.c
@@ -42,30 +42,33 @@
 #include "pcrResult.h"
 #include "dgv.h"
 #include "transMapStuff.h" 
 #include "vcfUi.h" 
 #include "bbiFile.h"
 #include "ensFace.h"
 #include "microarray.h"
 #include "trackVersion.h"
 #include "gtexUi.h"
 #include "genbank.h"
 #include "botDelay.h"
 #include "customComposite.h"
 #include "hicUi.h"
 #include "decoratorUi.h"
 #include "genark.h"
+#include "cart.h"
+#include "filePath.h"
+#include "md5.h"
     
 #ifdef USE_HAL 
 #include "halBlockViz.h"
 #endif 
 
 #define MAIN_FORM "mainForm"
 #define WIGGLE_HELP_PAGE  "../goldenPath/help/hgWiggleTrackHelp.html"
 
 /* for earlyBotCheck() function at the beginning of main() */
 #define delayFraction   0.25    /* standard penalty is 1.0 for most CGIs */
                                 /* this one is 0.25 */
 static boolean issueBotWarning = FALSE;
 
 struct cart *cart = NULL;	/* Cookie cart with UI settings */
 char *database = NULL;		/* Current database. */
@@ -3218,30 +3221,32 @@
                 printf("%s\"%.*s\"", COMMA_IF(not_first), nameLen, nameStart);
                 }
             }
         }
     hashElFreeList(&elList);
     }
 printf(closeDataElementsJSON);
 printf(",\"mdid\": \"%s\"", metaDataId);
 printf(",\"primaryKey\": \"%s\"", primaryKey);  // must exist
 if (maxCheckboxes) // only if present in trackDb.settings entry
     printf(",\"maxCheckboxes\": \"%s\"", maxCheckboxes);
 if (colorSettingsUrl) // only if present in trackDb.settings entry
     printf(",\"colorSettingsUrl\": \"%s\"", cgiEncode((char*) colorSettingsUrl));
 printf(",\"metadataUrl\": \"%s\"", cgiEncode((char*) metaDataUrl));
 printf(",\"track\": \"%s\"", tdb->track);
+if (isNotEmpty(cartOptionalString(cart, "udcTimeout")))
+    printf(",\"udcTimeout\": true");
 printf(closeJSON);
 /* --- END embedded JSON data --- */
 
 printf(metadataTableScriptElement);
 
 // cleanup
 slPairFreeValsAndList(&dataTypes);
 hashFree(&defaultOn);
 }
 
 void specificUi(struct trackDb *tdb, struct trackDb *tdbList, struct customTrack *ct, boolean ajax)
 /* Draw track specific parts of UI. */
 {
 char *track = tdb->track;
 char *db = database;
@@ -4095,33 +4100,172 @@
        }
    else if (sameString(operation, "undupe"))
        {
        newTrack = dupTrackSkipToSourceName(track);
        undupTrackInCartAndTrash(track, cart);
        }
    else
        {
        internalErr();
        }
    cartRemove(cart, opVar);
    }
 return newTrack;
 }
 
+/* Setting names whose file contents are safe to serve via hgFetch.
+ * Only admin-configured (native track) values are checked -- never hub or custom tracks.
+ * Do NOT add bigDataUrl or bigDataIndex here -- those may be restricted (we
+ * might change this later to instead respect the tableBrowser setting in trackDb). */
+static char *fetchableSettings[] = {"metaDataUrl", "colorSettingsUrl", NULL};
+
+boolean fileUrlMatchesHub(char *fileUrl, struct hubConnectStatus *hubStatus)
+/* Ignores fetchableSettings for now, whitelisting anything that sits inside
+ * the hub.txt directory structure.  Assumes fileUrl has been canonicalized. */
+{
+char baseDir[2048];
+splitPath(hubStatus->hubUrl, baseDir, NULL, NULL);
+return startsWith(baseDir, fileUrl);
+}
+
+static boolean fileUrlMatchesTrackSetting(char *fileUrl, struct trackDb *tdb)
+/* Check if fileUrl matches any whitelisted setting in this trackDb.
+ * Assumes fileUrl has been canonicalized. */
+{
+char **p;
+for (p = fetchableSettings; *p != NULL; p++)
+    {
+    char *val = trackDbSetting(tdb, *p);
+    if (val != NULL && sameString(val, fileUrl))
+        return TRUE;
+    }
+return FALSE;
+}
+
+void handleFileFetch(struct cart *cart)
+/* Checks if a requested file is a legal request based on an attached cart or
+ * native track.  If so, retrieves the file content via UDC and retransmits
+ * it as the page content. */
+{
+char *genome = NULL;
+getDbAndGenome(cart, &database, &genome, NULL);
+initGenbankTableNames(database);
+//knetUdcInstall();
+
+char *fileUrl = cartOptionalString(cart, "fileUrl");
+if (fileUrl == NULL)
+    {
+    puts("Status: 400 Bad Request");
+    errAbort("Missing required parameter: fileUrl");
+    }
+
+char *urlClone = cloneString(fileUrl);
+cgiDecode(urlClone, urlClone, strlen(urlClone));
+fileUrl = resolveDotDots(urlClone);
+freeMem(urlClone);
+
+boolean matchFound = FALSE;
+
+// Check if fileUrl falls under a connected hub's base directory
+struct slName *hubIds = hubConnectHubsInCart(cart);
+struct slName *thisHubId = hubIds;
+while (thisHubId != NULL)
+    {
+    struct hubConnectStatus *hubStatus = hubFromId(sqlUnsigned(thisHubId->name));
+    if (fileUrlMatchesHub(fileUrl, hubStatus))
+        {
+        matchFound = TRUE;
+        break;
+        }
+    thisHubId = thisHubId->next;
+    }
+
+// For native database tracks (not hub or custom tracks), check if fileUrl matches
+// a whitelisted trackDb setting.  Only native tracks are checked here because their
+// settings are admin-configured and trusted.  Hub and custom track settings are
+// user-controlled and could be used for SSRF attacks.
+if (!matchFound)
+    {
+    char *track = cartOptionalString(cart, "track");
+    char *sourceDb = cartOptionalString(cart, "sourceDb"); // for future quickLift use
+    if (sourceDb == NULL)
+        sourceDb = database;
+    if (track != NULL && !isHubTrack(track) && !isCustomTrack(track))
+        {
+        struct trackDb *tdb = tdbForTrack(sourceDb, track, NULL);
+        if (tdb != NULL)
+            matchFound = fileUrlMatchesTrackSetting(fileUrl, tdb);
+        }
+    }
+
+if (!matchFound)
+    {
+    puts("Status: 400 Bad Request");
+    errAbort("Supplied fileUrl does not match any connected hubs or track settings.");
+    }
+
+// By now we know that fileUrl points to something valid to fetch and return to the user.
+// Now we just have to fetch the file contents and retransmit it.
+
+int timeout = cartUsualInt(cart, "udcTimeout", 300);
+if (udcCacheTimeout() < timeout)
+    udcSetCacheTimeout(timeout);
+
+char maxAge[1024];
+safef(maxAge, sizeof(maxAge), "max-age=%d", timeout);
+printf("Cache-Control: %s\n", maxAge);
+
+// See if we're getting a "has it changed" request.
+// If so, return a 304 if nothing changed.
+char etag[1024];
+struct udcFile *udc = udcFileOpen(fileUrl, NULL);
+time_t mtime = udcUpdateTime(udc);
+safef(etag, sizeof(etag), "\"%ld\"", mtime);
+printf("ETag: %s\n", etag);
+udcFileClose(&udc);
+
+char *ifNone = getenv("HTTP_IF_NONE_MATCH");
+if (isNotEmpty(ifNone))
+    {
+    if (sameStringN(etag, ifNone, strlen(etag)-1)) // Apache can add -gzip to etags during transmission
+        {
+        puts("Status: 304 Not Modified\n");
+        freeMem(fileUrl);
+        return;
+        }
+    }
+
+puts("Content-Type: text/plain\n");
+char *content = udcFileReadAll(fileUrl, NULL, 0, NULL);
+puts(content);
+freeMem(content);
+freeMem(fileUrl);
+}
+
 void doMiddle(struct cart *theCart)
 /* Write body of web page. */
 {
+boolean isFileFetch = isNotEmpty(cartOptionalString(theCart, "fileUrl"));
+
+if (isFileFetch)
+    {
+    handleFileFetch(theCart);  // file fetch workaround for CORS issues
+    return;
+    }
+else
+    cartWriteHeaderAndCont(theCart, NULL, NULL); // "normal" hgTrackUi
+
 struct trackDb *tdbList = NULL;
 struct trackDb *tdb = NULL;
 char *track;
 struct customTrack *ct = NULL, *ctList = NULL;
 char *ignored;
 
 /* used to have hgBotDelayFrac(0.25) here, replaced with earlyBotCheck()
  * at the beginning of main() to output message here if in delay time
  * 2021-06-21 - Hiram
  */
 if (issueBotWarning)
     {
     char *ip = getenv("REMOTE_ADDR");
     botDelayMessage(ip, botDelayMillis);
     }
@@ -4246,28 +4390,28 @@
         }
     else
         safef(title, sizeof title, "%s", tdb->shortLabel);
     char *titleEnd = (tdbIsSuper(tdb) ? "Tracks" :
                tdbIsDownloadsOnly(tdb) ? DOWNLOADS_ONLY_TITLE : "Track Settings");
     htmlNoEscape();     // allow HTML tags to format title blue bar (using short label)
     cartWebStart(cart, database, "%s %s", title, titleEnd);
     htmlDoEscape();
     trackUi(tdb, tdbList, ct, FALSE);
     printf("<BR>\n");
     jsonPrintGlobals();
     webEnd();
     }
 }
 
-char *excludeVars[] = { "submit", "Submit", "g", NULL, "ajax", NULL,};
+char *excludeVars[] = { "submit", "Submit", "g", "fileUrl", "track", "sourceDb", NULL, "ajax", NULL,};
 
 int main(int argc, char *argv[])
 /* Process command line. */
 {
 long enteredMainTime = clock1000();
 /* 0, 0, == use default 10 second for warning, 20 second for immediate exit */
 issueBotWarning = earlyBotCheck(enteredMainTime, "hgTrackUi", delayFraction, 0, 0, "html");
 cgiSpoof(&argc, argv);
-cartEmptyShell(doMiddle, hUserCookie(), excludeVars, NULL);
+cartEmptyShellNoContent(doMiddle, hUserCookie(), excludeVars, NULL);
 cgiExitTime("hgTrackUi", enteredMainTime);
 return 0;
 }