a9fde73d32daf74780765442de44324061b01d66
markd
  Sun Jan 22 22:13:52 2023 -0800
Add URL resolver plugin functionality to allow an external program to convert
cloud URLs (s3:, gs: drs:, or really any non-HTTP RUL) to http/https URLs.
This can include signed URLs.  The cloud URL is used to index the UDC cache
rather than the resolved URL.  This allows for re-resolving signed URLs
if they time out.

Joint work by Max and Markd

diff --git src/inc/udc.h src/inc/udc.h
index ddfcf58..71ca230 100644
--- src/inc/udc.h
+++ src/inc/udc.h
@@ -61,30 +61,38 @@
 float udcReadFloat(struct udcFile *file, boolean isSwapped);
 /* Read and optionally byte-swap floating point number. */
 
 double udcReadDouble(struct udcFile *file, boolean isSwapped);
 /* Read and optionally byte-swap double-precision floating point number. */
 
 int udcGetChar(struct udcFile *file);
 /* Get next character from file or die trying. */
 
 char *udcReadLine(struct udcFile *file);
 /* Fetch next line from udc cache. */
 
 char *udcReadStringAndZero(struct udcFile *file);
 /* Read in zero terminated string from file.  Do a freeMem of result when done. */
 
+char *udcFileReadAllIfExists(char *url, char *cacheDir, size_t maxSize, size_t *retSize);
+/* Read a complete file via UDC. Return NULL if the file doesn't exist.  The
+ * cacheDir may be null in which case udcDefaultDir() will be used.  If
+ * maxSize is non-zero, check size against maxSize and abort if it's bigger.
+ * Returns file data (with an extra terminal for the common case where it's
+ * treated as a C string).  If retSize is non-NULL then returns size of file
+ * in *retSize. Do a freeMem or freez of the returned buffer when done. */
+
 char *udcFileReadAll(char *url, char *cacheDir, size_t maxSize, size_t *retSize);
 /* Read a complete file via UDC. The cacheDir may be null in which case udcDefaultDir()
  * will be used.  If maxSize is non-zero, check size against maxSize
  * and abort if it's bigger.  Returns file data (with an extra terminal for the
  * common case where it's treated as a C string).  If retSize is non-NULL then
  * returns size of file in *retSize. Do a freeMem or freez of the returned buffer
  * when done. */
 
 struct lineFile *udcWrapShortLineFile(char *url, char *cacheDir, size_t maxSize);
 /* Read in entire short (up to maxSize) url into memory and wrap a line file around it.
  * The cacheDir may be null in which case udcDefaultDir() will be used.  If maxSize
  * is zero then a default value (currently 64 meg) will be used. */
 
 void udcSeek(struct udcFile *file, bits64 offset);
 /* Seek to a particular (absolute) position in file. */
@@ -174,23 +182,19 @@
 
 void udcSetLog(FILE *fp);
 /* Tell UDC where to log its statistics. */
 
 void udcMMap(struct udcFile *file);
 /* Enable access to underlying file as memory using mmap.  udcMMapFetch
  * must be called to actually access regions of the file. */
 
 void *udcMMapFetch(struct udcFile *file, bits64 offset, bits64 size);
 /* Return pointer to a region of the file in memory, ensuring that regions is
  * cached.. udcMMap must have been called to enable access.  This must be
  * called for first access to a range of the file or erroneous (zeros) data
  * maybe returned.  Maybe called multiple times on a range or overlapping
  * returns. */
 
-//void setUrlResolver(char *prot, char* cmd);
-/* Define a command that can be used to resolve a protocol like "s3" to a final URL */
-
-
 bool udcIsResolvable(char *url);
 /* check if third-party protocol resolving (e.g. for "s3://") is enabled and if a URL can be resolved this way to HTTP */
 
 #endif /* UDC_H */