49ba5b9081440aaf210605cf61c748aff9a6ddc7 braney Fri Nov 6 14:47:15 2015 -0800 a very basic instrumentation of UDC (no redmine) diff --git src/inc/udc.h src/inc/udc.h index e7e5517..ae3e026 100644 --- src/inc/udc.h +++ src/inc/udc.h @@ -1,168 +1,170 @@ /* udc - url data cache - a caching system that keeps blocks of data fetched from URLs in * sparse local files for quick use the next time the data is needed. * * This cache is enormously simplified by there being no local _write_ to the cache, * just reads. * * The overall strategy of the implementation is to have a root cache directory * with a subdir for each file being cached. The directory for a single cached file * contains two files - "bitmap" and "sparseData" that contains information on which * parts of the URL are cached and the actual cached data respectively. The subdirectory name * associated with the file is constructed from the URL in a straightforward manner. * http://genome.ucsc.edu/cgi-bin/hgGateway * gets mapped to: * rootCacheDir/http/genome.ucsc.edu/cgi-bin/hgGateway/ * The URL protocol is the first directory under the root, and the remainder of the * URL, with some necessary escaping, is used to define the rest of the cache directory * structure, with each '/' after the protocol line translating into another directory * level. * * The bitmap file contains time stamp and size data as well as an array with one bit * for each block of the file that has been fetched. Currently the block size is 8K. */ #ifndef UDC_H #define UDC_H struct udcFile; /* Handle to a cached file. Inside of structure mysterious unless you are udc.c. */ struct udcFile *udcFileMayOpen(char *url, char *cacheDir); /* Open up a cached file. cacheDir may be null in which case udcDefaultDir() will be * used. Return NULL if file doesn't exist. */ struct udcFile *udcFileOpen(char *url, char *cacheDir); /* Open up a cached file. cacheDir may be null in which case udcDefaultDir() will be * used. Abort if if file doesn't exist. */ void udcFileClose(struct udcFile **pFile); /* Close down cached file. */ bits64 udcRead(struct udcFile *file, void *buf, bits64 size); /* Read a block from file. Return amount actually read. */ #define udcReadOne(file, var) udcRead(file, &(var), sizeof(var)) /* Read one variable from file or die. */ void udcMustRead(struct udcFile *file, void *buf, bits64 size); /* Read a block from file. Abort if any problem, including EOF before size is read. */ #define udcMustReadOne(file, var) udcMustRead(file, &(var), sizeof(var)) /* Read one variable from file or die. */ bits64 udcReadBits64(struct udcFile *file, boolean isSwapped); /* Read and optionally byte-swap 64 bit entity. */ bits32 udcReadBits32(struct udcFile *file, boolean isSwapped); /* Read and optionally byte-swap 32 bit entity. */ bits16 udcReadBits16(struct udcFile *file, boolean isSwapped); /* Read and optionally byte-swap 16 bit entity. */ float udcReadFloat(struct udcFile *file, boolean isSwapped); /* Read and optionally byte-swap floating point number. */ double udcReadDouble(struct udcFile *file, boolean isSwapped); /* Read and optionally byte-swap double-precision floating point number. */ int udcGetChar(struct udcFile *file); /* Get next character from file or die trying. */ char *udcReadLine(struct udcFile *file); /* Fetch next line from udc cache. */ char *udcReadStringAndZero(struct udcFile *file); /* Read in zero terminated string from file. Do a freeMem of result when done. */ char *udcFileReadAll(char *url, char *cacheDir, size_t maxSize, size_t *retSize); /* Read a complete file via UDC. The cacheDir may be null in which case udcDefaultDir() * will be used. If maxSize is non-zero, check size against maxSize * and abort if it's bigger. Returns file data (with an extra terminal for the * common case where it's treated as a C string). If retSize is non-NULL then * returns size of file in *retSize. Do a freeMem or freez of the returned buffer * when done. */ struct lineFile *udcWrapShortLineFile(char *url, char *cacheDir, size_t maxSize); /* Read in entire short (up to maxSize) url into memory and wrap a line file around it. * The cacheDir may be null in which case udcDefaultDir() will be used. If maxSize * is zero then a default value (currently 64 meg) will be used. */ void udcSeek(struct udcFile *file, bits64 offset); /* Seek to a particular (absolute) position in file. */ void udcSeekCur(struct udcFile *file, bits64 offset); /* Seek to a particular (from current) position in file. */ bits64 udcTell(struct udcFile *file); /* Return current file position. */ bits64 udcCleanup(char *cacheDir, double maxDays, boolean testOnly); /* Remove cached files older than maxDays old. If testOnly is set * no clean up is done, but the size of the files that would be * cleaned up is still. */ void udcParseUrl(char *url, char **retProtocol, char **retAfterProtocol, char **retColon); /* Parse the URL into components that udc treats separately. * *retAfterProtocol is Q-encoded to keep special chars out of filenames. * Free all *ret's except *retColon when done. */ void udcParseUrlFull(char *url, char **retProtocol, char **retAfterProtocol, char **retColon, char **retAuth); /* Parse the URL into components that udc treats separately. * *retAfterProtocol is Q-encoded to keep special chars out of filenames. * Free all *ret's except *retColon when done. */ char *udcDefaultDir(); /* Get default directory for cache. Use this for the udcFileOpen call if you * don't have anything better.... */ void udcSetDefaultDir(char *path); /* Set default directory for cache */ void udcDisableCache(); /* Switch off caching. Re-enable with udcSetDefaultDir */ #define udcDevicePrefix "udc:" /* Prefix used by convention to indicate a file should be accessed via udc. This is * followed by the local path name or a url, so in common practice you see things like: * udc:http://genome.ucsc.edu/goldenPath/hg18/tracks/someTrack.bb */ struct slName *udcFileCacheFiles(char *url, char *cacheDir); /* Return low-level list of files used in cache. */ char *udcPathToUrl(const char *path, char *buf, size_t size, char *cacheDir); /* Translate path into an URL, store in buf, return pointer to buf if successful * and NULL if not. */ long long int udcSizeFromCache(char *url, char *cacheDir); /* Look up the file size from the local cache bitmap file, or -1 if there * is no cache for url. */ unsigned long udcCacheAge(char *url, char *cacheDir); /* Return the age in seconds of the oldest cache file. If a cache file is * missing, return the current time (seconds since the epoch). */ int udcCacheTimeout(); /* Get cache timeout (if local cache files are newer than this many seconds, * we won't ping the remote server to check the file size and update time). */ void udcSetCacheTimeout(int timeout); /* Set cache timeout (if local cache files are newer than this many seconds, * we won't ping the remote server to check the file size and update time). */ time_t udcUpdateTime(struct udcFile *udc); /* return udc->updateTime */ boolean udcFastReadString(struct udcFile *f, char buf[256]); /* Read a string into buffer, which must be long enough * to hold it. String is in 'writeString' format. */ off_t udcFileSize(char *url); /* fetch remote or loca file size from given URL or path */ boolean udcExists(char *url); /* return true if a remote or local file exists */ boolean udcIsLocal(char *url); /* return true if url is not a http or ftp file, just a normal local file path */ +void udcSetLog(FILE *fp); +/* Tell UDC where to log its statistics. */ #endif /* UDC_H */