49ba5b9081440aaf210605cf61c748aff9a6ddc7
braney
Fri Nov 6 14:47:15 2015 -0800
a very basic instrumentation of UDC (no redmine)
diff --git src/lib/udc.c src/lib/udc.c
index 148b0d4..b0d078f 100644
--- src/lib/udc.c
+++ src/lib/udc.c
@@ -26,30 +26,31 @@
#include <sys/file.h>
#include "common.h"
#include "hash.h"
#include "obscure.h"
#include "bits.h"
#include "linefile.h"
#include "portable.h"
#include "sig.h"
#include "net.h"
#include "cheapcgi.h"
#include "udc.h"
#include "hex.h"
#include <dirent.h>
#include <openssl/sha.h>
+FILE *udcLogStream = NULL;
#define udcBlockSize (8*1024)
/* All fetch requests are rounded up to block size. */
#define udcMaxBytesPerRemoteFetch (udcBlockSize * 32)
/* Very large remote reads are broken down into chunks this size. */
struct connInfo
/* Socket descriptor and associated info, for keeping net connections open. */
{
int socket; /* Socket descriptor for data connection (or 0). */
bits64 offset; /* Current file offset of socket. */
int ctrlSocket; /* (FTP only) Control socket descriptor or 0. */
char *redirUrl; /* (HTTP(S) only) use redirected url */
};
@@ -65,53 +66,62 @@
bits64 size; /* Remote file size */
struct connInfo ci; /* Connection info for open net connection */
};
typedef boolean (*UdcInfoCallback)(char *url, struct udcRemoteFileInfo *retInfo);
/* Type for callback function that fetches file timestamp and size. */
struct udcProtocol
/* Something to handle a communications protocol like http, https, ftp, local file i/o, etc. */
{
struct udcProtocol *next; /* Next in list */
UdcDataCallback fetchData; /* Data fetcher */
UdcInfoCallback fetchInfo; /* Timestamp & size fetcher */
};
+struct ioStats
+/* Statistics concerning reads and seeks. */
+ {
+ bits64 numSeeks;
+ bits64 numReads;
+ bits64 bytesRead;
+ };
+
struct udcFile
/* A file handle for our caching system. */
{
struct udcFile *next; /* Next in list. */
char *url; /* Name of file - includes protocol */
char *protocol; /* The URL up to the first colon. http: etc. */
struct udcProtocol *prot; /* Protocol specific data and methods. */
time_t updateTime; /* Last modified timestamp. */
bits64 size; /* Size of file. */
bits64 offset; /* Current offset in file. */
char *cacheDir; /* Directory for cached file parts. */
char *bitmapFileName; /* Name of bitmap file. */
char *sparseFileName; /* Name of sparse data file. */
char *redirFileName; /* Name of redir file. */
int fdSparse; /* File descriptor for sparse data file. */
boolean sparseReadAhead; /* Read-ahead has something in the buffer */
char *sparseReadAheadBuf; /* Read-ahead buffer, if any */
bits64 sparseRAOffset; /* Read-ahead buffer offset */
struct udcBitmap *bits; /* udcBitMap */
bits64 startData; /* Start of area in file we know to have data. */
bits64 endData; /* End of area in file we know to have data. */
bits32 bitmapVersion; /* Version of associated bitmap we were opened with. */
struct connInfo connInfo; /* Connection info for open net connection. */
+ struct ioStats ioStats; /* Statistics on file access. */
};
struct udcBitmap
/* The control structure including the bitmap of blocks that are cached. */
{
struct udcBitmap *next; /* Next in list. */
bits32 blockSize; /* Number of bytes per block of file. */
bits64 remoteUpdate; /* Remote last update time. */
bits64 fileSize; /* File size */
bits32 version; /* Version - increments each time cache is stale. */
bits64 localUpdate; /* Time we last fetched new data into cache. */
bits64 localAccess; /* Time we last accessed data. */
boolean isSwapped; /* If true need to swap all bytes on read. */
int fd; /* File descriptor for file with current block. */
};
@@ -949,30 +959,32 @@
}
}
freeMem(newUrl);
}
}
struct udcFile *udcFileMayOpen(char *url, char *cacheDir)
/* Open up a cached file. cacheDir may be null in which case udcDefaultDir() will be
* used. Return NULL if file doesn't exist.
* Caching is inactive if defaultDir is NULL or the protocol is "transparent".
* */
{
if (cacheDir == NULL)
cacheDir = udcDefaultDir();
verbose(4, "udcfileOpen(%s, %s)\n", url, cacheDir);
+if (udcLogStream)
+ fprintf(udcLogStream, "Open %s\n", url);
/* Parse out protocol. Make it "transparent" if none specified. */
char *protocol = NULL, *afterProtocol = NULL, *colon;
boolean isTransparent = FALSE;
udcParseUrl(url, &protocol, &afterProtocol, &colon);
if (!colon)
{
freeMem(protocol);
protocol = cloneString("transparent");
freeMem(afterProtocol);
afterProtocol = cloneString(url);
isTransparent = TRUE;
}
struct udcProtocol *prot;
prot = udcProtocolNew(protocol);
@@ -1077,30 +1089,35 @@
freeMem(file->cacheDir);
freeMem(file->bitmapFileName);
freeMem(file->sparseFileName);
freeMem(file);
freeMem(protocol);
freeMem(afterProtocol);
return list;
}
void udcFileClose(struct udcFile **pFile)
/* Close down cached file. */
{
struct udcFile *file = *pFile;
if (file != NULL)
{
+ if (udcLogStream)
+ {
+ fprintf(udcLogStream, "Close %s %lld %lld %lld\n", file->url,
+ file->ioStats.numSeeks, file->ioStats.numReads, file->ioStats.bytesRead);
+ }
if (file->connInfo.socket != 0)
mustCloseFd(&(file->connInfo.socket));
if (file->connInfo.ctrlSocket != 0)
mustCloseFd(&(file->connInfo.ctrlSocket));
freeMem(file->url);
freeMem(file->protocol);
udcProtocolFree(&file->prot);
freeMem(file->cacheDir);
freeMem(file->bitmapFileName);
freeMem(file->sparseFileName);
freeMem(file->sparseReadAheadBuf);
if (file->fdSparse != 0)
mustCloseFd(&(file->fdSparse));
udcBitmapClose(&file->bits);
}
@@ -1383,35 +1400,37 @@
{
ok = FALSE;
verbose(4, "udcCachePreload version check failed %d vs %d",
bits->version, file->bitmapVersion);
}
if (!ok)
break;
}
return ok;
}
#define READAHEADBUFSIZE 4096
bits64 udcRead(struct udcFile *file, void *buf, bits64 size)
/* Read a block from file. Return amount actually read. */
{
+file->ioStats.numReads++;
// if not caching, just fetch the data
if (!udcCacheEnabled() && !sameString(file->protocol, "transparent"))
{
int actualSize = file->prot->fetchData(file->url, file->offset, size, buf, &(file->connInfo));
file->offset += actualSize;
+ file->ioStats.bytesRead += actualSize;
return actualSize;
}
/* Figure out region of file we're going to read, and clip it against file size. */
bits64 start = file->offset;
if (start > file->size)
return 0;
bits64 end = start + size;
if (end > file->size)
end = file->size;
size = end - start;
char *cbuf = buf;
/* use read-ahead buffer if present */
bits64 bytesRead = 0;
@@ -1423,30 +1442,31 @@
if (file->sparseReadAhead)
{
raStart = file->sparseRAOffset;
raEnd = raStart+READAHEADBUFSIZE;
if (start >= raStart && start < raEnd)
{
// copy bytes out of rabuf
bits64 endInBuf = min(raEnd, end);
bits64 sizeInBuf = endInBuf - start;
memcpy(cbuf, file->sparseReadAheadBuf + (start-raStart), sizeInBuf);
cbuf += sizeInBuf;
bytesRead += sizeInBuf;
start = raEnd;
size -= sizeInBuf;
file->offset += sizeInBuf;
+ file->ioStats.bytesRead += sizeInBuf;
if (size == 0)
break;
}
file->sparseReadAhead = FALSE;
mustLseek(file->fdSparse, start, SEEK_SET);
}
bits64 saveEnd = end;
if (size < READAHEADBUFSIZE)
{
file->sparseReadAhead = TRUE;
if (!file->sparseReadAheadBuf)
file->sparseReadAheadBuf = needMem(READAHEADBUFSIZE);
file->sparseRAOffset = start;
size = READAHEADBUFSIZE;
@@ -1476,30 +1496,31 @@
* case other code is moving around file pointer. */
mustLseek(file->fdSparse, start, SEEK_SET);
}
if (file->sparseReadAhead)
{
mustReadFd(file->fdSparse, file->sparseReadAheadBuf, size);
end = saveEnd;
size = end - start;
}
else
{
mustReadFd(file->fdSparse, cbuf, size);
file->offset += size;
+ file->ioStats.bytesRead += size;
bytesRead += size;
break;
}
}
return bytesRead;
}
void udcMustRead(struct udcFile *file, void *buf, bits64 size)
/* Read a block from file. Abort if any problem, including EOF before size is read. */
{
bits64 sizeRead = udcRead(file, buf, size);
if (sizeRead < size)
errAbort("udc couldn't read %llu bytes from %s, did read %llu", size, file->url, sizeRead);
}
@@ -1648,38 +1669,40 @@
}
struct lineFile *udcWrapShortLineFile(char *url, char *cacheDir, size_t maxSize)
/* Read in entire short (up to maxSize) url into memory and wrap a line file around it.
* The cacheDir may be null in which case udcDefaultDir() will be used. If maxSize
* is zero then a default value (currently 64 meg) will be used. */
{
if (maxSize == 0) maxSize = 64 * 1024 * 1024;
char *buf = udcFileReadAll(url, cacheDir, maxSize, NULL);
return lineFileOnString(url, TRUE, buf);
}
void udcSeekCur(struct udcFile *file, bits64 offset)
/* Seek to a particular position in file. */
{
+file->ioStats.numSeeks++;
file->offset += offset;
if (udcCacheEnabled())
mustLseek(file->fdSparse, offset, SEEK_CUR);
}
void udcSeek(struct udcFile *file, bits64 offset)
/* Seek to a particular position in file. */
{
+file->ioStats.numSeeks++;
file->offset = offset;
if (udcCacheEnabled())
mustLseek(file->fdSparse, offset, SEEK_SET);
}
bits64 udcTell(struct udcFile *file)
/* Return current file position. */
{
return file->offset;
}
static long bitRealDataSize(char *fileName)
/* Return number of real bytes indicated by bitmaps */
{
struct udcBitmap *bits = udcBitmapOpen(fileName);
@@ -1825,15 +1848,20 @@
/* return true if file is not a http or ftp file, just a local file */
{
// copied from above
char *protocol = NULL, *afterProtocol = NULL, *colon;
udcParseUrl(url, &protocol, &afterProtocol, &colon);
freez(&protocol);
freez(&afterProtocol);
return colon==NULL;
}
boolean udcExists(char *url)
/* return true if a local or remote file exists */
{
return udcFileSize(url)!=-1;
}
+
+void udcSetLog(FILE *fp)
+{
+udcLogStream = fp;
+}