src/hg/lib/customFactory.c 1.109
1.109 2009/11/09 22:31:15 angie
Added customFactoryEnableExtraChecking, intended to be called only by hgCustom. When the user first loads a custom track, it's a good time for more expensive testing (like opening remote BAM files). However, hgTracks etc. can save time by not duplicating that work every time they parse existing tracks. Currently only bamLoader uses this.
Index: src/hg/lib/customFactory.c
===================================================================
RCS file: /projects/compbio/cvsroot/kent/src/hg/lib/customFactory.c,v
retrieving revision 1.108
retrieving revision 1.109
diff -b -B -U 1000000 -r1.108 -r1.109
--- src/hg/lib/customFactory.c 5 Nov 2009 18:48:36 -0000 1.108
+++ src/hg/lib/customFactory.c 9 Nov 2009 22:31:15 -0000 1.109
@@ -1,2406 +1,2420 @@
/* customFactory - a polymorphic object for handling
* creating various types of custom tracks. */
#include "common.h"
#include "errCatch.h"
#include "hash.h"
#include "linefile.h"
#include "portable.h"
#include "obscure.h"
#include "pipeline.h"
#include "jksql.h"
#include "net.h"
#include "bed.h"
#include "psl.h"
#include "gff.h"
#include "wiggle.h"
#include "genePred.h"
#include "trackDb.h"
#include "hgConfig.h"
#include "hdb.h"
#include "hui.h"
#include "customTrack.h"
#include "customPp.h"
#include "customFactory.h"
#include "trashDir.h"
#include "jsHelper.h"
#include "encode/encodePeak.h"
#include "udc.h"
#include "bigWig.h"
#include "bigBed.h"
#ifdef USE_BAM
#include "bamFile.h"
#endif//def USE_BAM
static char const rcsid[] = "$Id$";
+static boolean doExtraChecking = FALSE;
+
/*** Utility routines used by many factories. ***/
char *customFactoryNextTilTrack(struct customPp *cpp)
/* Return next line. Return NULL at end of input or at line starting with
* "track." */
{
char *line = customPpNext(cpp);
if (line != NULL && startsWithWord("track", line))
{
customPpReuse(cpp, line);
line = NULL;
}
return line;
}
char *customFactoryNextRealTilTrack(struct customPp *cpp)
/* Return next "real" line (not blank, not comment).
* Return NULL at end of input or at line starting with
* "track." */
{
char *line = customPpNextReal(cpp);
if (line != NULL && startsWithWord("track", line))
{
customPpReuse(cpp, line);
line = NULL;
}
return line;
}
void customFactoryCheckChromNameDb(char *genomeDb, char *word, struct lineFile *lf)
/* Make sure it's a chromosome or a contig. Well, at the moment,
* just make sure it's a chromosome. */
{
if (!hgIsOfficialChromName(genomeDb, word))
lineFileAbort(lf, "%s not a recognized sequence (note: sequence names are case sensitive)",
word);
}
void customFactorySetupDbTrack(struct customTrack *track)
/* Fill in fields most database-resident custom tracks need. */
{
struct tempName tn;
char prefix[16];
static int dbTrackCount = 0;
struct sqlConnection *ctConn = hAllocConn(CUSTOM_TRASH);
++dbTrackCount;
safef(prefix, sizeof(prefix), "t%d", dbTrackCount);
track->dbTableName = sqlTempTableName(ctConn, prefix);
ctAddToSettings(track, "dbTableName", track->dbTableName);
trashDirFile(&tn, "ct", "ct", ".err");
track->dbStderrFile = cloneString(tn.forCgi);
track->dbDataLoad = TRUE;
track->dbTrack = TRUE;
hFreeConn(&ctConn);
}
static char *ctGenomeOrCurrent(struct customTrack *ct)
/* return database setting */
{
char *ctDb = ctGenome(ct);
if (ctDb == NULL)
ctDb = ct->genomeDb;
return ctDb;
}
static boolean sameType(char *a, char *b)
/* Case-sensitive comparison of first word if multiple words,
* so that we can compare types like "bed" vs. "bed 3 ." etc.
* Tolerates one null input but not two. */
{
if (a == NULL && b == NULL)
errAbort("sameType should not be called when both inputs are NULL.");
else if (a == NULL || b == NULL)
return FALSE;
char *aCopy = cloneString(a);
char *bCopy = cloneString(b);
char *aWord = firstWordInLine(aCopy);
char *bWord = firstWordInLine(bCopy);
boolean same = sameString(aWord, bWord);
freeMem(aCopy);
freeMem(bCopy);
return same;
}
/*** BED Factory ***/
static boolean rowIsBed(char **row, int wordCount, char *db)
/* Return TRUE if row is consistent with BED format. */
{
return wordCount >= 3 && wordCount <= bedKnownFields
&& hgIsOfficialChromName(db, row[0])
&& isdigit(row[1][0]) && isdigit(row[2][0]);
}
static boolean bedRecognizer(struct customFactory *fac,
struct customPp *cpp, char *type,
struct customTrack *track)
/* Return TRUE if looks like we're handling a bed track */
{
if (type != NULL && !sameType(type, fac->name))
return FALSE;
char *line = customFactoryNextRealTilTrack(cpp);
if (line == NULL)
return FALSE;
char *dupe = cloneString(line);
char *row[bedKnownFields+1];
int wordCount = chopLine(dupe, row);
char *ctDb = ctGenomeOrCurrent(track);
boolean isBed = rowIsBed(row, wordCount, ctDb);
freeMem(dupe);
if (isBed)
track->fieldCount = wordCount;
customPpReuse(cpp, line);
return isBed;
}
static boolean microarrayRecognizer(struct customFactory *fac,
struct customPp *cpp, char *type,
struct customTrack *track)
/* Return TRUE if looks like we're handling a microarray track */
{
return bedRecognizer(fac, cpp, type, track) && (track->fieldCount == 15);
}
static boolean coloredExonRecognizer(struct customFactory *fac,
struct customPp *cpp, char *type,
struct customTrack *track)
/* Return TRUE if looks like we're handling a colored-exon track */
{
return bedRecognizer(fac, cpp, type, track) && (track->fieldCount >= 14);
}
static struct pipeline *bedLoaderPipe(struct customTrack *track)
/* Set up pipeline that will load wig into database. */
{
/* running the single command:
* hgLoadBed -customTrackLoader -tmpDir=/data/tmp
* -maxChromNameLength=${nameLength} customTrash tableName stdin
* -customTrackLoader turns on options: -noNameIx -noHistory -ignoreEmpty
* -allowStartEqualEnd -allowNegativeScores -verbose=0
*/
struct dyString *tmpDy = newDyString(0);
char *cmd1[] = {"loader/hgLoadBed", "-customTrackLoader",
NULL, NULL, NULL, NULL, NULL, NULL, NULL};
char *tmpDir = cfgOptionDefault("customTracks.tmpdir", "/data/tmp");
struct stat statBuf;
int index = 2;
if (stat(tmpDir,&statBuf))
errAbort("can not find custom track tmp load directory: '%s'<BR>\n"
"create directory or specify in hg.conf customTracks.tmpdir", tmpDir);
dyStringPrintf(tmpDy, "-tmpDir=%s", tmpDir);
cmd1[index++] = dyStringCannibalize(&tmpDy); tmpDy = newDyString(0);
dyStringPrintf(tmpDy, "-maxChromNameLength=%d", track->maxChromName);
cmd1[index++] = dyStringCannibalize(&tmpDy); tmpDy = newDyString(0);
if(startsWithWord("bedGraph", track->dbTrackType))
{
/* we currently assume that last field is the bedGraph field. */
dyStringPrintf(tmpDy, "-bedGraph=%d", track->fieldCount);
cmd1[index++] = dyStringCannibalize(&tmpDy);
}
cmd1[index++] = CUSTOM_TRASH;
cmd1[index++] = track->dbTableName;
cmd1[index++] = "stdin";
assert(index <= ArraySize(cmd1));
/* the "/dev/null" file isn't actually used for anything, but it is used
* in the pipeLineOpen to properly get a pipe started that isn't simply
* to STDOUT which is what a NULL would do here instead of this name.
* This function exits if it can't get the pipe created
* The dbStderrFile will get stderr messages from hgLoadBed into the
* our private error log so we can send it back to the user
*/
return pipelineOpen1(cmd1, pipelineWrite | pipelineNoAbort,
"/dev/null", track->dbStderrFile);
}
static void pipelineFailExit(struct customTrack *track)
/* show up to three lines of error message to stderr and errAbort */
{
struct dyString *errDy = newDyString(0);
struct lineFile *lf;
char *line;
int i;
dyStringPrintf(errDy, "track load error (track name='%s'):<BR>\n", track->tdb->tableName);
lf = lineFileOpen(track->dbStderrFile, TRUE);
i = 0;
while( (i < 3) && lineFileNext(lf, &line, NULL))
{
dyStringPrintf(errDy, "%s<BR>\n", line);
++i;
// break out of loop after wibSizeLimit msg to avoid printing stuff from other commands in the pipe.
if(strstr(line, "wibSizeLimit"))
break;
}
lineFileClose(&lf);
if (i < 1)
dyStringPrintf(errDy, "unknown failure<BR>\n");
unlink(track->dbStderrFile);
errAbort("%s",dyStringCannibalize(&errDy));
}
static struct customTrack *bedFinish(struct customTrack *track,
boolean dbRequested)
/* Finish up bed tracks (and others that create track->bedList). */
{
/* Add type based on field count */
char buf[20];
safef(buf, sizeof(buf), "%s %d .", track->tdb->type != NULL && startsWithWord("bedGraph", track->tdb->type) ? "bedGraph" : "bed", track->fieldCount);
track->tdb->type = cloneString(buf);
track->dbTrackType = cloneString(buf);
safef(buf, sizeof(buf), "%d", track->fieldCount);
ctAddToSettings(track, "fieldCount", cloneString(buf));
/* If necessary add track offsets. */
int offset = track->offset;
if (offset != 0)
{
/* Add track offsets if any */
struct bed *bed;
for (bed = track->bedList; bed != NULL; bed = bed->next)
{
bed->chromStart += offset;
bed->chromEnd += offset;
if (track->fieldCount > 7)
{
bed->thickStart += offset;
bed->thickEnd += offset;
}
}
track->offset = 0; /* so DB load later won't do this again */
hashMayRemove(track->tdb->settingsHash, "offset"); /* nor the file reader*/
}
/* If necessary load database */
if (dbRequested)
{
customFactorySetupDbTrack(track);
struct pipeline *dataPipe = bedLoaderPipe(track);
FILE *out = pipelineFile(dataPipe);
struct bed *bed;
for (bed = track->bedList; bed != NULL; bed = bed->next)
{
bedOutputN(bed, track->fieldCount, out, '\t', '\n');
}
fflush(out); /* help see error from loader failure */
if(ferror(out) || pipelineWait(dataPipe))
pipelineFailExit(track); /* prints error and exits */
unlink(track->dbStderrFile); /* no errors, not used */
pipelineFree(&dataPipe);
}
return track;
}
static struct bed *customTrackBed(char *db, char *row[13], int wordCount,
struct hash *chromHash, struct lineFile *lf)
/* Convert a row of strings to a bed. */
{
struct bed * bed;
int count;
AllocVar(bed);
bed->chrom = hashStoreName(chromHash, row[0]);
customFactoryCheckChromNameDb(db, bed->chrom, lf);
bed->chromStart = lineFileNeedNum(lf, row, 1);
bed->chromEnd = lineFileNeedNum(lf, row, 2);
if (bed->chromEnd < 1)
lineFileAbort(lf, "chromEnd less than 1 (%d)", bed->chromEnd);
if (bed->chromEnd < bed->chromStart)
lineFileAbort(lf, "chromStart after chromEnd (%d > %d)",
bed->chromStart, bed->chromEnd);
if (wordCount > 3)
bed->name = cloneString(row[3]);
if (wordCount > 4)
bed->score = lineFileNeedNum(lf, row, 4);
if (wordCount > 5)
{
strncpy(bed->strand, row[5], sizeof(bed->strand));
if (bed->strand[0] != '+' && bed->strand[0] != '-' && bed->strand[0] != '.')
lineFileAbort(lf, "Expecting + or - in strand");
}
if (wordCount > 6)
bed->thickStart = lineFileNeedNum(lf, row, 6);
else
bed->thickStart = bed->chromStart;
if (wordCount > 7)
{
bed->thickEnd = lineFileNeedNum(lf, row, 7);
if (bed->thickEnd < bed->thickStart)
lineFileAbort(lf, "thickStart after thickEnd");
if ((bed->thickStart != 0) &&
((bed->thickStart < bed->chromStart) ||
(bed->thickStart > bed->chromEnd)))
lineFileAbort(lf,
"thickStart out of range (chromStart to chromEnd, or 0 if no CDS)");
if ((bed->thickEnd != 0) &&
((bed->thickEnd < bed->chromStart) ||
(bed->thickEnd > bed->chromEnd)))
lineFileAbort(lf,
"thickEnd out of range for %s:%d-%d, thick:%d-%d (chromStart to chromEnd, or 0 if no CDS)",
bed->name, bed->chromStart, bed->chromEnd,
bed->thickStart, bed->thickEnd);
}
else
bed->thickEnd = bed->chromEnd;
if (wordCount > 8)
{
char *comma;
/* Allow comma separated list of rgb values here */
comma = strchr(row[8], ',');
if (comma)
{
int rgb = bedParseRgb(row[8]);
if (rgb < 0)
lineFileAbort(lf,
"Expecting 3 comma separated numbers for r,g,b bed item color.");
else
bed->itemRgb = rgb;
}
else
bed->itemRgb = lineFileNeedNum(lf, row, 8);
}
if (wordCount > 9)
bed->blockCount = lineFileNeedNum(lf, row, 9);
if (wordCount > 10)
{
sqlSignedDynamicArray(row[10], &bed->blockSizes, &count);
if (count != bed->blockCount)
lineFileAbort(lf, "expecting %d elements in array", bed->blockCount);
}
if (wordCount > 11)
{
int i;
int lastEnd, lastStart;
sqlSignedDynamicArray(row[11], &bed->chromStarts, &count);
if (count != bed->blockCount)
lineFileAbort(lf, "expecting %d elements in array", bed->blockCount);
// tell the user if they appear to be using absolute starts rather than
// relative... easy to forget! Also check block order, coord ranges...
lastStart = -1;
lastEnd = 0;
for (i=0; i < bed->blockCount; i++)
{
/*
printf("%d:%d %s %s s:%d c:%u cs:%u ce:%u csI:%d bsI:%d ls:%d le:%d<BR>\n", lineIx, i, bed->chrom, bed->name, bed->score, bed->blockCount, bed->chromStart, bed->chromEnd, bed->chromStarts[i], bed->blockSizes[i], lastStart, lastEnd);
*/
if (bed->chromStarts[i]+bed->chromStart >= bed->chromEnd)
{
if (bed->chromStarts[i] >= bed->chromStart)
lineFileAbort(lf,
"BED chromStarts offsets must be relative to chromStart, "
"not absolute. Try subtracting chromStart from each offset "
"in chromStarts.");
else
lineFileAbort(lf,
"BED chromStarts[i]+chromStart must be less than chromEnd.");
}
lastStart = bed->chromStarts[i];
lastEnd = bed->chromStart + bed->chromStarts[i] + bed->blockSizes[i];
}
if (bed->chromStarts[0] != 0)
lineFileAbort(lf,
"BED blocks must span chromStart to chromEnd. "
"BED chromStarts[0] must be 0 (==%d) so that (chromStart + "
"chromStarts[0]) equals chromStart.", bed->chromStarts[0]);
i = bed->blockCount-1;
if ((bed->chromStart + bed->chromStarts[i] + bed->blockSizes[i]) !=
bed->chromEnd)
{
lineFileAbort(lf,
"BED blocks must span chromStart to chromEnd. (chromStart + "
"chromStarts[last] + blockSizes[last]) must equal chromEnd.");
}
}
if (wordCount > 12)
// get the microarray/colored-exon fields
{
bed->expCount = (int)sqlUnsigned(row[12]);
sqlSignedDynamicArray(row[13], &bed->expIds, &count);
if (count != bed->expCount)
lineFileAbort(lf, "expecting %d elements in expIds array (bed field 14)",
bed->expCount);
if (wordCount == 15)
{
sqlFloatDynamicArray(row[14], &bed->expScores, &count);
if (count != bed->expCount)
lineFileAbort(lf, "expecting %d elements in expScores array (bed field 15)",
bed->expCount);
}
}
return bed;
}
static struct customTrack *bedLoader(struct customFactory *fac,
struct hash *chromHash,
struct customPp *cpp, struct customTrack *track, boolean dbRequested)
/* Load up bed data until get next track line. */
{
char *line;
char *db = ctGenomeOrCurrent(track);
while ((line = customFactoryNextRealTilTrack(cpp)) != NULL)
{
char *row[bedKnownFields];
int wordCount = chopLine(line, row);
struct lineFile *lf = cpp->fileStack;
lineFileExpectAtLeast(lf, track->fieldCount, wordCount);
struct bed *bed = customTrackBed(db, row, wordCount, chromHash, lf);
slAddHead(&track->bedList, bed);
}
slReverse(&track->bedList);
return bedFinish(track, dbRequested);
}
static struct customTrack *bedGraphLoader(struct customFactory *fac,
struct hash *chromHash,
struct customPp *cpp, struct customTrack *track, boolean dbRequested)
/* Load up bedGraph data until get next track line. */
{
char buf[20];
bedLoader(fac, chromHash, cpp, track, dbRequested);
/* Trailing period in "bedGraph N ." confuses bedGraphMethods, so replace type */
freeMem(track->tdb->type);
safef(buf, sizeof(buf), "bedGraph %d", track->fieldCount);
track->tdb->type = cloneString(buf);
return track;
}
static struct customTrack *microarrayLoader(struct customFactory *fac,
struct hash *chromHash,
struct customPp *cpp, struct customTrack *track, boolean dbRequested)
/* Load up microarray data until get next track line. */
{
struct customTrack *ct = bedLoader(fac, chromHash, cpp, track, dbRequested);
freeMem(track->tdb->type);
/* /\* freeMem(track->dbTrackType); *\/ */
track->tdb->type = cloneString("array");
/* track->dbTrackType = cloneString("expRatio"); */
return ct;
}
static struct customTrack *coloredExonLoader(struct customFactory *fac,
struct hash *chromHash,
struct customPp *cpp, struct customTrack *track, boolean dbRequested)
/* Load up microarray data until get next track line. */
{
struct customTrack *ct = bedLoader(fac, chromHash, cpp, track, dbRequested);
freeMem(track->tdb->type);
track->tdb->type = cloneString("coloredExon");
return ct;
}
static struct customFactory bedFactory =
/* Factory for bed tracks */
{
NULL,
"bed",
bedRecognizer,
bedLoader,
};
static struct customFactory bedGraphFactory =
{
NULL,
"bedGraph",
bedRecognizer,
bedGraphLoader,
};
static struct customFactory microarrayFactory =
/* Factory for bed tracks */
{
NULL,
"array",
microarrayRecognizer,
microarrayLoader,
};
static struct customFactory coloredExonFactory =
/* Factory for bed tracks */
{
NULL,
"coloredExon",
coloredExonRecognizer,
coloredExonLoader,
};
/**** ENCODE PEAK Factory - closely related to BED but not quite ***/
static boolean encodePeakRecognizer(struct customFactory *fac,
struct customPp *cpp, char *type,
struct customTrack *track)
/* Return TRUE if looks like we're handling an encodePeak track */
{
enum encodePeakType pt = 0;
if (type != NULL && !sameType(type, fac->name) &&
!sameString(type, "narrowPeak") && !sameString(type, "broadPeak") && !sameString(type, "gappedPeak"))
return FALSE;
char *line = customFactoryNextRealTilTrack(cpp);
if (line == NULL)
return FALSE;
char *dupe = cloneString(line);
char *row[ENCODE_PEAK_KNOWN_FIELDS+1];
int wordCount = chopLine(dupe, row);
pt = encodePeakInferType(wordCount, type);
if (pt)
track->fieldCount = wordCount;
freeMem(dupe);
customPpReuse(cpp, line);
return (pt != 0);
}
static struct pipeline *encodePeakLoaderPipe(struct customTrack *track)
/* Set up pipeline that will load the encodePeak into database. */
{
/* running the single command:
* hgLoadBed -customTrackLoader -sqlTable=loader/encodePeak.sql -renameSqlTable
* -trimSqlTable -notItemRgb -tmpDir=/data/tmp
* -maxChromNameLength=${nameLength} customTrash tableName stdin
*/
struct dyString *tmpDy = newDyString(0);
char *cmd1[] = {"loader/hgLoadBed", "-customTrackLoader",
"-sqlTable=loader/encodePeak.sql", "-renameSqlTable", "-trimSqlTable", "-notItemRgb", NULL, NULL, NULL, NULL, NULL, NULL};
char *tmpDir = cfgOptionDefault("customTracks.tmpdir", "/data/tmp");
struct stat statBuf;
int index = 6;
if (stat(tmpDir,&statBuf))
errAbort("can not find custom track tmp load directory: '%s'<BR>\n"
"create directory or specify in hg.conf customTracks.tmpdir", tmpDir);
dyStringPrintf(tmpDy, "-tmpDir=%s", tmpDir);
cmd1[index++] = dyStringCannibalize(&tmpDy); tmpDy = newDyString(0);
dyStringPrintf(tmpDy, "-maxChromNameLength=%d", track->maxChromName);
cmd1[index++] = dyStringCannibalize(&tmpDy);
cmd1[index++] = CUSTOM_TRASH;
cmd1[index++] = track->dbTableName;
cmd1[index++] = "stdin";
assert(index <= ArraySize(cmd1));
/* the "/dev/null" file isn't actually used for anything, but it is used
* in the pipeLineOpen to properly get a pipe started that isn't simply
* to STDOUT which is what a NULL would do here instead of this name.
* This function exits if it can't get the pipe created
* The dbStderrFile will get stderr messages from hgLoadBed into the
* our private error log so we can send it back to the user
*/
return pipelineOpen1(cmd1, pipelineWrite | pipelineNoAbort,
"/dev/null", track->dbStderrFile);
}
/* Need customTrackEncodePeak */
static struct encodePeak *customTrackEncodePeak(char *db, char **row, enum encodePeakType pt,
struct hash *chromHash, struct lineFile *lf)
/* Convert a row of strings to a bed. */
{
struct encodePeak *peak = encodePeakLineFileLoad(row, pt, lf);
hashStoreName(chromHash, peak->chrom);
customFactoryCheckChromNameDb(db, peak->chrom, lf);
return peak;
}
/* remember to set all the custom track settings necessary */
static struct customTrack *encodePeakFinish(struct customTrack *track, struct encodePeak *peakList, enum encodePeakType pt)
/* Finish up bed tracks (and others that create track->bedList). */
{
struct encodePeak *peak;
char buf[20];
track->fieldCount = ENCODEPEAK_NUM_COLS;
if ((pt == narrowPeak) || (pt == broadPeak))
track->fieldCount = ENCODE_PEAK_NARROW_PEAK_FIELDS;
track->tdb->type = cloneString("encodePeak");
track->dbTrackType = cloneString("encodePeak");
safef(buf, sizeof(buf), "%d", track->fieldCount);
ctAddToSettings(track, "fieldCount", cloneString(buf));
/* If necessary add track offsets. */
int offset = track->offset;
if (offset != 0)
{
/* Add track offsets if any */
for (peak = peakList; peak != NULL; peak = peak->next)
{
peak->chromStart += offset;
peak->chromEnd += offset;
}
track->offset = 0; /* so DB load later won't do this again */
hashMayRemove(track->tdb->settingsHash, "offset"); /* nor the file reader*/
}
/* If necessary load database */
customFactorySetupDbTrack(track);
struct pipeline *dataPipe = encodePeakLoaderPipe(track);
FILE *out = pipelineFile(dataPipe);
for (peak = peakList; peak != NULL; peak = peak->next)
encodePeakOutputWithType(peak, encodePeak, out);
fflush(out); /* help see error from loader failure */
if(ferror(out) || pipelineWait(dataPipe))
pipelineFailExit(track); /* prints error and exits */
unlink(track->dbStderrFile); /* no errors, not used */
pipelineFree(&dataPipe);
return track;
}
static struct customTrack *encodePeakLoader(struct customFactory *fac,
struct hash *chromHash,
struct customPp *cpp, struct customTrack *track, boolean dbRequested)
/* Load up encodePeak data until get next track line. */
{
char *line;
char *db = ctGenomeOrCurrent(track);
struct encodePeak *peakList = NULL;
enum encodePeakType pt = encodePeakInferType(track->fieldCount, track->tdb->type);
if (!dbRequested)
errAbort("encodePeak custom track type unavailable without custom trash database. Please set that up in your hg.conf");
while ((line = customFactoryNextRealTilTrack(cpp)) != NULL)
{
char *row[ENCODE_PEAK_KNOWN_FIELDS];
int wordCount = chopLine(line, row);
struct lineFile *lf = cpp->fileStack;
lineFileExpectAtLeast(lf, track->fieldCount, wordCount);
struct encodePeak *peak = customTrackEncodePeak(db, row, pt, chromHash, lf);
slAddHead(&peakList, peak);
}
slReverse(&peakList);
return encodePeakFinish(track, peakList, pt);
}
static struct customFactory encodePeakFactory =
/* Factory for bed tracks */
{
NULL,
"encodePeak",
encodePeakRecognizer,
encodePeakLoader,
};
/*** GFF/GTF Factory - converts to BED ***/
static boolean rowIsGff(char *db, char **row, int wordCount)
/* Return TRUE if format of this row is consistent with being a .gff */
{
boolean isGff = FALSE;
if (wordCount >= 8 && wordCount <= 9)
{
/* Check that strand is + - or . */
char *strand = row[6];
char c = strand[0];
if (c == '.' || c == '+' || c == '-')
{
if (strand[1] == 0)
{
if (hgIsOfficialChromName(db, row[0]))
{
if (isdigit(row[3][0]) && isdigit(row[4][0]))
isGff = TRUE;
}
}
}
}
return isGff;
}
static boolean gffRecognizer(struct customFactory *fac,
struct customPp *cpp, char *type,
struct customTrack *track)
/* Return TRUE if looks like we're handling a gff track */
{
if (type != NULL && !sameType(type, fac->name))
return FALSE;
char *line = customPpNextReal(cpp);
if (line == NULL)
return FALSE;
char *dupe = cloneString(line);
char *row[10];
int wordCount = chopTabs(dupe, row);
boolean isGff = rowIsGff(track->genomeDb, row, wordCount);
if (isGff)
track->gffHelper = gffFileNew("custom input");
freeMem(dupe);
customPpReuse(cpp, line);
return isGff;
}
static boolean gtfRecognizer(struct customFactory *fac,
struct customPp *cpp, char *type,
struct customTrack *track)
/* Return TRUE if looks like we're handling a gtf track.
First run the GFF recognizer, then check for GTF group syntax */
{
boolean isGtf = FALSE;
if (type != NULL && !sameType(type, fac->name))
return FALSE;
/* GTF is an extension of GFF, so run the GFF recognizer first.
* This will also create a GFF file handle for the track */
if (!gffRecognizer(fac, cpp, type, track))
return FALSE;
char *line = customPpNextReal(cpp);
if (gffHasGtfGroup(line))
isGtf = TRUE;
customPpReuse(cpp, line);
return isGtf;
}
static double gffGroupAverageScore(struct gffGroup *group, double defaultScore)
/* Return average score of GFF group, or average if none. */
{
double cumScore = 0;
int count = 0;
struct gffLine *line;
/* First see if any non-zero */
for (line = group->lineList; line != NULL; line = line->next)
{
if (line->score != 0.0)
++count;
}
if (count <= 0)
return defaultScore;
/* Calculate and return average score. */
for (line = group->lineList; line != NULL; line = line->next)
cumScore += line->score;
return cumScore / count;
}
static char *niceGeneName(char *name)
/* Return a nice version of name. */
{
static char buf[128];
char *e;
strncpy(buf, name, sizeof(buf));
if ((e = strchr(buf, ';')) != NULL)
*e = 0;
eraseWhiteSpace(buf);
stripChar(buf, '%');
stripChar(buf, '\'');
stripChar(buf, '"');
return buf;
}
static struct bed *gffHelperFinish(struct gffFile *gff, struct hash *chromHash)
/* Create bedList from gffHelper. */
{
struct genePred *gp;
struct bed *bedList = NULL, *bed;
struct gffGroup *group;
int i, blockCount, chromStart, exonStart;
gffGroupLines(gff);
for (group = gff->groupList; group != NULL; group = group->next)
{
/* First convert to gene-predictions since this is almost what we want. */
if (gff->isGtf)
gp = genePredFromGroupedGtf(gff, group, niceGeneName(group->name),
genePredNoOptFld, genePredGxfDefaults);
else
gp = genePredFromGroupedGff(gff, group, niceGeneName(group->name),
NULL, genePredNoOptFld, genePredGxfDefaults);
if (gp != NULL)
{
/* Make a bed out of the gp. */
AllocVar(bed);
bed->chrom = hashStoreName(chromHash, gp->chrom);
bed->chromStart = chromStart = gp->txStart;
bed->chromEnd = gp->txEnd;
bed->name = cloneString(gp->name);
bed->score = gffGroupAverageScore(group, 1000);
bed->strand[0] = gp->strand[0];
bed->thickStart = gp->cdsStart;
bed->thickEnd = gp->cdsEnd;
bed->blockCount = blockCount = gp->exonCount;
AllocArray(bed->blockSizes, blockCount);
AllocArray(bed->chromStarts, blockCount);
for (i=0; i<blockCount; ++i)
{
exonStart = gp->exonStarts[i];
bed->chromStarts[i] = exonStart - chromStart;
bed->blockSizes[i] = gp->exonEnds[i] - exonStart;
}
genePredFree(&gp);
slAddHead(&bedList, bed);
}
}
return bedList;
}
static struct customTrack *gffLoader(struct customFactory *fac,
struct hash *chromHash,
struct customPp *cpp, struct customTrack *track, boolean dbRequested)
/* Load up gff data until get next track line. */
{
char *line;
char *ctDb = ctGenomeOrCurrent(track);
while ((line = customFactoryNextRealTilTrack(cpp)) != NULL)
{
char *row[9];
int wordCount = chopTabs(line, row);
struct lineFile *lf = cpp->fileStack;
customFactoryCheckChromNameDb(ctDb, row[0], lf);
gffFileAddRow(track->gffHelper, 0, row, wordCount, lf->fileName,
lf->lineIx);
}
track->bedList = gffHelperFinish(track->gffHelper, chromHash);
gffFileFree(&track->gffHelper);
track->fieldCount = 12;
return bedFinish(track, dbRequested);
}
static struct customFactory gffFactory =
/* Factory for gff tracks */
{
NULL,
"gff",
gffRecognizer,
gffLoader,
};
static struct customFactory gtfFactory =
/* Factory for gtf tracks. Shares loader with gffFactory */
{
NULL,
"gtf",
gtfRecognizer,
gffLoader,
};
/*** PSL Factory - converts to BED ***/
static boolean checkStartEnd(char *sSize, char *sStart, char *sEnd)
/* Make sure start < end <= size */
{
int start, end, size;
start = atoi(sStart);
end = atoi(sEnd);
size = atoi(sSize);
return start < end && end <= size;
}
static boolean rowIsPsl(char **row, int wordCount)
/* Return TRUE if format of this row is consistent with being a .psl */
{
int i, len;
char *s, c;
int blockCount;
if (wordCount != PSL_NUM_COLS)
return FALSE;
for (i=0; i<=7; ++i)
if (!isdigit(row[i][0]))
return FALSE;
s = row[8];
len = strlen(s);
if (len < 1 || len > 2)
return FALSE;
for (i=0; i<len; ++i)
{
c = s[i];
if (c != '+' && c != '-')
return FALSE;
}
if (!checkStartEnd(row[10], row[11], row[12]))
return FALSE;
if (!checkStartEnd(row[14], row[15], row[16]))
return FALSE;
if (!isdigit(row[17][0]))
return FALSE;
blockCount = atoi(row[17]);
for (i=18; i<=20; ++i)
if (countChars(row[i], ',') != blockCount)
return FALSE;
return TRUE;
}
static boolean pslRecognizer(struct customFactory *fac,
struct customPp *cpp, char *type,
struct customTrack *track)
/* Return TRUE if looks like we're handling a bed track */
{
if (type != NULL && !sameType(type, fac->name))
return FALSE;
char *line = customPpNextReal(cpp);
if (line == NULL)
return FALSE;
boolean isPsl = FALSE;
if (startsWith("psLayout version", line))
{
isPsl = TRUE;
}
else
{
char *dupe = cloneString(line);
char *row[PSL_NUM_COLS+1];
int wordCount = chopLine(dupe, row);
isPsl = rowIsPsl(row, wordCount);
freeMem(dupe);
}
customPpReuse(cpp, line);
return isPsl;
}
static struct bed *customTrackPsl(char *db, boolean isProt, char **row,
int wordCount, struct hash *chromHash, struct lineFile *lf)
/* Convert a psl format row of strings to a bed. */
{
struct psl *psl = pslLoad(row);
struct bed *bed;
int i, blockCount, *chromStarts, chromStart, *blockSizes;
/* A tiny bit of error checking on the psl. */
if (psl->qStart >= psl->qEnd || psl->qEnd > psl->qSize
|| psl->tStart >= psl->tEnd || psl->tEnd > psl->tSize)
{
lineFileAbort(lf, "mangled psl format");
}
customFactoryCheckChromNameDb(db, psl->tName, lf);
if (!isProt)
{
isProt = pslIsProtein(psl);
}
/* Allocate bed and fill in from psl. */
AllocVar(bed);
bed->chrom = hashStoreName(chromHash, psl->tName);
bed->score = 1000 - 2*pslCalcMilliBad(psl, TRUE);
if (bed->score < 0) bed->score = 0;
strncpy(bed->strand, psl->strand, sizeof(bed->strand));
bed->strand[1] = 0;
bed->blockCount = blockCount = psl->blockCount;
bed->blockSizes = blockSizes = (int *)psl->blockSizes;
psl->blockSizes = NULL;
bed->chromStarts = chromStarts = (int *)psl->tStarts;
psl->tStarts = NULL;
bed->name = psl->qName;
psl->qName = NULL;
if (isProt)
{
for (i=0; i<blockCount; ++i)
{
blockSizes[i] *= 3;
}
/* Do a little trimming here. Arguably blat should do it
* instead. */
for (i=1; i<blockCount; ++i)
{
int lastEnd = blockSizes[i-1] + chromStarts[i-1];
if (chromStarts[i] < lastEnd)
chromStarts[i] = lastEnd;
}
}
/* Switch minus target strand to plus strand. */
if (psl->strand[1] == '-')
{
int chromSize = psl->tSize;
reverseInts(bed->blockSizes, blockCount);
reverseInts(chromStarts, blockCount);
for (i=0; i<blockCount; ++i)
{
chromStarts[i] = chromSize - chromStarts[i] - blockSizes[i];
}
}
bed->thickStart = bed->chromStart = chromStart = chromStarts[0];
bed->thickEnd = bed->chromEnd = chromStarts[blockCount-1] + blockSizes[blockCount-1];
/* Convert coordinates to relative. */
for (i=0; i<blockCount; ++i)
chromStarts[i] -= chromStart;
pslFree(&psl);
return bed;
}
static struct customTrack *pslLoader(struct customFactory *fac,
struct hash *chromHash,
struct customPp *cpp, struct customTrack *track, boolean dbRequested)
/* Load up psl data until get next track line. */
{
char *line;
boolean pslIsProt = FALSE;
char *ctDb = ctGenomeOrCurrent(track);
while ((line = customFactoryNextRealTilTrack(cpp)) != NULL)
{
/* Skip over pslLayout version lines noting if they are
* protein though */
if (startsWith("psLayout version", line))
{
pslIsProt = (stringIn("protein", line) != NULL);
int i;
for (i=0; i<3; ++i)
customFactoryNextRealTilTrack(cpp);
continue;
}
char *row[PSL_NUM_COLS];
int wordCount = chopLine(line, row);
struct lineFile *lf = cpp->fileStack;
lineFileExpectAtLeast(lf, PSL_NUM_COLS, wordCount);
struct bed *bed = customTrackPsl(ctDb, pslIsProt, row,
wordCount, chromHash, lf);
slAddHead(&track->bedList, bed);
}
slReverse(&track->bedList);
track->fieldCount = 12;
return bedFinish(track, dbRequested);
}
static struct customFactory pslFactory =
/* Factory for psl tracks */
{
NULL,
"psl",
pslRecognizer,
pslLoader,
};
static boolean mafRecognizer(struct customFactory *fac,
struct customPp *cpp, char *type,
struct customTrack *track)
/* Return TRUE if looks like we're handling a maf track */
{
if (type != NULL && !sameType(type, fac->name))
return FALSE;
char *line = customPpNext(cpp);
if (line == NULL)
return FALSE;
boolean isMaf = FALSE;
if (startsWith("##maf version", line))
{
isMaf = TRUE;
}
customPpReuse(cpp, line);
return isMaf;
}
static void mafLoaderBuildTab(struct customTrack *track, char *mafFile)
/* build maf tab file and load in database */
{
customFactorySetupDbTrack(track);
struct dyString *tmpDy = newDyString(0);
char *cmd1[] = {"loader/hgLoadMaf", "-verbose=0", "-custom", NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL};
char **cmds[] = {cmd1, NULL};
char *tmpDir = cfgOptionDefault("customTracks.tmpdir", "/data/tmp");
struct stat statBuf;
struct tempName tn;
trashDirFile(&tn, "ct", "ct", ".pos");
if (stat(tmpDir,&statBuf))
errAbort("can not find custom track tmp load directory: '%s'<BR>\n"
"create directory or specify in hg.conf customTracks.tmpdir", tmpDir);
dyStringPrintf(tmpDy, "-tmpDir=%s", tmpDir);
cmd1[3] = dyStringCannibalize(&tmpDy); tmpDy = newDyString(0);
dyStringPrintf(tmpDy, "-loadFile=%s", mafFile);
cmd1[4] = dyStringCannibalize(&tmpDy); tmpDy = newDyString(0);
dyStringPrintf(tmpDy, "-refDb=%s", track->genomeDb);
cmd1[5] = dyStringCannibalize(&tmpDy); tmpDy = newDyString(0);
dyStringPrintf(tmpDy, "-maxNameLen=%d", track->maxChromName);
cmd1[6] = dyStringCannibalize(&tmpDy); tmpDy = newDyString(0);
dyStringPrintf(tmpDy, "-defPos=%s", tn.forCgi);
cmd1[7] = dyStringCannibalize(&tmpDy);
cmd1[8] = CUSTOM_TRASH;
cmd1[9] = track->dbTableName;
struct pipeline *dataPipe = pipelineOpen(cmds,
pipelineWrite | pipelineNoAbort, "/dev/null", track->dbStderrFile);
if(pipelineWait(dataPipe))
pipelineFailExit(track); /* prints error and exits */
pipelineFree(&dataPipe);
unlink(track->dbStderrFile); /* no errors, not used */
track->wigFile = NULL;
struct lineFile *lf = lineFileOpen(tn.forCgi, TRUE);
char *line;
int size;
lineFileNeedNext(lf, &line, &size);
lineFileClose(&lf);
unlink(tn.forCgi);
ctAddToSettings(track, "firstItemPos", cloneString(line));
}
static struct customTrack *mafLoader(struct customFactory *fac,
struct hash *chromHash,
struct customPp *cpp, struct customTrack *track, boolean dbRequested)
{
FILE *f;
char *line;
struct tempName tn;
struct hash *settings = track->tdb->settingsHash;
if (!dbRequested)
errAbort("Maf files have to be in database");
track->dbTrackType = cloneString(fac->name);
track->wiggle = TRUE;
/* If mafFile setting already exists, then we are reloading, not loading.
* Just make sure files still exist. */
if (hashFindVal(settings, "mafFile"))
{
track->wibFile = hashFindVal(settings, "mafFile");
if (!fileExists(track->wibFile))
return FALSE;
}
/* MafFile setting doesn't exist, so we are loading from stream. */
else
{
char *maxByteStr = cfgOption("customTracks.maxBytes");
/* Make up wib file name and add to settings. */
trashDirFile(&tn, "ct", "ct", ".maf");
track->wibFile = cloneString(tn.forCgi);
ctAddToSettings(track, "mafFile", track->wibFile);
char *mafFile = cloneString(track->wibFile);
/* Actually create maf file. */
f = mustOpen(mafFile, "w");
if (maxByteStr != NULL)
{
long maxBytes = sqlUnsignedLong(maxByteStr);
long numBytesLeft = maxBytes;
while ((line = customFactoryNextTilTrack(cpp)) != NULL)
{
numBytesLeft -= strlen(line);
if (numBytesLeft < 0)
{
unlink(mafFile);
errAbort("exceeded upload limit of %ld bytes\n", maxBytes);
}
fprintf(f, "%s\n", line);
}
}
else
while ((line = customFactoryNextTilTrack(cpp)) != NULL)
fprintf(f, "%s\n", line);
carefulClose(&f);
mafLoaderBuildTab(track, mafFile);
}
char tdbType[256];
safef(tdbType, sizeof(tdbType), "maf");
track->tdb->type = cloneString(tdbType);
return track;
}
static struct customFactory mafFactory =
/* Factory for maf tracks */
{
NULL,
"maf",
mafRecognizer,
mafLoader,
};
/*** Wig Factory - for wiggle tracks ***/
static boolean wigRecognizer(struct customFactory *fac,
struct customPp *cpp, char *type,
struct customTrack *track)
/* Return TRUE if looks like we're handling a wig track */
{
return (sameOk(type, fac->name) || sameType(type, "wig"));
}
static struct pipeline *wigLoaderPipe(struct customTrack *track)
/* Set up pipeline that will load wig into database. */
{
/* Run the two commands in a pipeline:
* loader/wigEncode -verbose=0 -wibSizeLimit=300000000 stdin stdout \
* ${wibFile} | \
* loader/hgLoadWiggle -verbose=0 -noHistory -tmpDir=/data/tmp \
* -maxChromNameLength=${nameLength} -chromInfoDb=${database} \
* -pathPrefix=[.|/] ${db} ${table} stdin
*/
struct dyString *tmpDy = newDyString(0);
char *cmd1[] = {"loader/wigEncode", "-verbose=0", "-wibSizeLimit=300000000",
"stdin", "stdout", NULL, NULL};
char *cmd2[] = {"loader/hgLoadWiggle", "-verbose=0", "-noHistory", NULL, NULL,
NULL, NULL, NULL, NULL, "stdin", NULL};
char **cmds[] = {cmd1, cmd2, NULL};
char *tmpDir = cfgOptionDefault("customTracks.tmpdir", "/data/tmp");
struct stat statBuf;
cmd1[5] = track->wibFile;
if (stat(tmpDir,&statBuf))
errAbort("can not find custom track tmp load directory: '%s'<BR>\n"
"create directory or specify in hg.conf customTracks.tmpdir", tmpDir);
dyStringPrintf(tmpDy, "-tmpDir=%s", tmpDir);
cmd2[3] = dyStringCannibalize(&tmpDy); tmpDy = newDyString(0);
dyStringPrintf(tmpDy, "-maxChromNameLength=%d", track->maxChromName);
cmd2[4] = dyStringCannibalize(&tmpDy); tmpDy = newDyString(0);
dyStringPrintf(tmpDy, "-chromInfoDb=%s", track->genomeDb);
cmd2[5] = dyStringCannibalize(&tmpDy);
/* a system could be using /trash/ absolute reference, and nothing to do with
* local references, so don't confuse it with ./ a double // will work
*/
if (startsWith("/", trashDir()))
cmd2[6] = "-pathPrefix=/";
else
cmd2[6] = "-pathPrefix=.";
cmd2[7] = CUSTOM_TRASH;
cmd2[8] = track->dbTableName;
/* the "/dev/null" file isn't actually used for anything, but it is used
* in the pipeLineOpen to properly get a pipe started that isn't simply
* to STDOUT which is what a NULL would do here instead of this name.
* This function exits if it can't get the pipe created
* The dbStderrFile will get stderr messages from this pipeline into the
* our private error file so we can return the errors to the user.
*/
return pipelineOpen(cmds, pipelineWrite | pipelineNoAbort,
"/dev/null", track->dbStderrFile);
}
static void wigDbGetLimits(struct sqlConnection *conn, char *tableName,
double *retUpperLimit, double *retLowerLimit, int *retSpan)
/* Figure out upper/lower limits of wiggle table. */
{
char query[512];
safef(query,sizeof(query),
"select min(lowerLimit),max(lowerLimit+dataRange) from %s",
tableName);
struct sqlResult *sr = sqlGetResult(conn, query);
char **row;
if ((row = sqlNextRow(sr)) != NULL);
{
if (row[0]) *retLowerLimit = sqlDouble(row[0]);
if (row[1]) *retUpperLimit = sqlDouble(row[1]);
}
sqlFreeResult(&sr);
safef(query,sizeof(query),
"select span from %s group by span", tableName);
int span = sqlQuickNum(conn, query);
if (span == 0)
span = 1;
*retSpan = span;
}
/* HACK ALERT - The table browser needs to be able to encode its wiggle
* data. This function is temporarily global until a proper method
* is used to work this business into the table browser custom
* tracks. Currently this is also called from customSaveTracks()
* in customTrack.c in violation of this object's hidden methods.
*/
void wigLoaderEncoding(struct customTrack *track, char *wigAscii,
boolean dbRequested)
/* encode wigAscii file into .wig and .wib files */
{
/* Need to figure upper and lower limits. */
double lowerLimit = 0.0;
double upperLimit = 100.0;
int span = 1;
/* Load database if requested */
if (dbRequested)
{
/* TODO: see if can avoid extra file copy in this case. */
customFactorySetupDbTrack(track);
/* Load ascii file into database via pipeline. */
struct pipeline *dataPipe = wigLoaderPipe(track);
FILE *in = mustOpen(wigAscii, "r");
FILE *out = pipelineFile(dataPipe);
char c;
int fputcErr = 0;
while ((c = fgetc(in)) != EOF && fputcErr != EOF)
fputcErr = fputc(c, out);
carefulClose(&in);
fflush(out); /* help see error from loader failure */
#if 0 // enable this for help debugging
fprintf(stderr, "%s\n", pipelineDesc(dataPipe));
#endif
if(ferror(out) || pipelineWait(dataPipe))
pipelineFailExit(track); /* prints error and exits */
unlink(track->dbStderrFile); /* no errors, not used */
pipelineFree(&dataPipe);
track->wigFile = NULL;
/* Figure out lower and upper limits with db query */
struct sqlConnection *ctConn = hAllocConn(CUSTOM_TRASH);
char buf[64];
wigDbGetLimits(ctConn, track->dbTableName,
&upperLimit, &lowerLimit, &span);
hFreeConn(&ctConn);
safef(buf, sizeof(buf), "%d", span);
ctAddToSettings(track, "spanList", cloneString(buf));
}
else
{
/* Make up wig file name (by replacing suffix of wib file name)
* and add to settings. */
track->wigFile = cloneString(track->wibFile);
chopSuffix(track->wigFile);
strcat(track->wigFile, ".wig");
ctAddToSettings(track, "wigFile", track->wigFile);
struct wigEncodeOptions options;
ZeroVar(&options); /* all is zero */
options.lift = 0;
options.noOverlap = FALSE;
options.flagOverlapSpanData = TRUE;
options.wibSizeLimit = 300000000; /* 300Mb limit*/
wigAsciiToBinary(wigAscii, track->wigFile,
track->wibFile, &upperLimit, &lowerLimit, &options);
if (options.wibSizeLimit >= 300000000)
warn("warning: reached data limit for wiggle track '%s' "
"%lld >= 300,000,000<BR>\n",
track->tdb->shortLabel, options.wibSizeLimit);
}
unlink(wigAscii);/* done with this, remove it */
freeMem(track->wigAscii);
char tdbType[256];
safef(tdbType, sizeof(tdbType), "wig %g %g", lowerLimit, upperLimit);
track->tdb->type = cloneString(tdbType);
}
static struct customTrack *wigLoader(struct customFactory *fac,
struct hash *chromHash,
struct customPp *cpp, struct customTrack *track, boolean dbRequested)
/* Load up wiggle data until get next track line. */
{
FILE *f;
char *line;
struct tempName tn;
struct hash *settings = track->tdb->settingsHash;
track->dbTrackType = cloneString(fac->name);
track->wiggle = TRUE;
/* If wibFile setting already exists, then we are reloading, not loading.
* Just make sure files still exist. */
if (hashFindVal(settings, "wibFile"))
{
track->wibFile = hashFindVal(settings, "wibFile");
if (!fileExists(track->wibFile))
return FALSE;
/* In database case wigFile is in database table, and that
* table's existence is checked by framework, so we need not
* bother. */
track->wigFile = hashFindVal(settings, "wigFile");
if (track->wigFile != NULL)
{
if (!fileExists(track->wigFile))
{
return FALSE;
}
}
}
/* WibFile setting doesn't exist, so we are loading from ascii stream. */
else
{
/* Make up wib file name and add to settings. */
trashDirFile(&tn, "ct", "ct", ".wib");
track->wibFile = cloneString(tn.forCgi);
ctAddToSettings(track, "wibFile", track->wibFile);
/* Don't add wigAscii to settings - not needed. */
char *wigAscii = cloneString(track->wibFile);
chopSuffix(wigAscii);
strcat(wigAscii, ".wia");
/* Actually create wigAscii file. */
f = mustOpen(wigAscii, "w");
while ((line = customFactoryNextRealTilTrack(cpp)) != NULL)
fprintf(f, "%s\n", line);
carefulClose(&f);
wigLoaderEncoding(track, wigAscii, dbRequested);
}
return track;
}
static struct customFactory wigFactory =
/* Factory for wiggle tracks */
{
NULL,
"wiggle_0",
wigRecognizer,
wigLoader,
};
/*** Big Wig Factory - for big client-side wiggle tracks ***/
static boolean bigWigRecognizer(struct customFactory *fac,
struct customPp *cpp, char *type,
struct customTrack *track)
/* Return TRUE if looks like we're handling a wig track */
{
return (sameType(type, "bigWig"));
}
static struct customTrack *bigWigLoader(struct customFactory *fac,
struct hash *chromHash,
struct customPp *cpp, struct customTrack *track, boolean dbRequested)
/* Load up wiggle data until get next track line. */
{
/* Not much to this. A bigWig has nothing here but a track line. */
struct hash *settings = track->tdb->settingsHash;
char *bigDataUrl = hashFindVal(settings, "bigDataUrl");
if (bigDataUrl == NULL)
errAbort("Missing bigDataUrl setting from track of type=bigWig");
struct bbiFile *bbi = bigWigFileOpen(bigDataUrl); // Just for error checking
bbiFileClose(&bbi);
return track;
}
static struct customFactory bigWigFactory =
/* Factory for wiggle tracks */
{
NULL,
"bigWig",
bigWigRecognizer,
bigWigLoader,
};
/*** Big Bed Factory - for big client-side BED tracks ***/
static boolean bigBedRecognizer(struct customFactory *fac,
struct customPp *cpp, char *type,
struct customTrack *track)
/* Return TRUE if looks like we're handling a wig track */
{
return (sameType(type, "bigBed"));
}
static struct customTrack *bigBedLoader(struct customFactory *fac,
struct hash *chromHash,
struct customPp *cpp, struct customTrack *track, boolean dbRequested)
/* Load up big bed data until get next track line. */
{
/* Not much to this. A bigBed has nothing here but a track line. */
struct hash *settings = track->tdb->settingsHash;
char *bigDataUrl = hashFindVal(settings, "bigDataUrl");
if (bigDataUrl == NULL)
errAbort("Missing bigDataUrl setting from track of type=bigBed");
struct bbiFile *bbi = bigBedFileOpen(bigDataUrl); // Just for error checking
bbiFileClose(&bbi);
return track;
}
static struct customFactory bigBedFactory =
/* Factory for bigBed tracks */
{
NULL,
"bigBed",
bigBedRecognizer,
bigBedLoader,
};
#ifdef USE_BAM
/*** BAM Factory - for client-side BAM alignment files ***/
static boolean bamRecognizer(struct customFactory *fac, struct customPp *cpp, char *type,
struct customTrack *track)
/* Return TRUE if looks like we're handling a bam track */
{
return (sameType(type, "bam"));
}
static struct customTrack *bamLoader(struct customFactory *fac, struct hash *chromHash,
struct customPp *cpp, struct customTrack *track,
boolean dbRequested)
/* Process the bam track line. */
{
struct hash *settings = track->tdb->settingsHash;
char *bigDataUrl = hashFindVal(settings, "bigDataUrl");
if (bigDataUrl == NULL)
errAbort("Missing bigDataUrl setting from track of type=bam (%s)", track->tdb->shortLabel);
-if (!bamFileExists(bigDataUrl))
+if (doExtraChecking)
+ {
+ if (!bamFileExists(bigDataUrl))
errAbort("Can't access %s's bigDataUrl %s and/or the associated index file %s.bai",
track->tdb->shortLabel, bigDataUrl, bigDataUrl);
+ }
return track;
}
static struct customFactory bamFactory =
/* Factory for bam tracks */
{
NULL,
"bam",
bamRecognizer,
bamLoader,
};
#endif//def USE_BAM
/*** Framework for custom factories. ***/
static struct customFactory *factoryList;
static void customFactoryInit()
/* Initialize custom track factory system. */
{
if (factoryList == NULL)
{
/* mafFactory has to be first because it
* doesn't strip comments
*/
slAddHead(&factoryList, &mafFactory);
slAddTail(&factoryList, &wigFactory);
slAddTail(&factoryList, &bigWigFactory);
slAddTail(&factoryList, &chromGraphFactory);
slAddTail(&factoryList, &pslFactory);
slAddTail(&factoryList, >fFactory);
slAddTail(&factoryList, &gffFactory);
slAddTail(&factoryList, &bedFactory);
slAddTail(&factoryList, &bigBedFactory);
slAddTail(&factoryList, &bedGraphFactory);
slAddTail(&factoryList, µarrayFactory);
slAddTail(&factoryList, &coloredExonFactory);
slAddTail(&factoryList, &encodePeakFactory);
#ifdef USE_BAM
slAddTail(&factoryList, &bamFactory);
#endif//def USE_BAM
}
}
struct customFactory *customFactoryFind(char *genomeDb, struct customPp *cpp,
char *type, struct customTrack *track)
/* Figure out factory that can handle this track. The track is
* loaded from the track line if any, and type is the type element
* if any from that track. */
{
struct customFactory *fac;
customFactoryInit();
for (fac = factoryList; fac != NULL; fac = fac->next)
if (fac->recognizer(fac, cpp, type, track))
break;
return fac;
}
void customFactoryAdd(struct customFactory *fac)
/* Add factory to global custom track factory list. */
{
slAddTail(&factoryList, fac);
}
static void parseRgb(char *s, int lineIx,
unsigned char *retR, unsigned char *retG, unsigned char *retB)
/* Turn comma separated list to RGB vals. */
{
int wordCount;
char *row[4];
wordCount = chopString(s, ",", row, ArraySize(row));
if ((wordCount != 3) || (!isdigit(row[0][0]) || !isdigit(row[1][0]) || !isdigit(row[2][0])))
errAbort("line %d of custom input, Expecting 3 comma separated numbers in color definition.", lineIx);
*retR = atoi(row[0]);
*retG = atoi(row[1]);
*retB = atoi(row[2]);
}
static void customTrackUpdateFromSettings(struct customTrack *track,
char *genomeDb,
char *line, int lineIx)
/* replace settings in track with those from new track line */
{
char *pLine = line;
nextWord(&pLine);
line = skipLeadingSpaces(pLine);
struct hash *newSettings = hashVarLine(line, lineIx);
struct hashCookie hc = hashFirst(newSettings);
struct hashEl *hel = NULL;
while ((hel = hashNext(&hc)) != NULL)
ctAddToSettings(track, hel->name, hel->val);
struct trackDb *tdb = track->tdb;
struct hash *hash = tdb->settingsHash;
if (hash == NULL) // make sure we have a settings hash
hash = tdb->settingsHash = newHash(7);
char *val;
if ((val = hashFindVal(hash, "name")) != NULL)
{
if (*val)
tdb->shortLabel = cloneString(val);
else
tdb->shortLabel = cloneString("My Track");
stripChar(tdb->shortLabel,'"'); /* no quotes please */
stripChar(tdb->shortLabel,'\''); /* no quotes please */
tdb->tableName = customTrackTableFromLabel(tdb->shortLabel);
/* also use name for description, if not specified */
tdb->longLabel = cloneString(tdb->shortLabel);
}
if ((val = hashFindVal(hash, "description")) != NULL)
{
if (*val)
tdb->longLabel = cloneString(val);
else
tdb->longLabel = cloneString("My Custom Track");
stripChar(tdb->longLabel,'"'); /* no quotes please */
stripChar(tdb->longLabel,'\''); /* no quotes please */
}
tdb->type = hashFindVal(hash, "tdbType");
/* might be an old-style wigType track */
if (NULL == tdb->type)
tdb->type = hashFindVal(hash, "wigType");
/* might be a user-submitted CT that we're reading for the first time */
if (NULL == tdb->type)
tdb->type = hashFindVal(hash, "type");
track->genomeDb = cloneString(genomeDb);
track->dbTrackType = hashFindVal(hash, "dbTrackType");
track->dbTableName = hashFindVal(hash, "dbTableName");
if (track->dbTableName)
{
track->dbDataLoad = TRUE;
track->dbTrack = TRUE;
}
if ((val = hashFindVal(hash, "fieldCount")) != NULL)
track->fieldCount = sqlUnsigned(val);
if ((val = hashFindVal(hash, "htmlFile")) != NULL)
{
if (fileExists(val))
{
readInGulp(val, &track->tdb->html, NULL);
track->htmlFile = cloneString(val);
}
}
if ((val = hashFindVal(hash, "chromosomes")) != NULL)
sqlStringDynamicArray(val, &track->tdb->restrictList, &track->tdb->restrictCount);
if ((val = hashFindVal(hash, "htmlUrl")) != NULL)
{
/* adding error trapping because various net.c functions can errAbort */
struct errCatch *errCatch = errCatchNew();
if (errCatchStart(errCatch))
{
struct dyString *ds = NULL;
int sd = netUrlOpen(val);
if (sd >= 0)
{
char *newUrl = NULL;
int newSd = 0;
if (netSkipHttpHeaderLinesHandlingRedirect(sd, val, &newSd, &newUrl))
/* redirect can modify the url */
{
if (newUrl)
{
freeMem(newUrl);
sd = newSd;
}
ds = netSlurpFile(sd);
close(sd);
track->tdb->html = dyStringCannibalize(&ds);
}
}
}
errCatchEnd(errCatch);
if (errCatch->gotError)
warn("%s", errCatch->message->string);
errCatchFree(&errCatch);
}
tdb->url = hashFindVal(hash, "url");
if ((val = hashFindVal(hash, "visibility")) != NULL)
{
if (isdigit(val[0]))
{
tdb->visibility = atoi(val);
if (tdb->visibility > tvSquish)
errAbort("line %d of custom input: Expecting visibility 0 to 4 got %s", lineIx, val);
}
else
{
tdb->visibility = hTvFromString(val);
}
}
if ((val = hashFindVal(hash, "group")) != NULL)
tdb->grp = val;
if ((val = hashFindVal(hash, "useScore")) != NULL)
tdb->useScore = !sameString(val, "0");
if ((val = hashFindVal(hash, "priority")) != NULL)
tdb->priority = atof(val);
if ((val = hashFindVal(hash, "color")) != NULL)
parseRgb(cloneString(val), lineIx,
&tdb->colorR, &tdb->colorG, &tdb->colorB);
if ((val = hashFindVal(hash, "altColor")) != NULL)
parseRgb(cloneString(val), lineIx,
&tdb->altColorR, &tdb->altColorG, &tdb->altColorB);
else
{
/* If they don't explicitly set the alt color make it a lighter version
* of color. */
tdb->altColorR = (tdb->colorR + 255)/2;
tdb->altColorG = (tdb->colorG + 255)/2;
tdb->altColorB = (tdb->colorB + 255)/2;
}
if ((val = hashFindVal(hash, "offset")) != NULL)
track->offset = atoi(val);
if ((val = hashFindVal(hash, "maxChromName")) != NULL)
track->maxChromName = sqlSigned(val);
else
{
char *ctDb = ctGenomeOrCurrent(track);
track->maxChromName = hGetMinIndexLength(ctDb);
}
if ((line != NULL) && !strstr(line, "tdbType"))
{
/* for "external" (user-typed) track lines, save for later display
* in the manager CGI */
struct dyString *ds = dyStringNew(0);
/* exclude special setting used by table browser to indicate file needs
* to be parsed by the factory */
char *unparsed;
if ((unparsed = stringIn(CT_UNPARSED, line)) != NULL)
{
char *nextTok = skipToSpaces(unparsed);
if (!nextTok)
nextTok = "";
*unparsed = 0;
dyStringPrintf(ds, "track %s %s", line, nextTok);
}
else
dyStringPrintf(ds, "track %s", line);
ctAddToSettings(track, "origTrackLine", dyStringCannibalize(&ds));
}
char *tmp = tdb->tableName;
tdb->tableName = jsStripJavascript(tmp);
freeMem(tmp);
tmp = tdb->shortLabel;
tdb->shortLabel = jsStripJavascript(tmp);
freeMem(tmp);
tmp = tdb->longLabel;
tdb->longLabel = jsStripJavascript(tmp);
freeMem(tmp);
}
char *browserLinesToSetting(struct slName *browserLines)
/* Create a setting with browser lines separated by semi-colons */
{
if (!browserLines)
return NULL;
struct dyString *ds = dyStringNew(0);
struct slName *bl = NULL;
for (bl = browserLines; bl != NULL; bl = bl->next)
{
dyStringAppend(ds, bl->name);
dyStringAppend(ds, ";");
}
return dyStringCannibalize(&ds);
}
struct slName *ctBrowserLines(struct customTrack *ct)
/* retrieve browser lines from setting */
{
char *setting;
if ((setting = trackDbSetting(ct->tdb, "browserLines")) == NULL)
return NULL;
return slNameListFromString(setting, ';');
}
static char *browserLinePosition(struct slName *browserLines)
/* return position from browser lines, or NULL if not found */
{
struct slName *bl;
int wordCt;
char *words[64];
for (bl = browserLines; bl != NULL; bl = bl->next)
{
wordCt = chopLine(cloneString(bl->name), words);
if (wordCt == 3 && sameString("position", words[1]))
return words[2];
}
return NULL;
}
void customTrackUpdateFromConfig(struct customTrack *ct, char *genomeDb,
char *config, struct slName **retBrowserLines)
/* update custom track from config containing track line and browser lines
* Return browser lines */
{
if (!config)
return;
struct lineFile *lf = lineFileOnString("config", TRUE, config);
char *line;
struct slName *browserLines = NULL;
while (lineFileNextReal(lf, &line))
if (startsWithWord("track", line))
customTrackUpdateFromSettings(ct, genomeDb, line, 1);
else if (startsWithWord("browser", line))
slNameAddTail(&browserLines, line);
else
errAbort("expecting track or browser line, got: %s", line);
char *setting = browserLinesToSetting(browserLines);
if (setting)
{
ctAddToSettings(ct, "browserLines", setting);
char *initialPos = browserLinePosition(browserLines);
if (initialPos)
ctAddToSettings(ct, "initialPos", initialPos);
}
lineFileClose(&lf);
if (retBrowserLines)
*retBrowserLines = browserLines;
}
char *customTrackUserConfig(struct customTrack *ct)
/* return user-defined track line and browser lines */
{
struct dyString *ds = dyStringNew(0);
char *userTrackLine = ctOrigTrackLine(ct);
if (userTrackLine)
{
dyStringAppend(ds, userTrackLine);
dyStringAppend(ds, "\n");
}
struct slName *bl = NULL;
for (bl = ctBrowserLines(ct); bl != NULL; bl = bl->next)
{
dyStringAppend(ds, bl->name);
dyStringAppend(ds, "\n");
}
return (dyStringCannibalize(&ds));
}
static struct customTrack *trackLineToTrack(char *genomeDb, char *line, int lineIx)
/* Convert a track specification line to a custom track structure. */
{
/* Make up basic track with associated tdb. Fill in settings
* from var=val pairs in line. */
struct customTrack *track;
AllocVar(track);
struct trackDb *tdb = customTrackTdbDefault();
track->tdb = tdb;
customTrackUpdateFromSettings(track, genomeDb, line, lineIx);
return track;
}
static struct lineFile *customLineFile(char *text, boolean isFile)
/* Figure out input source, handling URL's and compression */
{
if (!text)
return NULL;
struct lineFile *lf = NULL;
if (isFile)
{
if (stringIn("://", text))
lf = netLineFileOpen(text);
else
lf = lineFileOpen(text, TRUE);
}
else
{
if (startsWith("compressed://",text))
{
char *words[3];
char *mem;
unsigned long size;
chopByWhite(text,words,3);
mem = (char *)sqlUnsignedLong(words[1]);
size = sqlUnsignedLong(words[2]);
lf = lineFileDecompressMem(TRUE, mem, size);
}
else
{
lf = lineFileOnString("custom track", TRUE, text);
}
}
return lf;
}
char *customDocParse(char *text)
/* Return description text, expanding URLs as for custom track data */
{
char *line;
struct lineFile *lf = customLineFile(text, FALSE);
if (!lf)
return NULL;
/* wrap a doc customPp object around it. */
struct customPp *cpp = customDocPpNew(lf);
/* extract doc */
struct dyString *ds = dyStringNew(0);
while ((line = customPpNextReal(cpp)) != NULL)
{
dyStringAppend(ds, line);
dyStringAppend(ds, "\n");
}
return dyStringCannibalize(&ds);
}
char *ctGenome(struct customTrack *ct)
/* return database setting, using old-style var name if present*/
{
char *setting = trackDbSetting(ct->tdb, "db");
if (setting)
return setting;
return trackDbSetting(ct->tdb, "genome");
}
static struct customTrack *customFactoryParseOptionalDb(char *genomeDb, char *text,
boolean isFile, struct slName **retBrowserLines,
boolean mustBeCurrentDb)
/* Parse text into a custom set of tracks. Text parameter is a
* file name if 'isFile' is set. If mustBeCurrentDb, die if custom track
* is for some database other than genomeDb. */
{
struct customTrack *trackList = NULL, *track = NULL;
char *line = NULL;
struct hash *chromHash = newHash(8);
float prio = 0.0;
struct sqlConnection *ctConn = NULL;
boolean dbTrack = ctDbUseAll();
if (dbTrack)
ctConn = hAllocConn(CUSTOM_TRASH);
struct lineFile *lf = customLineFile(text, isFile);
/* wrap a customPp object around it. */
struct customPp *cpp = customPpNew(lf);
lf = NULL;
/* Loop through this once for each track. */
while ((line = customPpNextReal(cpp)) != NULL)
{
/* Parse out track line and save it in track var.
* First time through make up track var from thin air
* if no track line. Find out explicit type setting if any.
* Also make sure settingsHash is set up. */
lf = cpp->fileStack;
if (startsWithWord("track", line))
{
track = trackLineToTrack(genomeDb, line, cpp->fileStack->lineIx);
}
else if (trackList == NULL)
/* In this case we handle simple files with a single track
* and no track line. */
{
char defaultLine[256];
safef(defaultLine, sizeof defaultLine,
"track name='%s' description='%s'",
CT_DEFAULT_TRACK_NAME, CT_DEFAULT_TRACK_DESCR);
track = trackLineToTrack(genomeDb, defaultLine, 1);
customPpReuse(cpp, line);
}
else
{
errAbort("Expecting 'track' line, got %s\nline %d of %s",
line, lf->lineIx, lf->fileName);
}
if (!track)
continue;
/* verify database for custom track */
char *ctDb = ctGenome(track);
if (mustBeCurrentDb)
{
if (ctDb == NULL)
ctDb = genomeDb;
else if (differentString(ctDb, genomeDb))
errAbort("can't load %s data into %s custom tracks",
ctDb, genomeDb);
}
struct customTrack *oneList = NULL, *oneTrack;
if (track->dbDataLoad)
/* Database tracks already mostly loaded, just check that table
* still exists (they are removed when not accessed for a while). */
{
if (!ctConn || !ctDbTableExists(ctConn, track->dbTableName))
continue;
if ( startsWith("maf", track->tdb->type))
{
struct hash *settings = track->tdb->settingsHash;
char *fileName;
if ((fileName = hashFindVal(settings, "mafFile")) == NULL)
continue;
if (!fileExists(fileName))
continue;
}
track->wiggle = startsWith("wig ", track->tdb->type);
if (track->wiggle)
{
if (!verifyWibExists(ctConn, track->dbTableName))
continue;
}
oneList = track;
}
else
/* Main case - we have to find a track factory that recognizes
* this track, and call it. Also we may need to do some work
* to load track into database. */
{
/* Load track from appropriate factory */
char *type = track->tdb->type;
struct customFactory *fac = customFactoryFind(genomeDb, cpp, type, track);
if (fac == NULL)
{
struct lineFile *lf = cpp->fileStack;
/* Check for case of empty track with no type. This
* is silently ignored for backwards compatibility */
if (type == NULL)
{
char *line = customFactoryNextRealTilTrack(cpp);
customPpReuse(cpp, line);
if (line == NULL)
continue;
else
{
errAbort("Unrecognized format line %d of %s:\n\t%s (note: chrom names are case sensitive)",
lf->lineIx, lf->fileName, emptyForNull(line));
}
}
else
{
errAbort("Unrecognized format type=%s line %d of %s",
type, lf->lineIx, lf->fileName);
}
}
char *dataUrl = NULL;
if (lf->fileName && (
startsWith("http://" , lf->fileName) ||
startsWith("https://", lf->fileName) ||
startsWith("ftp://" , lf->fileName)
))
dataUrl = cloneString(lf->fileName);
oneList = fac->loader(fac, chromHash, cpp, track, dbTrack);
/* Save a few more settings. */
for (oneTrack = oneList; oneTrack != NULL; oneTrack = oneTrack->next)
{
ctAddToSettings(track, "tdbType", oneTrack->tdb->type);
if (dbTrack && oneTrack->dbTrackType != NULL)
ctAddToSettings(track, "dbTrackType", oneTrack->dbTrackType);
if (!trackDbSetting(track->tdb, "inputType"))
ctAddToSettings(track, "inputType", fac->name);
if (dataUrl)
ctAddToSettings(track, "dataUrl", dataUrl);
if (!ctGenome(track) && ctDb)
ctAddToSettings(track, "db", ctDb);
}
}
trackList = slCat(trackList, oneList);
}
struct slName *browserLines = customPpTakeBrowserLines(cpp);
char *initialPos = browserLinePosition(browserLines);
/* Finish off tracks -- add auxiliary settings, fill in some defaults,
* and adjust priorities so tracks are not all on top of each other
* if no priority given. */
for (track = trackList; track != NULL; track = track->next)
{
if (track->tdb->priority == 0)
{
prio += 0.001;
track->tdb->priority = prio;
}
if (initialPos)
ctAddToSettings(track, "initialPos", initialPos);
if (track->bedList)
{
/* save item count and first item because if track is
* loaded to the database, the bedList will be available next time */
struct dyString *ds = dyStringNew(0);
dyStringPrintf(ds, "%d", slCount(track->bedList));
ctAddToSettings(track, "itemCount", cloneString(ds->string));
dyStringClear(ds);
dyStringPrintf(ds, "%s:%d-%d", track->bedList->chrom,
track->bedList->chromStart, track->bedList->chromEnd);
ctAddToSettings(track, "firstItemPos", cloneString(ds->string));
dyStringFree(&ds);
}
char *setting = browserLinesToSetting(browserLines);
if (setting)
ctAddToSettings(track, "browserLines", setting);
trackDbPolish(track->tdb);
}
/* Save return variables, clean up, and go home. */
if (retBrowserLines != NULL)
*retBrowserLines = browserLines;
customPpFree(&cpp);
hFreeConn(&ctConn);
return trackList;
}
struct customTrack *customFactoryParse(char *genomeDb, char *text, boolean isFile,
struct slName **retBrowserLines)
/* Parse text into a custom set of tracks. Text parameter is a
* file name if 'isFile' is set. Die if the track is not for genomeDb. */
{
return customFactoryParseOptionalDb(genomeDb, text, isFile, retBrowserLines, TRUE);
}
struct customTrack *customFactoryParseAnyDb(char *genomeDb, char *text, boolean isFile,
struct slName **retBrowserLines)
/* Parse text into a custom set of tracks. Text parameter is a
* file name if 'isFile' is set. Track does not have to be for hGetDb(). */
{
return customFactoryParseOptionalDb(genomeDb, text, isFile, retBrowserLines, FALSE);
}
static void readAndIgnore(char *fileName)
/* Read a few bytes from fileName, so its access time is updated. */
{
char buf[256];
FILE *f = mustOpen(fileName, "r");
mustGetLine(f, buf, sizeof(buf));
fclose(f);
}
static boolean testFileSettings(struct trackDb *tdb, char *ctFileName)
/* Return TRUE unless tdb has a setting that ends in File but doesn't
* specify an existing local file. */
{
boolean isLive = TRUE;
struct hashEl *fileSettings = trackDbSettingsLike(tdb, "*File");
struct hashEl *s;
for (s = fileSettings; s != NULL; s = s->next)
{
char *fileName = (char *)(s->val);
if (fileExists(fileName))
{
readAndIgnore(fileName);
verbose(4, "setting %s: %s\n", s->name, fileName);
}
else
{
isLive = FALSE;
verbose(3, "Custom track %s setting-file %s=%s does not exist\n",
ctFileName, s->name, fileName);
break;
}
}
hashElFreeList(&fileSettings);
return isLive;
}
// TODO: remove touchOldUdcSettings very soon
// after the rollout of the next code, i.e. on 2009-08-24.
static void touchOldUdcSettings(struct trackDb *tdb)
/* Touch existing local udcCache bitmap and sparse files. */
{
char *url = trackDbSetting(tdb, "dataUrl");
if (url)
{
struct slName *el, *list = udcFileCacheFiles(url, udcDefaultDir());
for (el = list; el; el = el->next)
{
if (fileExists(el->name))
{
readAndIgnore(el->name);
verbose(4, "setting dataUrl: %s\n", el->name);
}
}
slFreeList(&list);
}
}
static void touchUdcSettings(struct trackDb *tdb)
/* Touch existing local udcCache bitmap and sparse files. */
{
touchOldUdcSettings(tdb); // remove this line after 2009-08-24, see above.
char *url = trackDbSetting(tdb, "bigDataUrl");
if (url)
{
struct slName *el, *list = udcFileCacheFiles(url, udcDefaultDir());
for (el = list; el; el = el->next)
{
if (fileExists(el->name))
{
readAndIgnore(el->name);
verbose(4, "setting bigDataUrl: %s\n", el->name);
}
}
slFreeList(&list);
}
}
void customFactoryTestExistence(char *genomeDb, char *fileName, boolean *retGotLive,
boolean *retGotExpired)
/* Test existence of custom track fileName. If it exists, parse it just
* enough to tell whether it refers to database tables and if so, whether
* they are alive or have expired. If they are live, touch them to keep
* them active. */
{
struct customTrack *trackList = NULL, *track = NULL;
char *line = NULL;
struct sqlConnection *ctConn = NULL;
boolean dbTrack = ctDbUseAll();
if (!fileExists(fileName))
{
if (retGotExpired)
*retGotExpired = TRUE;
return;
}
struct lineFile *lf = customLineFile(fileName, TRUE);
/* wrap a customPp object around it. */
struct customPp *cpp = customPpNew(lf);
lf = NULL;
if (dbTrack)
ctConn = hAllocConn(CUSTOM_TRASH);
/* Loop through this once for each track. */
while ((line = customPpNextReal(cpp)) != NULL)
{
boolean isLive = TRUE;
/* Parse out track line and save it in track var.
* First time through make up track var from thin air
* if no track line. Find out explicit type setting if any.
* Also make sure settingsHash is set up. */
lf = cpp->fileStack;
if (startsWithWord("track", line))
{
track = trackLineToTrack(genomeDb, line, cpp->fileStack->lineIx);
}
else if (trackList == NULL)
/* In this case we handle simple files with a single track
* and no track line. */
{
char defaultLine[256];
safef(defaultLine, sizeof defaultLine,
"track name='%s' description='%s'",
CT_DEFAULT_TRACK_NAME, CT_DEFAULT_TRACK_DESCR);
track = trackLineToTrack(genomeDb, defaultLine, 1);
customPpReuse(cpp, line);
}
else
{
errAbort("Expecting 'track' line, got %s\nline %d of %s",
line, lf->lineIx, lf->fileName);
}
assert(track);
assert(track->tdb);
/* don't verify database for custom track -- we might be testing existence
* for another database. */
/* Handle File* settings */
isLive = (isLive && testFileSettings(track->tdb, fileName));
/* Handle bigDataUrl udc settings */
touchUdcSettings(track->tdb);
if (track->dbDataLoad)
/* Track was loaded into the database -- check if it still exists. */
{
if (ctConn && ctDbTableExists(ctConn, track->dbTableName))
{
if (isLive)
{
/* Touch database table to keep it alive. */
ctTouchLastUse(ctConn, track->dbTableName, TRUE);
}
}
else
{
isLive = FALSE;
verbose(3, "Custom track %s dbTableName %s does not exist\n",
fileName, track->dbTableName);
}
}
else
/* Track data in this file. */
{
while (customFactoryNextRealTilTrack(cpp))
/* Skip data lines until we get to next track or EOF. */
;
}
if (isLive)
{
if (retGotLive)
*retGotLive = TRUE;
}
else
{
if (retGotExpired)
*retGotExpired = TRUE;
}
slAddHead(&trackList, track);
}
customPpFree(&cpp);
hFreeConn(&ctConn);
}
char *ctInitialPosition(struct customTrack *ct)
/* return initial position plucked from browser lines,
* or position of first element if none specified */
{
char buf[128];
char *pos = trackDbSetting(ct->tdb, "initialPos");
if (!pos && ct->bedList)
{
safef(buf, sizeof(buf), "%s:%d-%d", ct->bedList->chrom,
ct->bedList->chromStart, ct->bedList->chromEnd);
pos = buf;
}
if (pos)
return cloneString(pos);
return NULL;
}
char *ctDataUrl(struct customTrack *ct)
/* return URL where data can be reloaded, if any */
{
return trackDbSetting(ct->tdb, "dataUrl");
}
char *ctHtmlUrl(struct customTrack *ct)
/* return URL where doc can be reloaded, if any */
{
return trackDbSetting(ct->tdb, "htmlUrl");
}
char *ctInputType(struct customTrack *ct)
/* return type of input */
{
char *type = trackDbSetting(ct->tdb, "inputType");
if (type == NULL)
type = trackDbSetting(ct->tdb, "tdbType");
return type;
}
int ctItemCount(struct customTrack *ct)
/* return number of 'items' in track, or -1 if unknown */
{
if (ct->bedList)
{
return (slCount(ct->bedList));
}
char *val = trackDbSetting(ct->tdb, "itemCount");
if (val)
return (sqlUnsigned(val));
return -1;
}
char *ctFirstItemPos(struct customTrack *ct)
/* return position of first item in track, or NULL if wiggle or
* other "non-item" track */
{
return trackDbSetting(ct->tdb, "firstItemPos");
}
char *ctOrigTrackLine(struct customTrack *ct)
/* return initial setting by user for track line */
{
return trackDbSetting(ct->tdb, "origTrackLine");
}
+
+void customFactoryEnableExtraChecking(boolean enable)
+/* Enable/disable extra checking of custom tracks in customFactoryParse.
+ * E.g. extra checking is great the first time we read in a custom track,
+ * but we don't need it for every hgTracks call. */
+{
+doExtraChecking = enable;
+}
+