7ac21bf1c3ef2b54beec22d23f6a6d389e229d92
galt
  Thu Feb 21 02:25:27 2019 -0800
fixing it to handle 0 custom tracks to backup by skipping the backup button. Fixed the show to match the make and correctly deal with both complete custom-trash-table types as well as newer bigDataUrl cts. Also fixed a bug where the origTrackLine is missing from CTs generated by Table Browser. Currently just try to detect the missing value, and if tdbType is wig then feed in a simple track line for generic wiggles.

diff --git src/hg/hgSession/backup.c src/hg/hgSession/backup.c
index ea89aaf..7564b6b 100644
--- src/hg/hgSession/backup.c
+++ src/hg/hgSession/backup.c
@@ -377,41 +377,73 @@
     }
 return result;
 }
 
 
 /* hold extra data that does not fit in the track struct itself,
  * but which is track-specific */
 struct ctExtra
     {
     struct ctExtra *next;
     char *name;  // important to identity of the track in hgCustom
     char *browserLines;
     char *trackLine;
     int tableRows;
     unsigned long tableDataLength;
+    char *bigDataUrl;
     };
 
 struct downloadResults
     {
     struct downloadResults *next;
     char *db;
     char *ctPath;    // path to ct file in trash
     struct customTrack *cts;  // tracks
     struct ctExtra *ctExtras; // parallel additional info to the tracks
     };
 
+
+void addFieldToTrackLine(struct dyString *dy, struct customTrack *track, char *field, boolean isRequired)
+/* add optional or required field and its value to a track line */
+{
+char *val = hashFindVal(track->tdb->settingsHash, field);
+if (val)
+    {
+    dyStringPrintf(dy, " %s='%s'", field, val);
+    }
+else
+    {
+    if (isRequired)
+	errAbort("required field %s is missing from custom track settings", field);
+    }
+}
+
+char *fabricateWigTrackline(struct customTrack *track)
+/* make up a missing trackline for a ct exported from Table Browser as a wig */
+{
+struct dyString *dy = dyStringNew(1024);
+dyStringPrintf(dy, "track ");
+addFieldToTrackLine(dy, track,"name", TRUE);  // Required
+dyStringPrintf(dy, " type='wiggle_0'");
+addFieldToTrackLine(dy, track,"description", FALSE);
+addFieldToTrackLine(dy, track,"visibility", FALSE);
+addFieldToTrackLine(dy, track,"priority", FALSE);
+addFieldToTrackLine(dy, track,"altColor", FALSE);
+return dyStringCannibalize(&dy);
+}
+ 
+
 struct downloadResults *processCtsForDownloadInternals(char *contents, char **pTrackHubsVar)
 /* Process the saved session cart contents,
  * looking for custom-tracks which are db-specific. */
 {
 
 struct downloadResults *resultList = NULL;
 
 if (!contents)
     return resultList;
 
 char *contentsToChop = cloneString(contents);
 char *namePt = contentsToChop;
 
 struct sqlConnection *ctConn = hAllocConn(CUSTOM_TRASH);
 
@@ -456,54 +488,68 @@
 
 		// what to save in ct output
 
 		// name is important because hgCustom identifies by this string
 		char *name = hashFindVal(track->tdb->settingsHash, "name");
 		extra->name = cloneString(name);
 
                 char *browserLines = hashFindVal(track->tdb->settingsHash, "browserLines");
 		// to handle multiple-lines replace semi-colon ";" with newline.
 		if (browserLines)
 		    replaceChar(browserLines,';','\n');	
 		extra->browserLines = cloneString(browserLines);
 
                 char *origTrackLine = hashFindVal(track->tdb->settingsHash, "origTrackLine");
 		extra->trackLine = cloneString(origTrackLine);
+		if (!extra->trackLine) // Table Browser creates some wiggles without an origTrackLine
+		    {
+		    char *tdbType = hashFindVal(track->tdb->settingsHash, "tdbType");
+		    if (tdbType && startsWith("wig ", tdbType))
+			{
+			extra->trackLine = fabricateWigTrackline(track);
+			}
+		    }
 
 		// is it weird that the loader customFactoryTestExistence() did not do this for me?
 		char *wibFilePath = hashFindVal(track->tdb->settingsHash, "wibFile");
 		if (wibFilePath && fileExists(wibFilePath))
 		    {
 		    track->wibFile = wibFilePath;
 		    }
 		char *mafFilePath = hashFindVal(track->tdb->settingsHash, "mafFile");
 		if (mafFilePath && fileExists(mafFilePath))
 		    {
 		    track->wibFile = mafFilePath;
 		    }
 		// old vcf type (not vcfTabix)
 		char *vcfFilePath = hashFindVal(track->tdb->settingsHash, "vcfFile");
 		if (vcfFilePath && fileExists(vcfFilePath))
 		    {
 		    track->wibFile = vcfFilePath;
 		    }
 
 		if (track->dbTrack && track->dbDataLoad && track->dbTableName)
 		    {
 		    extra->tableRows = sqlTableSizeIfExists(ctConn, track->dbTableName); 
 		    extra->tableDataLength = sqlTableDataSizeFromSchema(ctConn, CUSTOM_TRASH, track->dbTableName);
 		    }
+
+
+		char *bigDataUrl = hashFindVal(track->tdb->settingsHash, "bigDataUrl");
+		if (bigDataUrl)
+		    extra->bigDataUrl = bigDataUrl;
+
 		slAddHead(&result->ctExtras,extra);
 		}
 	    slReverse(&result->ctExtras);
 
 	    }
 
 	slAddHead(&resultList,result);
 	}
     else if (sameString("trackHubs", namePt))
 	{
 	cgiDecode(dataPt, dataPt, strlen(dataPt));
 	if (pTrackHubsVar)
 	    *pTrackHubsVar = cloneString(dataPt);
 	}
 
@@ -522,78 +568,88 @@
  * looking for custom-tracks which are db-specific. */
 {
 
 struct downloadResults *resultList = processCtsForDownloadInternals(contents, NULL);
 
 printf("<H2>Custom Tracks</H2>");
 
 printf("You can backup the custom tracks which you previously uploaded to UCSC Genome Browser servers.<br>"
     "It will create a single .tar.gz file with custom track data,<br>"
     "which you can then download to your own machine for use as a backup.<br>"
     "\n");
 
 struct downloadResults *result = NULL;
 
 
-int nonBigDataUrlCount = 0;
 int ctCount = 0;
 long totalDataToDownload = 0;
 
 for (result=resultList; result; result=result->next)
     {
 
     if (!result->ctPath)  // ct.bed was missing for the db
 	continue;
 
     printf("<h3>Database %s</h3>\n", result->db);
 
+    //printf("result->ctPath %s <br>\n", result->ctPath);  // DEBUG REMOVE
+
     if (startsWith("hub_", result->db))
 	{
 	unsigned hubId = isLiveHub(result->db, NULL, TRUE);
 	if (hubId == 0)
 	    {
 	    printf("Skipping HUB Database %s. "
 		"This hub db is not currently available and will not get backed up.<BR><BR>\n", result->db);
 	    continue; // skip db if not active.
 	    }
 	}
     else
 	{
 	if (!isActiveDb(result->db))
 	    {
 	    printf("Skipping Database %s. "
 		"The db is not currently active and will not get backed up.<BR><BR>\n", result->db);
 	    continue; // skip db if not active.
 	    }
 	}
 
     struct customTrack *cts = result->cts, *track = NULL;
     struct ctExtra *extras = result->ctExtras, *extra = NULL;
 
     printf("<table>\n");
 
     for (track=cts,extra=extras; track; track=track->next,extra=extra->next)
 	{
+	//printf("track name %s <br>\n", extra->name);  // DEBUG REMOVE
+	//printf("track->dbTrack %d <br>\n", track->dbTrack);  // DEBUG REMOVE
+	//printf("track->dbTableName %s <br>\n", track->dbTableName);  // DEBUG REMOVE
+	//printf("track->dbTrackType %s <br>\n", track->dbTrackType);  // DEBUG REMOVE
+	//printf("track->dbDataLoad %d <br>\n", track->dbDataLoad);  // DEBUG REMOVE
+	//printf("extra->bigDataUrl %s <br>\n", extra->bigDataUrl);  // DEBUG REMOVE
+
+	boolean wibMissing = track->wibFile && !fileExists(track->wibFile);
+
+	if (extra->bigDataUrl || (track->dbTrack && track->dbDataLoad && track->dbTableName && !wibMissing))
+	    {
 	    long trackDataToDownload = 0;
 
 	    ++ctCount;
 
-	// handle user-friendly sizes GB MB KB and B
-	if (track->dbTrack && track->dbDataLoad && track->dbTableName)
+	    if (!extra->bigDataUrl)
 		{
 		char greek[32];
-	    ++nonBigDataUrlCount;
 		sprintWithGreekByte(greek, sizeof(greek), extra->tableDataLength);
 		trackDataToDownload += extra->tableDataLength;
 
 		// handle wiggle cts which have an additional wig binary
 		// wibFile=../trash/ct/hgtct_genome_542_dc1750.wib
 		// wibFile=../trash/ct/ct_hgwdev_galt_e83f_3892d0.maf 
 		// wibFIle=../trash/ct/ct_hgwdev_galt_4ba3_415cc0.vcf
 		if (track->wibFile && fileExists(track->wibFile))
 		    {
 		    long wibFileSize = fileSize(track->wibFile);
 		    char greek[32];
 		    sprintWithGreekByte(greek, sizeof(greek), wibFileSize);
 		    trackDataToDownload += wibFileSize;
 		    }
 		}
@@ -603,43 +659,47 @@
 		{
 		long htmlFileSize = fileSize(track->htmlFile);
 		char greek[32];
 		sprintWithGreekByte(greek, sizeof(greek), htmlFileSize);
 		trackDataToDownload += htmlFileSize;
 		}
 
 	    char greek[32];
 	    sprintWithGreekByte(greek, sizeof(greek), trackDataToDownload);
 	    if (trackDataToDownload == 0)  // suppress 0.0 B
 		greek[0] = 0; // empty string
 	    printf("<tr><td>%s</td><td>%s</td></tr>\n", extra->name, greek); // track name
 
 	    totalDataToDownload += trackDataToDownload;
 	    }
+	}
 
     printf("</table>\n");
 
     }
 
 printf("<br>\n");
 printf("%d custom tracks found.<br>\n", ctCount);
 
 char greek[32];
 sprintWithGreekByte(greek, sizeof(greek), totalDataToDownload);
 printf("Total custom track data to backup: %s ", greek);
 
+if (ctCount > 0)
+    {
     cgiMakeButton(hgsMakeDownloadPrefix, "create custom tracks backup archive");
+    }
 
 printf("<br>\n");
 printf("<br>\n");
 
 }
 
 
 void showDownloadSessionCtData(struct hashEl *downloadList)
 /* Show download page for the given session */
 {
 char query[512];
 char **row = NULL;
 struct sqlResult *sr = NULL;
 
 puts("Content-Type:text/html\n");
@@ -879,30 +939,31 @@
 while(TRUE)
     {
     randPath=makeRandomKey(128+33); // at least 128 bits of protection, 33 for the world population size.
     // Avoid the possibility of somehow doing more than one instance of the session
     // backup at the same time.  
     // On the extremely rare chance that the directory already exists,
     // just call the function again.
     safef(tempOutRand, sizeof tempOutRand, "%s/%s", tempOut, randPath);
     if (makeDir(tempOutRand))
 	break;
     ++randCount;
     if (randCount > 100) // should never happen
 	errAbort("unable to create random output dir.");  
     }
 
+int ctCount = 0;
 int foundCount = 0;
 if ((row = sqlNextRow(sr)) != NULL)
     {
     ++foundCount;
 
     char *contents = row[0];
 
 
     char *trackHubsVar = NULL;
 
     struct downloadResults *resultList = processCtsForDownloadInternals(contents, &trackHubsVar);
 
     struct downloadResults *result = NULL;
 
     for (result=resultList; result; result=result->next)
@@ -935,71 +996,89 @@
 	else
 	    {
 	    if (!isActiveDb(result->db))
 		{
 		continue; // skip db if not active.
 		}
 	    }
 
 
 	printf("<h3>%s</h3>\n", result->db);
 	dyStringPrintf(dyProg,"<h3>%s</h3>\n", result->db); 
 	updateProgessFile(backgroundProgress, dyProg);
 	lazarusLives(20 * 60);
 
 	
+	//printf("result->ctPath %s <br>\n", result->ctPath);  // DEBUG REMOVE
+
+
 	if (startsWith("hub_", result->db))
 	    {
 	    // Save the hubUrl now in case it gets restored on another machine
 	    // that does NOT have that hub loaded on it.
 	    // save output as hubUrl text file 
 	    char hubUrlPath[1024];
 	    safef(hubUrlPath, sizeof hubUrlPath, "%s/hubUrl", outDbDir);
 	    FILE *f = mustOpen(hubUrlPath, "w");
 	    fprintf(f, "%s", hubUrl);  // NO NEWLINE
 	    carefulClose(&f);
 	    }
 
 	char cwd[PATH_LEN];
 	if (!getcwd(cwd, sizeof(cwd)))
 	    errnoAbort("unable to get current dir.");		
 
 	for (track=cts,extra=extras; track; track=track->next,extra=extra->next)
 	    {
 
+	    //printf("track name %s <br>\n", extra->name);  // DEBUG REMOVE
+	    //printf("track->dbTrack %d <br>\n", track->dbTrack);  // DEBUG REMOVE
+	    //printf("track->dbTableName %s <br>\n", track->dbTableName);  // DEBUG REMOVE
+	    //printf("track->dbTrackType %s <br>\n", track->dbTrackType);  // DEBUG REMOVE
+	    //printf("track->dbDataLoad %d <br>\n", track->dbDataLoad);  // DEBUG REMOVE
+	    //printf("extra->bigDataUrl %s <br>\n", extra->bigDataUrl);  // DEBUG REMOVE
+
+	    boolean wibMissing = track->wibFile && !fileExists(track->wibFile);
+
+	    if (extra->bigDataUrl || (track->dbTrack && track->dbDataLoad && track->dbTableName && !wibMissing))
+		{
+
+		++ctCount;
 		printf("%s <br>\n", extra->name);
 
 		dyStringPrintf(dyProg, "%s <br>\n", extra->name);
 		updateProgessFile(backgroundProgress, dyProg);
 		lazarusLives(20 * 60);
 
+
 		char outNameCt[2014];
 		safef(outNameCt, sizeof outNameCt, "%s/%s.ct", outDbDir, extra->name);
 		FILE *fct = mustOpen(outNameCt, "w");
 
 		// write the track header
 		if (extra->browserLines)
 		    fprintf(fct, "%s", extra->browserLines);  // should have ; converted \n already
-	    fprintf(fct, "%s\n", extra->trackLine);
 
-	    boolean wibMissing = track->wibFile && !fileExists(track->wibFile);
+		if (!extra->trackLine)
+		    errAbort("origTrackLine is NULL!");
+		fprintf(fct, "%s\n", extra->trackLine);
     
-	    if (track->dbTrack && track->dbDataLoad && track->dbTableName && !wibMissing)
-		{
 
 		//printf("%s <br>\n", track->wibFile);  // DEBUG REMOVE
 		
+		if (!extra->bigDataUrl)
+		    {
 
 		    // handle wiggle cts which have an additional wig binary
 		    // wibFile='../trash/ct/hgtct_genome_542_dc1750.wib'
 		    // wibFile='../trash/ct/ct_hgwdev_galt_e83f_3892d0.maf'
 		    // wibFile='../trash/ct/ct_hgwdev_galt_4ba3_415cc0.vcf'
 		    // symlink for speed, file should not change
 		    if (track->wibFile)
 			{
 			if (endsWith(track->wibFile, ".wib"))
 			    {
 			    // dump wig as ascii
 			    char outNameWig[2014];
 			    safef(outNameWig, sizeof outNameWig, "%s/%s.wig", outDbDir, extra->name);
 			    doOutWigData(track->dbTableName, CUSTOM_TRASH, outNameWig); // no easy way to append it to .ct directly 
 			    // append text to ct
@@ -1010,49 +1089,54 @@
 			    {
 			    // append text to ct
 			    appendTrashFileToCt(fct, track->wibFile);
 			    }
 			if (endsWith(track->wibFile, ".vcf"))
 			    {
 			    // append text to ct
 			    appendTrashFileToCt(fct, track->wibFile);
 			    }
 			}
 		    else  // simple BED-like
 			{
 			saveSqlDataForTable(track->dbTableName, fct);
 			}
 
+		    }
 		// handle extra htmlFile track doc
 		// htmlFile=../trash/ct/ct_hgwdev_galt_d011_383c50.html 
 		// symlink for speed, file should not change
 		if (track->htmlFile && fileExists(track->htmlFile))
 		    {
 		    makeTrashFileLink(track->htmlFile, cwd, outDbDir, extra->name);
 		    }
-		}
 
 		carefulClose(&fct);
 		}
+
+	    }
 	}
 
     }
 sqlFreeResult(&sr);
  
 if (foundCount == 0)
     errAbort("No session found for hgsid=%u", cartSessionRawId(cart));
 
+if (ctCount == 0)
+    errAbort("No custom tracks found for hgsid=%u", cartSessionRawId(cart));
+
 char archiveName[1024];
 safef(archiveName, sizeof archiveName, "savedSessionCtRaw.tar.gz");
 
 dyStringPrintf(dyProg, "<br>\n");
 int saveDySize = dyProg->stringSize;
 dyStringPrintf(dyProg, "creating and compressing archive %s <br>\n", archiveName);
 updateProgessFile(backgroundProgress, dyProg);
 lazarusLives(20 * 60);
 
 // create the archive
 char cmd[2048];
 safef(cmd, sizeof cmd, "cd %s; tar -cpzhf %s *", tempOutRand, archiveName);
 mustSystem(cmd);
 
 dyProg->stringSize = saveDySize;  // restore prev size, popping.