404563416231e2b5a57eeee4fe09b0d9d2b77e60
max
  Wed Apr 2 05:50:41 2014 -0700
changing hgLiftOver strategy in the browserbox: downloading liftoverfiles to local disk instead of streaming them through http, refs #11957
and refs #12717

diff --git src/hg/lib/hdb.c src/hg/lib/hdb.c
index 614786a..8028db5 100644
--- src/hg/lib/hdb.c
+++ src/hg/lib/hdb.c
@@ -27,30 +27,32 @@
 #include "grp.h"
 #include "twoBit.h"
 #include "ra.h"
 #include "genbank.h"
 #include "chromInfo.h"
 #ifndef GBROWSE
 #include "axtInfo.h"
 #include "ctgPos.h"
 #include "hubConnect.h"
 #include "customTrack.h"
 #include "hgFind.h"
 #endif /* GBROWSE */
 #include "hui.h"
 #include "trackHub.h"
 #include "udc.h"
+#include "paraFetch.h"
+#include "filePath.h"
 
 
 #ifdef LOWELAB
 #define DEFAULT_PROTEINS "proteins060115"
 #define DEFAULT_GENOME "Pyrobaculum aerophilum"
 #else
 #define DEFAULT_PROTEINS "proteins"
 #define DEFAULT_GENOME "Human"
 #endif
 
 
 static struct sqlConnCache *hdbCc = NULL;  /* cache for primary database connection */
 static struct sqlConnCache *centralCc = NULL;
 static char *centralDb = NULL;
 static struct sqlConnCache *centralArchiveCc = NULL;
@@ -1253,86 +1255,125 @@
 struct sqlConnection *conn = hAllocConn(db);
 struct sqlResult *sr;
 char **row;
 
 sr = sqlGetResult(conn, "NOSQLINJ select chrom from chromInfo");
 while ((row = sqlNextRow(sr)) != NULL)
     {
     struct slName *el = slNameNew(row[0]);
     slAddHead(&list, el);
     }
 sqlFreeResult(&sr);
 hFreeConn(&conn);
 return list;
 }
 
-char *hReplaceGbdb(char* fileName)
- /* Returns a gbdb filename, potentially rewriting it according to hg.conf
-  * If the settings gbdbLoc1 and gbdbLoc2 are found, try them in order, by 
-  * replacing /gbdb/ with the new locations.
-  * If after the replacement of gbdbLoc1 the resulting fileName does not exist,
-  * gbdbLoc2 is used.
-  * This function does not guarantee that the filename exists.
-  * We assume /gbdb/ does not appear somewhere inside a fileName.
-  * Result has to be free'd.
- * */
+char *hReplaceGbdbLocal(char* fileName)
+ /* Returns a gbdb filename, potentially rewriting it according to hg.conf's gbdbLoc1 */
+ /* Result has to be freed */
 {
 if (fileName==NULL)
     return fileName;
 
 char* newGbdbLoc = cfgOption("gbdbLoc1");
 char* path;
 
 // if no config option set or not a /gbdb filename, then just return
-// otherwise replace /gbdb/ with the new prefix and return if exists.
+// otherwise replace /gbdb/ with the new prefix and return it
 if (newGbdbLoc==NULL || !startsWith("/gbdb/", fileName))
    return cloneString(fileName);
 
 path = replaceChars(fileName, "/gbdb/", newGbdbLoc);
+return path;
+}
+
+char *hReplaceGbdb(char* fileName)
+ /* Returns a gbdb filename, potentially rewriting it according to hg.conf
+  * If the settings gbdbLoc1 and gbdbLoc2 are found, try them in order, by 
+  * replacing /gbdb/ with the new locations.
+  * If after the replacement of gbdbLoc1 the resulting fileName does not exist,
+  * gbdbLoc2 is used.
+  * This function does not guarantee that the returned filename exists.
+  * We assume /gbdb/ does not appear somewhere inside a fileName.
+  * Result has to be free'd.
+ * */
+{
+char *path = hReplaceGbdbLocal(fileName);
 if (fileExists(path))
     return path;
 
 // if the file did not exist, replace with gbdbLoc2
-newGbdbLoc = cfgOption("gbdbLoc2");
+char* newGbdbLoc = cfgOption("gbdbLoc2");
 if (newGbdbLoc==NULL)
     return path;
 
 freeMem(path);
 path = replaceChars(fileName, "/gbdb/", newGbdbLoc);
 return path;
 }
 
 char *hReplaceGbdbSeqDir(char *path, char *db)
 /* similar to hReplaceGbdb, but accepts a nib or 2bit "directory" (basename) under
  * gbdb, like /gbdb/hg19 (which by jkLib is translated to /gbdb/hg19/hg19.2bit).
  hReplaceGbdb would check only if the dir exists. For 2bit basename, we
  have to check if the 2bit file exists, do the rewriting, then strip off the
  2bit filename again. 
  This function works with .nib directories, but nib does not support opening
- from URLs.  As of Feb 2014, only hg16 and anoGam1 use a .nib directory.
+ from URLs.  As of Feb 2014, only hg16 and anoGam1 have no .2bit file.
 */
 {
 char buf[4096];
 safef(buf, sizeof(buf), "%s/%s.2bit", path, db);
 char *newPath = hReplaceGbdb(buf);
 
 char dir[4096];
 splitPath(newPath, dir, NULL, NULL);
 freeMem(newPath);
 return cloneString(dir);
 }
 
+char* hReplaceGbdbMustDownload(char* path)
+/* given a location in /gbdb, rewrite it to the new location using gbdbLoc1 and download it
+ * if needed from gbdbLoc2.
+ * Used for adding files on the fly on mirrors or the box.
+ * Returns local path, needs to be freed. Aborts if download failed.
+ * */
+{
+char* locPath = hReplaceGbdbLocal(path);
+// skip if file is there
+if (fileExists(locPath))
+    return locPath;
+// skip if we already got a url or cannot rewrite a path
+char* url = hReplaceGbdb(path);
+if (sameString(locPath, url))
+    return locPath;
+
+// check that we can write to the dest dir
+if ((access(locPath, W_OK))==0)
+    errAbort("trying to download %s but no write access to %s\n", url, locPath);
+
+char locDir[1024];
+splitPath(locPath, locDir, NULL, NULL);
+makeDirsOnPath(locDir);
+
+bool success = parallelFetch(url, locPath, 1, 1, TRUE, FALSE);
+if (! success)
+    errAbort("Could not download %s to %s\n", url, locPath);
+
+return locPath;
+}
+
 char *hTryExtFileNameC(struct sqlConnection *conn, char *extFileTable, unsigned extFileId, boolean abortOnError)
 /* Get external file name from table and ID.  Typically
  * extFile table will be 'extFile' or 'gbExtFile'
  * If abortOnError is true, abort if the id is not in the table or if the file
  * fails size check, otherwise return NULL if either of those checks fail.   
  * Please freeMem the result when you are done with it. 
  * (requires conn passed in) 
  */
 {
 char query[256];
 struct sqlResult *sr;
 char **row;
 long long dbSize, diskSize;
 char *path;