03a7221f2973a860ee4363183fc5c582ce0d6178
max
  Tue May 19 10:25:26 2026 -0700
tons of changes to hubtools, committing now for chris

diff --git src/utils/hubtools/hubtools src/utils/hubtools/hubtools
index 4fd573ffdf6..0bc7a88cbbc 100755
--- src/utils/hubtools/hubtools
+++ src/utils/hubtools/hubtools
@@ -1,25 +1,42 @@
 #!/usr/bin/env python3
 
 import logging, sys, optparse, os, json, subprocess, shutil, string, glob, tempfile, re
-import shlex
+import shlex, urllib, time, tarfile, hashlib
+
 from pathlib import Path
 from collections import defaultdict, OrderedDict
 from os.path import join, basename, dirname, isfile, relpath, abspath, splitext, isdir, normpath
+from urllib.parse import unquote
+import concurrent.futures
+
+requestsLoaded = True
+try:
+    import requests # if this fails, install the library with 'python -m pip install requests --user'
+except:
+    # most code now can work around the absence of the requests library
+    requestsLoaded = False
+
 #import pyyaml   # not loaded here, so it's not a hard requirement, is lazy loaded in parseMetaYaml()
 
 # ==== functions =====
+
+# debugging: when activated with -d, output more information and do not remove any temp files
+debugMode = False
+# full extra verbose output of all HTTP requests sent, there is no command line option for this
+doVerbose = False
+
 # allowed file types by hubtools up
 # copied from the javascript file hgHubConnect.js
 fileTypeExtensions = {
     "bigBed": [ ".bb", ".bigbed" ],
     "bam": [ ".bam" ],
     "vcf": [ ".vcf" ],
     "vcfTabix": [ ".vcf.gz", "vcf.bgz" ],
     "bigWig": [ ".bw", ".bigwig" ],
     "hic": [ ".hic" ],
     "cram": [ ".cram" ],
     "bigBarChart": [ ".bigbarchart" ],
     "bigGenePred": [ ".bgp", ".biggenepred" ],
     "bigMaf": [ ".bigmaf" ],
     "bigInteract": [ ".biginteract" ],
     "bigPsl": [ ".bigpsl" ],
@@ -47,110 +64,117 @@
     int[blockCount] blockSizes; "Comma separated list of block sizes"
     int[blockCount] chromStarts; "Start positions relative to chromStart"
 """.split("\n")
 
 def parseArgs():
     " setup logging, parse command line arguments and options. -h shows auto-generated help page "
     parser = optparse.OptionParser("""usage: %prog [options] <cmd> - create and edit UCSC track hubs
 
     hubtools make <assemblyCode e.g. hg38>: create a track hub for all bigBed/bigWig files under a directory.
     Creates single-file hub.txt, and tries to guess reasonable settings from the file names:
     - bigBed/bigWig files in the current directory will be top level tracks
     - big* files in subdirectories become composites
     - for every filename, the part before the first dot becomes the track base name
     - if a directory has more than 80% of track base names with both a bigBed
       and bigWig file, views are activated for this composite
-    - track attributes can be changed using tracks.ra, tracks.tsv, tracks.json or tracks.yaml files, in
-      each top or subdirectory
-    - The first column of tracks.tsv must be 'track'.
-    - tracks.json and tracks.yaml must have objects at the top level, the attributes are <trackName> or
-      the special attribute "hub".
+    - track attributes can be changed using tracks.tsv or json/ra/yaml files, in
+      each top or subdirectory. Both subdir and top directory are searched, subdir
+      values override any values specified in a parent directory.
+    - tracks.tsv (or tracks.json/ra/yaml) must have a first column named 'track'. For json and yaml,
+      the key is the track name. ".hub" is a special key to provide email/shortLabel of the hub.
+    - the track name in this column can be either the full one <parent>__fileBasename__type or just
+      the fileBasename, this makes it easier. In practice, filenames are unique enough and don't
+      usually clash between subdirs or filetype, in practice.
+    - the track order is the one on disk, so relatively random. To order tracks, create a hub,
+      use the "tab" command below, reorder tracks in tsv file, then re-run the "make" command.
 
     hubtools up <hubName>: upload files to hubSpace
     - needs ~/.hubtools.conf with a line apiKey="xxxx". Create a key by going to
       My Data > Track Hubs > Track Development on https://genome.ucsc.edu
     - uploads all files from the -i directory or the current dir if not specified.
 
     hubtools jbrowse <url> <db> : convert Jbrowse trackList.json files to hub.txt.
     - <url> is the URL to the Jbrowse2 installation, e.g. http://furlonglab.embl.de/FurlongBrowser/
     - <db> is assembly identifier
 
     hubtools tab <fname>: convert a hub.txt or trackDb.txt to tab-sep format, easier to bulk-edit with sed/cut/etc.
+       The resulting file if named tracks.tsv can be used as inputs for future runs.
     - <fname> is the input filename. "hub" and "genome" stanzas are skipped.
     - output goes to stdout
 
     hubtools conv -i myTsvDir
     - convert .tsv files in inDir to .bigBed files in current directory
 
-    hubtools archive -i xxx/archive -o outDir
-    - Create a hub from an extracted xxx.tar.gz file "archive" directory. You can download a .tar.gz from
-      in the Genome Browser under My Data > My Session. The archive contains all your custom tracks.
-      It is usually easier to use the 'ct' command, as this will automatically download an archive
-      an convert it, given a full genome browser URL with an "hgsid" parameter on the URL.
-
-    hubtools ct <hgTracksUrl>
-    - Download all custom tracks from a Genome Browser URL, either as a /s/ stable shortlink or 
-      the temporary hgsid=xxxx URL
+    hubtools ct <urlOrFile> [-o outDir]
+    - Create a hub in current dir or outDir from any URL with a hgsid, or a
+      session URL, or a local xxx.tar.gz track backup directory (See: My Data >
+      My Session) Processes all genomes with custom tracks. Downloads
+      bigDataUrl files into outDir.
 
     Examples:
-    hubtools conv -i myTsvs/
-    hubtools make hg38
+    hubtools conv -i myTsvs/ # convert tsv to bigBeds in current dir
+    hubtools make hg38 # make a hub in current dir
     hubtools jbrowse http://furlonglab.embl.de/FurlongBrowser/ dm3
-    hubtools tab hub.txt > tracks.tsv
-    hubtools session
+    hubtools tab hub.txt > tracks.tsv  # convert a hub to a .tsv file
 
-    tar xvfz SC_20230723_backup.tar.gz
-    hubtools archive -i archive -o hub
-    hubtools ct 'https://genome-euro.ucsc.edu/cgi-bin/hgTracks?db=hg38&position=chr7%3A155798978%2D155810671&hgsid=345826202_8d3Mpumjaw9IXYI2RDaSt5xMoUhH'
+    hubtools ct SC_20230723_backup.tar.gz -o hub # convert a backup archive to a track hub
+    hubtools ct 'https://genome-euro.ucsc.edu/cgi-bin/hgTracks?db=hg38&hgsid=345826202_8d3Mpumjaw9IXYI2RDaSt5xMoUhH'
+    # convert all custom tracks to a track hub
 
-    For the "hubtools make" step, you can specify additional options for your tracks via tracks.json/.yaml or tracks.tsv:
+    For the "hubtools make" and "ct" steps, you can specify additional options for your tracks via tracks.json/.yaml or tracks.tsv:
 
-    tracks.json can look like this, can have more keys, one per track, or the special key "hub":
+    tracks.json can look like this, can have more keys, one per track, or the special key ".hub":
     {
-        "hub" :      { "hub": "mouse_motor_atac", "shortLabel":"scATAC-seq Developing Cranial Motor Neurons" },
+        ".hub" :      { "hub": "mouse_motor_atac", "shortLabel":"scATAC-seq Developing Cranial Motor Neurons" },
         "myTrack" :  { "shortLabel" : "My nice track" }
     }
     tracks.tsv should look like this, other columns can be added:
 
         #track<tab>shortLabel
         myTrack<tab>My nice track
 
     For a list of all possible fields/columns see https://genome.ucsc.edu/goldenpath/help/trackDb/trackDbHub.html:
     """)
 
 
     parser.add_option("-i", "--inDir", dest="inDir", action="store", help="Input directory where files are stored. Default is current directory")
     parser.add_option("-o", "--outDir", dest="outDir", action="store", help="Input directory where hub.txt file is created. Default is same as input directory.")
 
     #parser.add_option("", "--igv", dest="igv", action="store", help="import an igv.js trackList.json file hierarchy")
-    parser.add_option("-d", "--debug", dest="debug", action="store_true", help="show verbose debug messages")
-    parser.add_option("-u", "--upload", dest="upload", action="store_true", help="upload all files from outDir to hubSpace")
+    parser.add_option("-d", "--debug", dest="debug", action="store_true", help="show debug messages, stacktrace on abort and do not delete temp files")
+    #parser.add_option("-u", "--upload", dest="upload", action="store_true", help="upload all files from outDir to hubSpace")
+    parser.add_option("", "--download", dest="doDownload", action="store_true", help="Download all bigDataUrl files to outDir")
     (options, args) = parser.parse_args()
 
     if len(args)==0:
         parser.print_help()
         exit(1)
 
+    global debugMode
     if options.debug:
+        debugMode = True
         logging.basicConfig(level=logging.DEBUG)
     else:
         logging.basicConfig(level=logging.INFO)
     return args, options
 
 def errAbort(msg):
     " print and abort) "
     logging.error(msg)
+    if debugMode:
+        raise # show stacktrace
+    else:
         sys.exit(1)
 
 def makedirs(d):
     if not isdir(d):
         os.makedirs(d)
 
 def parseConf(fname):
     " parse a hg.conf style file, return as dict key -> value (all strings) "
     logging.debug("Parsing "+fname)
     conf = {}
     for line in open(fname):
         line = line.strip()
         if line.startswith("#"):
             continue
         elif line.startswith("include "):
@@ -254,59 +278,62 @@
 def parseMetaJson(fname):
     " parse a json file and merge it into meta and return "
     logging.debug("Reading %s as json" % fname)
     newMeta = json.load(open(fname))
     return newMeta
 
 def parseMetaYaml(fname):
     " parse yaml file "
     import yaml # if this doesn't work, run 'pip install pyyaml'
     with open(fname) as stream:
         try:
             return yaml.safe_load(stream)
         except yaml.YAMLError as exc:
             logging.error(exc)
 
-def parseMeta(inDir):
-    " parse a tab-sep file with headers and return an ordered dict firstField -> dictionary "
-    fname = join(inDir, "tracks.tsv")
+def parseMeta(inDirs):
+    """ parse a tab-sep file with headers and return an ordered dict firstField -> dictionary.
+    Takes a list of input directories and the last directory overrides values in parent directories.
+    """
     meta = OrderedDict()
+    for inDir in inDirs:
+        fname = join(inDir, "tracks.tsv")
         if isfile(fname):
             tsvMeta = parseMetaTsv(fname)
             meta = allMetaOverride(meta, tsvMeta)
 
         fname = join(inDir, "tracks.json")
         if isfile(fname):
             jsonMeta = parseMetaJson(fname)
             meta = allMetaOverride(meta, jsonMeta)
 
         fname = join(inDir, "tracks.ra")
         if isfile(fname):
             raMeta = parseMetaRa(fname)
             meta = allMetaOverride(meta, raMeta)
 
         fname = join(inDir, "tracks.yaml")
         if isfile(fname):
             yamlMeta = parseMetaYaml(fname)
             meta = allMetaOverride(meta, yamlMeta)
 
         logging.debug("Got meta from %s: %s" % (inDir, str(meta)))
     return meta
 
 def writeHubGenome(ofh, db, inMeta):
     " create a hub.txt and genomes.txt file, hub.txt is just a template "
-    meta = inMeta.get("hub", {})
+    meta = inMeta.get(".hub", {})
     ofh.write("hub autoHub\n")
     ofh.write("shortLabel %s\n" % meta.get("shortLabel", "Auto-generated hub"))
     ofh.write("longLabel %s\n" % meta.get("longLabel", "Auto-generated hub"))
     #ofh.write("genomesFile genomes.txt\n")
     if "descriptionUrl" in meta:
         ofh.write("descriptionUrl %s\n" % meta["descriptionUrl"])
     ofh.write("email %s\n" % meta.get("email", "yourEmail@example.com"))
     ofh.write("useOneFile on\n\n")
 
     ofh.write("genome %s\n\n" % db)
     return ofh
 
 def readSubdirs(inDir, subDirs):
     " given a list of dirs, find those that are composite dirs (not supporting supertracks for now) "
     compDicts, superDicts = {}, {}
@@ -499,102 +526,102 @@
             doneTracks.add(trackBase)
 
     logging.debug("Track ordering from meta data used: %s" % newFiles.keys())
 
     # then add all other tracks
     for trackBase, fileData in fileDict.items():
         if trackBase not in doneTracks:
             newFiles[trackBase] = fileDict[trackBase]
             logging.debug("Not specified in meta, so adding at the end: %s" % trackBase)
 
     logging.debug("Final track order is: %s" % newFiles.keys())
 
     assert(len(newFiles)==len(fileDict))
     return newFiles
 
-def writeTdb(inDir, dirDict, dirType, tdbDir, ofh):
+def makeTrackDbEntries(inDir, dirDict, dirType, tdbDir, ofh):
     " given a dict with basename -> type -> filenames, write track entries to ofh "
     # this code is getting increasingly complex because it supports composites/views and pairing of bigBed/bigWig files
     # either this needs better comments or maybe a separate code path for this rare use case
     global compCount
 
     fnameDict = dirDict[dirType]
 
     for parentName, typeDict in fnameDict.items():
         if parentName is None: # top level tracks: use top tracks.tsv
-            subDir = inDir
-        else: # container tracks -> use tracks.tsv in the subdirectory
-            subDir = join(inDir, parentName)
+            subDirs = [inDir]
+        else: # container tracks -> use tracks.tsv in the parent and subdirectory
+            subDirs = [inDir, join(inDir, parentName)]
 
-        parentMeta = parseMeta(subDir)
+        parentMeta = parseMeta(subDirs)
 
         indent = 0
         parentHasViews = False
 
         groupMeta = {}
 
         if dirType=="comps":
             tdb = {
                     "track" : parentName,
                     "shortLabel": parentName,
                     "visibility" : "dense",
                     "compositeTrack" : "on",
                     "autoScale" : "group",
                     "type" : "bed 4"
                   }
             metaOverride(tdb, parentMeta)
             groupMeta = parentMeta
 
             parentHasViews = mostFilesArePaired(typeDict)
 
             if parentHasViews:
                 tdb["subGroup1"] = "view Views PK=Peaks SIG=Signals"
                 logging.info("Container track %s has >80%% of paired files, activating views" % parentName)
 
             writeStanza(ofh, indent, tdb)
             indent = 4
 
             if parentHasViews:
                 # we have composites with paired files? -> write the track stanzas for the two views
-                groupMeta = parseMeta(subDir)
+                groupMeta = parseMeta(subDirs)
                 tdbViewPeaks = {
                         "track" : parentName+"ViewPeaks",
                         "shortLabel" : parentName+" Peaks",
                         "parent" : parentName,
                         "view" : "PK",
                         "visibility" : "dense",
                         "type" : "bigBed",
                         "scoreFilter" : "off",
                         "viewUi" : "on"
                         }
                 metaOverride(tdbViewPeaks, parentMeta)
                 writeStanza(ofh, indent, tdbViewPeaks)
 
                 tdbViewSig = {
                         "track" : parentName+"ViewSignal",
                         "shortLabel" : parentName+" Signal",
                         "parent" : parentName,
                         "view" : "SIG",
                         "visibility" : "dense",
                         "type" : "bigWig",
                         "viewUi" : "on"
                         }
                 metaOverride(tdbViewSig, parentMeta)
                 writeStanza(ofh, indent, tdbViewSig)
         else:
             # no composites
-            groupMeta = parseMeta(subDir)
+            groupMeta = parseMeta(subDirs)
 
         typeDict = reorderTracks(typeDict, groupMeta)
 
         for trackBase, typeFnames in typeDict.items():
             for fileType, absFnames in typeFnames.items():
                 assert(len(absFnames)==1) # for now, not sure what to do when we get multiple basenames of the same file type
                 absFname = absFnames[0]
                 fileBase = basename(absFname)
                 relFname = relpath(absFname, tdbDir)
 
                 labelSuff = ""
                 if parentHasViews:
                     if fileType=="bigWig":
                         labelSuff = " Signal"
                     elif fileType=="bigBed":
@@ -630,40 +657,122 @@
 
                     if fileType=="bigBed":
                         tdb["parent"] = parentName+"ViewPeaks"+" "+onOff
                         tdb["subGroups"] = "view=PK"
                     else:
                         tdb["parent"] = parentName+"ViewSignal"+" "+onOff
                         tdb["subGroups"] = "view=SIG"
 
                 metaOverride(tdb, groupMeta)
 
                 if trackName in groupMeta and "visibility" in groupMeta[trackName]:
                     del tdb["visibility"]
 
                 writeStanza(ofh, indent, tdb)
 
+def httpReqOld(url, asBytes=False, asJson=False, params=None, method="get", max_redirects=10):
+    " get data with HTTP and return it. Does not use the requests library. "
+    # Parse URL
+    import http.client
+    import urllib.parse
+    parsed_url = urllib.parse.urlparse(url)
+    connection = http.client.HTTPSConnection(parsed_url.netloc) if parsed_url.scheme == "https" else http.client.HTTPConnection(parsed_url.netloc)
+
+    if params is not None and len(params)!=0:
+        query_string = urllib.parse.urlencode(params or {})
+        path = parsed_url.path or "/"
+        if method.lower() == "get" and query_string:
+            path += "?" + query_string
+    else:
+        path = url
+    
+    # Set headers and body
+    headers = {}
+    body = None
+    if method.lower() == "post":
+        headers["Content-Type"] = "application/x-www-form-urlencoded"
+        body = query_string
+
+    logging.debug(f"HTTP {method.upper()} {url}")
+
+    try:
+        # Send the request
+        connection.request(method.upper(), path, body=body, headers=headers)
+        response = connection.getresponse()
+
+        # Handle redirects
+        if 300 <= response.status < 400 and response.getheader("Location"):
+            if max_redirects <= 0:
+                raise RuntimeError("Too many redirects")
+            
+            redirect_url = urllib.parse.urljoin(url, response.getheader("Location"))
+            logging.debug(f"Redirecting to {redirect_url}, with parameters {params}")
+            return httpReqOld(redirect_url, asBytes, asJson, params, method, max_redirects - 1)
+
+        # Process the response
+        content = response.read()
+        if asBytes:
+            return content
+        elif asJson:
+            return json.loads(content.decode("utf-8"))
+        else:
+            return content.decode("utf-8")
+    except Exception as e:
+        logging.error(f"Error fetching the URL: {e}")
+        raise RuntimeError(f"Error fetching the URL: {e}")
+    finally:
+        connection.close()
+
+def httpReq(url, asBytes=False, asJson=False, params=None):
+    " get URL with http get request, return content "
+    if not requestsLoaded:
+        return httpReqOld(url, asBytes, asJson, params)
+
+    if doVerbose:
+        import http.client
+        http.client.HTTPConnection.debuglevel=1
+        logging.basicConfig()
+        logging.getLogger().setLevel(logging.DEBUG)
+        req_log = logging.getLogger('requests.packages.urllib3')
+        req_log.setLevel(logging.DEBUG)
+        req_log.propagate = True
+
+    logging.debug("HTTP GET %s" % url)
+    try:
+        resp = requests.get(url, params=params, allow_redirects=True)
+    except requests.RequestException as e:
+        errAbort(f"Error fetching the URL: {e}")
+
+    if asBytes:
+        logging.debug("HTTP GET with raw bytes")
+        return resp.content
+    elif asJson:
+        logging.debug("HTTP GET with JSON")
+        return resp.json()
+    else:
+        logging.debug("HTTP GET with text")
+        return resp.text
+
 def importJbrowse(baseUrl, db, outDir):
     " import an IGV trackList.json hierarchy "
-    import requests
     outFn = join(outDir, "hub.txt")
     ofh = open(outFn, "w")
     writeHubGenome(ofh, db, {})
 
     trackListUrl = baseUrl+"/data/trackList.json"
     logging.info("Loading %s" % trackListUrl)
-    trackList = requests.get(trackListUrl).json()
+    trackList = httpReq(trackListUrl, asJson=True)
     tdbs = []
     for tl in trackList["tracks"]:
         if "type" in tl and tl["type"]=="SequenceTrack":
             logging.info("Genome is: "+tl["label"])
             continue
         if tl["storeClass"]=="JBrowse/Store/SeqFeature/NCList":
             logging.info("NCList found: ",tl)
             continue
         tdb = {}
         tdb["track"] = tl["label"]
         tdb["shortLabel"] = tl["key"]
         if "data_file" in tl:
             tdb["bigDataUrl"] = baseUrl+"/data/"+tl["data_file"]
         else:
             tdb["bigDataUrl"] = baseUrl+"/data/"+tl["urlTemplate"]
@@ -820,74 +929,96 @@
         if "hub" in stanza or "genome" in stanza:
             continue
         allFields.update(stanza.keys())
         stanzas.append(stanza)
 
     if "track" in allFields:
         allFields.remove("track")
     if "shortLabel" in allFields:
         allFields.remove("shortLabel")
 
     hasLongLabel = False
     if "longLabel" in allFields:
         allFields.remove("longLabel")
         hasLongLabel = True
 
+    hasType = False
+    if "type" in allFields:
+        allFields.remove("type")
+        hasType = True
+
+    appendFields = []
+    if "bigDataUrl" in allFields:
+        allFields.remove("bigDataUrl")
+        appendFields.append("bigDataUrl")
+
     sortedFields = sorted(list(allFields))
     # make sure that track shortLabel and longLabel come first and always there, handy for manual edits
+    if hasType:
+        sortedFields.insert(0, "type")
     if hasLongLabel:
         sortedFields.insert(0, "longLabel")
     sortedFields.insert(0, "shortLabel")
     sortedFields.insert(0, "track")
 
+    # make sure some fields are always at the end
+    for af in appendFields:
+        sortedFields.append(af)
+
     ofh = sys.stdout
     ofh.write("#")
     ofh.write("\t".join(sortedFields))
     ofh.write("\n")
 
     for s in stanzas:
         row = []
         for fieldName in sortedFields:
             row.append(s.get(fieldName, ""))
         ofh.write("\t".join(row))
         ofh.write("\n")
 
 def guessFieldDesc(fieldNames):
     "given a list of field names, try to guess to which fields in bed12 they correspond "
     logging.info("No field description specified, guessing fields from TSV headers: %s" % fieldNames)
     fieldDesc = {}
     skipFields = set()
 
     for fieldIdx, fieldName in enumerate(fieldNames):
         caseField = fieldName.lower()
         if caseField in ["chrom", "chromosome"]:
             name = "chrom"
-        elif caseField.endswith(" id") or caseField.endswith("accession"):
+        elif caseField.endswith(" id") or caseField.endswith("accession") or caseField.endswith("alternate"):
             name = "name"
         elif caseField in ["start", "chromstart", "position"]:
             name = "start"
         elif caseField in ["Reference"]:
             name = "refAllele"
         elif caseField in ["strand"]:
             name = "strand"
         elif caseField in ["score"]:
             name = "score"
         else:
             continue
         fieldDesc[name] = fieldIdx
         skipFields.add(fieldIdx)
-    logging.info("TSV <-> BED correspondance: %s" % fieldDesc)
+
+    #logging.info("TSV <-> BED correspondance: %s" % fieldDesc)
+    logging.info("TSV <-> BED correspondance:")
+    for bedName, fieldIdx in fieldDesc.items():
+        tsvName = fieldNames[fieldIdx]
+        logging.info("TSV field %s -> BED field %s" % (tsvName, bedName))
+
     return fieldDesc, skipFields
 
 def parseFieldDesc(fieldDescStr, fieldNames):
     " given a string chrom=1,start=2,end=3,... return dict {chrom:1,...} "
     if not fieldDescStr:
         return guessFieldDesc(fieldNames)
 
     fieldDesc, skipFields = guessFieldDesc(fieldNames)
 
     if fieldDescStr:
         for part in fieldDescStr.split(","):
             fieldName, fieldIdx = part.split("=")
             fieldIdx = int(fieldIdx)
             fieldDesc[fieldName] = fieldIdx
             skipFields.add(fieldIdx)
@@ -940,73 +1071,84 @@
 
         bedRow.append(val)
 
     # now handle all other fields
     for fieldIdx, val in enumerate(row):
         if not fieldIdx in skipFields:
             bedRow.append( row[fieldIdx] )
 
     return bedRow
 
 def fetchChromSizes(db, outputFileName):
     " find on local disk or download a <db>.sizes text file "
     # Construct the URL based on the database name
     url = f"https://hgdownload.cse.ucsc.edu/goldenPath/{db}/database/chromInfo.txt.gz"
     # Send a request to download the file
-    response = requests.get(url, stream=True)
-    # Check if the request was successful
-    if response.status_code == 200:
+    chromSizesData = httpReq(url, asBytes=True)
     # Open the output gzip file for writing
     with gzip.open(outputFileName, 'wt') as outFile:
         # Open the response content as a gzip file in text mode
-            with gzip.GzipFile(fileobj=response.raw, mode='r') as inFile:
+        with gzip.GzipFile(fileobj=chromSizesData, mode='r') as inFile:
             # Read the content using csv reader to handle tab-separated values
             reader = csv.reader(inFile, delimiter='\t')
             writer = csv.writer(outFile, delimiter='\t', lineterminator='\n')
             # Iterate through each row, and retain only the first two fields
             for row in reader:
                 writer.writerow(row[:2])  # Write only the first two fields
-    else:
-        raise Exception(f"Failed to download file from {url}, status code: {response.status_code}")
     logging.info("Downloaded %s to %s" % (db, outputFileName))
 
+def getLocalDataPath(fname):
+    " return local filename in directory for hubtool files "
+    localDataDir = os.path.expanduser("~/.local/hubtools")
+    if not isdir(localDataDir):
+        logging.info("Creating directory "+localDataDir)
+        os.makedirs(localDataDir)
+    fname = join(localDataDir, fname)
+    return fname
+
+def getAsFname(fileType):
+    " download an .as file into the local dadta directory "
+    fname = getLocalDataPath(fileType+".as")
+    urls = {
+            "bigNarrowPeak" : "https://genome.ucsc.edu/goldenpath/help/examples/bigNarrowPeak.as"
+            }
+    if not isfile(fname):
+        url = urls[fileType]
+        downloadUrl(url, fname)
+    return fname
+
 def getChromSizesFname(db):
-    " return fname of chrom sizes, download into ~/.local/ucscData/ if not found "
+    " return fname of chrom sizes, download into ~/.local/hubtools/ if not found "
     fname = "/hive/data/genomes/%s/chrom.sizes" % db
     if isfile(fname):
         return fname
 
-    dataDir = "~/.local/hubtools"
-    fname = join(dataDir, "%s.sizes" % db)
-    fname = os.path.expanduser(fname)
+    fname = getLocalDataPath("%s.sizes" % db)
     if not isfile(fname):
         makedirs(dataDir)
         fetchChromSizes(db, fname)
     return fname
 
 def convTsv(db, tsvFname, outBedFname, outAsFname, outBbFname):
     " convert tsv files in inDir to outDir, assume that they all have one column for chrom, start and end. Try to guess these or fail. "
-    #tsvFname, outBedFname, outAsFname = args
-
     # join and output merged bed
     bigCols = set() # col names of columns with > 255 chars
 
-    #bedFh = open(bedFname, "w")
     unsortedBedFh = tempfile.NamedTemporaryFile(suffix=".bed", dir=dirname(outBedFname), mode="wt")
     fieldNames = None
-    #isOneBased = options.oneBased
-    isOneBased = True
+
+    isOneBased = True # useful in the future maybe, name=0,start=1z,...
     bedFieldsDesc = None # in the future, the user may want to input a string like name=0,start=1, but switch this off for now
 
     for line in open(tsvFname):
         row = line.rstrip("\r\n").split("\t")
         if fieldNames is None:
             fieldNames = row
             fieldDesc, notExtraFields = parseFieldDesc(bedFieldsDesc, fieldNames)
             continue
 
         # note fields with data > 255 chars.
         for colName, colData in zip(fieldNames, row):
             if len(colData)>255:
                 bigCols.add(colName)
 
         bedRow = makeBedRow(row, fieldDesc, notExtraFields, isOneBased)
@@ -1095,125 +1237,798 @@
 
     return result
 
 def readTrackLines(fnames):
     " read the first line and convert to a dict of all fnames. "
     logging.debug("Reading track lines from %s" % fnames)
     ret = {}
     for fn in fnames:
         line1 = open(fn).readline().rstrip("\n")
         notTrack = line1.replace("track ", "", 1)
         tdb = parseTrackLine(notTrack)
         ret[fn] = tdb
     return ret
 
 def stripFirstLine(inputFilename, outputFilename):
-    " written by chatGpt: copies all lines to output, except the first line "
+    " chatGpt: copies all lines to output, except the first line "
     with open(inputFilename, 'r') as infile, open(outputFilename, 'w') as outfile:
         # Skip the first line
         next(infile)
         # Write the rest of the lines to the output file
         for line in infile:
             outfile.write(line)
 
-def bedToBigBed(inFname, db, outFname):
+def bedToBigBed(inFname, db, outFname, asFname=None, bedType=None):
     " convert bed to bigbed file, handles chrom.sizes download "
     chromSizesFname = getChromSizesFname(db)
-    cmd = ["bedToBigBed", inFname, chromSizesFname, outFname]
-    logging.debug("Running %s" % cmd)
+    cmd = ["bedToBigBed", inFname, chromSizesFname, outFname, "-tab"]
+    if asFname:
+        cmd.append("-as="+asFname)
+    if bedType:
+        cmd.append("-type="+bedType)
+    logging.debug("Running %s" % " ".join(cmd))
     logging.info(f'Converting {inFname} to {outFname}. (chromSizes: {chromSizesFname}')
     subprocess.check_call(cmd)
 
-def convCtDb(db, inDir, outDir):
+def downloadUrlOld(url, local_file_name):
+    """ Downloads the content of the given URL and saves it to a local file. Does not use the requests library. """
+    if os.path.isfile(local_file_name):
+        logging.info("Not downloading %s, %s already exists" % (url, local_file_name))
+        return
+
+    # Parse the URL
+    import http.client
+    import urllib.parse
+    parsed_url = urllib.parse.urlparse(url)
+    connection = http.client.HTTPSConnection(parsed_url.netloc) if parsed_url.scheme == "https" else http.client.HTTPConnection(parsed_url.netloc)
+    path = parsed_url.path or "/"
+    if parsed_url.query:
+        path += "?" + parsed_url.query
+
+    try:
+        # Send a GET request
+        connection.request("GET", path)
+        response = connection.getresponse()
+
+        # Check for HTTP errors
+        if response.status >= 400:
+            logging.info(f"Unable to download {url}: HTTP error {response.status}")
+            return
+
+        # Write the content to the specified local file in chunks
+        tmpFname = local_file_name + ".tmp"
+        with open(tmpFname, 'wb') as file:
+            while True:
+                chunk = response.read(65536)  # Read in chunks
+                if not chunk:
+                    break
+                file.write(chunk)
+
+        logging.info(f"Downloaded '{url}' to '{local_file_name}'.")
+        os.rename(tmpFname, local_file_name)
+    except Exception as e:
+        logging.error(f"An error occurred while downloading the URL: {e}")
+        raise
+    finally:
+        connection.close()
+
+def downloadUrl(url, local_file_name):
+    """ Downloads the content of the given URL and saves it to a local file.  """
+    if isfile(local_file_name):
+        logging.info("Not downloading %s, %s already exists" % (url, local_file_name))
+        return
+
+    import requests
+    try:
+        # Send a GET request to the URL
+        response = requests.get(url, stream=True)
+        response.raise_for_status()  # Raise an exception for HTTP errors
+        # Write the content to the specified local file
+        tmpFname = local_file_name+".tmp"
+        with open(tmpFname, 'wb') as file:
+            for chunk in response.iter_content(chunk_size=65536):  # Download in chunks
+                file.write(chunk)
+        logging.info(f"Downloaded '{url}' to '{local_file_name}'.")
+        os.rename(tmpFname, local_file_name)
+
+    except requests.exceptions.HTTPError as e:
+        logging.info(f"Unable to download %s: HTTP error {e}")
+
+    except requests.exceptions.RequestException as e:
+        logging.error(f"An error occurred while downloading the URL: {e}")
+        raise
+
+def downloadUrlsParallel(url_filename_list, max_threads=12):
+    """ given a list of [url, localFname], download the files with 12 parallel threads """
+    logging.info("Downloading %s files with %d parallel threads" % (len(url_filename_list), max_threads))
+    executor = concurrent.futures.ThreadPoolExecutor(max_threads)
+    func = downloadUrl
+    if not requestsLoaded:
+        func = downloadUrlOld
+    futures = [executor.submit(func, url, local_filename) for url, local_filename in url_filename_list]
+
+    # wait for all futures to complete (this will handle exceptions)
+    for future in concurrent.futures.as_completed(futures):
+        try:
+            future.result()  # Block until this particular future is done
+        except Exception as e:
+            logging.error(f"Error in thread: {e}")
+
+def makeLegalTrackName(s):
+    " remove characters that are not allowed for track names "
+    s = s.replace(" ", "_")
+    # the only problem of this is that you can run into duplicated track names, e.g. "MyTrack!!" and "MyTrack!" are both "MyTrack"
+    return re.sub('[^A-Za-z_0-9]+', '', s)
+
+def mustBeLegalTrackName(s):
+    " error abort if s is not a legal track name "
+    if makeLegalTrackName(s)!=s:
+        errAbort("The name '%s' is not a legal name for a track. Only alphanumeric characters and underscore are allowed." % s)
+
+def narrowPeakToBigNarrowPeak(textFname, ofh):
+    " convert old narrow peak text format to .bed format for bigNarrowPeak "
+    #ofh = open(tmpFname, "w")
+    for line in open(textFname):
+        row = line.rstrip("\r\n").split("\t")
+        # chr1    9356548 9356648 .       0       .       182     5.0945  -1  50
+        chrom, chromStart, chromEnd, name, score, strand, signal, pVal, qVal, peak = row
+        score = int(score)
+        if score > 1000:
+            score = 1000
+        outRow = (chrom, chromStart, chromEnd, name, str(score), strand, signal, pVal, qVal, peak)
+        ofh.write("\t".join(outRow))
+        ofh.write("\n")
+    ofh.flush()
+
+def broadPeakToBed(textFname, ofh):
+    " convert old broad peak text format to .bed format for bigBed "
+    #ofh = open(tmpFname, "w")
+    for line in open(textFname):
+        row = line.rstrip("\r\n").split("\t")
+        chrom, chromStart, chromEnd, name, score, strand, signal, pVal, qVal = row[:9]
+        score = int(score)
+        if score > 1000:
+            score = 1000
+        outRow = (chrom, chromStart, chromEnd, name, str(score), strand, signal, pVal, qVal)
+        ofh.write("\t".join(outRow))
+        ofh.write("\n")
+    ofh.flush()
+
+def convertTextToBin(db, textFname, tdb, outDir):
+    " convert a text file to a binary file, e.g. bed to bigBed given input file and trackDb dictionary. Updates tdb dictionary with type "
+    outBase = join(outDir, tdb["track"])
+
+    trackType = "bed"
+    if "type" in tdb:
+        trackType = tdb["type"]
+
+    logging.info("Converting %s of type %s to binary" % (textFname, trackType))
+    outFname = outBase+".bb"
+
+    if trackType.startswith("bed"):
+        bedToBigBed(textFname, db, outFname)
+        tdb["type"] = "bigBed"
+    elif trackType=="narrowPeak":
+        asFname = getAsFname("bigNarrowPeak")
+        tmpFh = makeTempFile(dir=outDir, suffix=".bed")
+        narrowPeakToBigNarrowPeak(textFname, tmpFh)
+        bedToBigBed(tmpFh.name, db, outFname, asFname=asFname, bedType="bed6+4")
+        tdb["type"] = "bigBed 6+"
+        tdb["spectrum"] = "on"
+    elif trackType=="broadPeak":
+        asFname = getAsFname("bigBroadPeak")
+        tmpFh = makeTempFile(dir=outDir, suffix=".bed")
+        broadPeakToBed(textFname, tmpFh)
+        bedToBigBed(tmpFh.name, db, outFname, asFname=asFname, bedType="bed6+4")
+        tdb["type"] = "bigNarrowPeak"
+        tdb["spectrum"] = "on"
+
+    else:
+        errAbort("No support yet for track type '%s'. Please contact us." % trackType)
+
+    tdb["bigDataUrl"] = basename(outFname)
+    return tdb
+
+def convCtDb(hubInDir, db, inDir, outDir, doDownload):
     " convert one db part of a track archive to an output directory "
+    meta = parseMeta([hubInDir])
+
     findGlob = join(inDir, "*.ct")
     inFnames = glob.glob(findGlob)
     if len(inFnames)==0:
         logging.info("No *.ct files found in %s" % findGlob)
-        return
+        return []
 
     tdbData = readTrackLines(inFnames)
 
     makedirs(outDir)
     hubTxtFname = join(outDir, "hub.txt")
     ofh = open(hubTxtFname, "wt")
-
-    meta = parseMeta(inDir)
     writeHubGenome(ofh, db, meta)
 
+    getUrlsFnames = []
+    doneFnames = set()
     tdbIdx = 0
     for fname, tdb in tdbData.items():
+        tdb["shortLabel"] = tdb["name"]
+
         tdbIdx += 1
         # custom track names can include spaces, spec characters, etc. Strip all those
-        # append a number to make sure that the result is unique
+        # append a number to make sure that the result is unique and a legal track name
         track = tdb["name"]
-        track = re.sub('[^A-Za-z0-9]+', '', track)+"_"+str(tdbIdx)
+        track = makeLegalTrackName(track)+"_"+str(tdbIdx)
         tdb["track"] = track
-
-        tdb["shortLabel"] = tdb["name"]
         del tdb["name"]
+
         tdb["longLabel"] = tdb["description"]
         del tdb["description"]
 
         if "bigDataUrl" not in tdb:
-            bedFname = join(outDir, tdb["track"]+".bed")
-            bbFname = join(outDir, tdb["track"]+".bb")
-            stripFirstLine(fname, bedFname)
-            bedToBigBed(bedFname, db, bbFname)
-            os.remove(bedFname)
-            tdb["bigDataUrl"] = tdb["track"]+".bb"
-            tdb["type"] = "bigBed"
+            textFname = join(outDir, tdb["track"]+".txt")
+            stripFirstLine(fname, textFname)
+
+            tdb = convertTextToBin(db, textFname, tdb, outDir)
+
+            os.remove(textFname)
+        else:
+            url = tdb["bigDataUrl"]
+            if doDownload:
+                uniqueFname = basename(url)
+                if uniqueFname in doneFnames:
+                    # File name is not unique:
+                    # we need to make the file name unique in a way that does not touch the suffix structure: prefix with hash
+                    base_url = url.rsplit('/', 1)[0] # part before the last slash = directory
+                    #shortHash = base64.urlsafe_b64encode(hashlib.sha1(base_url.encode()).digest())[:10].decode("ascii")
+                    shortHash = hashlib.sha1(base_url.encode()).hexdigest()[:8]
+                    uniqueFname = shortHash+"_"+uniqueFname
+
+                assert(uniqueFname not in doneFnames) # eight hex digits should be enough for everyone
+                doneFnames.add(uniqueFname)
+
+                outFname = join(outDir, uniqueFname)
+                getUrlsFnames.append((url, outFname))
+                tdb["bigDataUrl"] = uniqueFname
+            else:
+                logging.debug("Not downloading %s, option to download was not set" % url)
 
         writeStanza(ofh, 0, tdb)
 
     logging.info("Wrote %s" % hubTxtFname)
     ofh.close()
 
-def convArchDir(inDir, outDir):
+    return getUrlsFnames
+
+def convArchDir(hubInfoDir, inDir, outDir, doDownload):
     " convert a directory created from the .tar.gz file downloaded via our track archive feature "
+    logging.info("Converting track archive in %s to a new track hub in %s" % (inDir, outDir))
     dbContent = os.listdir(inDir)
     dbDirs = []
     for db in dbContent:
         subDir = join(inDir, db)
         if isdir(subDir):
             dbDirs.append((db, subDir))
 
     if len(dbDirs)==0:
-        errAbort("No directories found under %s. Extract the tarfile and point this program at the 'archive' directory." % inDir)
+        errAbort("No directories found under %s. Is this really a UCSC track backup archive .tar.gz file?" % inDir)
 
+    allBigDataUrls = []
     for db, inSubDir in dbDirs:
         logging.debug("Processing %s, db=%s" % (inSubDir, db))
         outSubDir = join(outDir, db)
-        convCtDb(db, inSubDir, outSubDir)
+        dbUrls = convCtDb(hubInfoDir, db, inSubDir, outSubDir, doDownload)
+        allBigDataUrls.extend(dbUrls)
+
+    downloadUrlsParallel( allBigDataUrls )
 
 def hgsidFromUrl(url):
-    " return hgsid given a URL. written by chatGPT "
-    # Parse the URL
+    " return the part after hgsid= from a URL "
     parsed_url = urllib.parse.urlparse(url)
     server_name = f"{parsed_url.scheme}://{parsed_url.netloc}"
     query_params = urllib.parse.parse_qs(parsed_url.query)
-    hgsid = query_params.get('hgsid')
+    hgsid = query_params.get('hgsid')[0]
+    return server_name, hgsid  # Extracting the first value from the list
+
+def hgsidFromPage(url):
+    " return hgsid given a URL. Uses HTTP fetch and extracts hgsid from html page. "
+    # Parse the URL
+    # short session URLs first go through one redirect
+    logging.info("Getting hgsid from page %s" % url)
+    pageText = httpReq(url)
+
+    hgsid = None
+    for l in pageText.splitlines():
+        # <INPUT TYPE=HIDDEN NAME='hgsid' VALUE='347013469_676goC3Wia55Duav9QqA0zLNKL5Q'>
+        if l.startswith("<INPUT TYPE=HIDDEN NAME='hgsid' VALUE='"):
+            logging.debug("HTML line with hgsid is %s" % repr(l))
+            hgsid = l.split("NAME='hgsid' VALUE='")[1].split("'")[0]
+            logging.debug("HGSID is %s" % repr(hgsid))
+            break
+    if hgsid is None:
+        errAbort("Could not find hgsid on page %s" % url)
+
+    parsed_url = urllib.parse.urlparse(url)
+    server_name = f"{parsed_url.scheme}://{parsed_url.netloc}"
+    #query_params = urllib.parse.parse_qs(parsed_url.query)
+    #hgsid = query_params.get('hgsid')
     # Return the hgsid value if it exists
-    if hgsid:
-        return server_name, hgsid[0]  # Extracting the first value from the list
-    errAbort("hgsid parameter not found in "+url)
+    return server_name, hgsid  # Extracting the first value from the list
+
+def downloadCartLines(url):
+    """
+    Downloads the cart from the specified URL and extracts the main content,
+    removing the header and footer. returns a list of lines
+    XX Would be nice if we had a JSON API Endpoint for this, rather than parsing broken HTML.
+    """
+    # Download the content
+    data = httpReq(url)
+    if not "<PRE>" in data:
+        errAbort("No <PRE> found in cartDump, internal error")
+
+    data = data.split("<PRE>")[1].split("</TT>")[0] ## argh, HTML error in our output
+    return data.splitlines()
+
+def getCart(cartDumpUrl):
+    " return the cart as a dict key->val given a cart dump URL.  "
+    cartLines = downloadCartLines(cartDumpUrl)
+    cart = {}
+    for l in cartLines:
+        key, val = l.split(" ", maxsplit=1)
+        cart[key] = val
+    return cart
+
+def ctUrlToTdb(ctUrl):
+    " grab a ctfile from a url, and parse it, return a list of tdb dicts, changing the format to hub statements "
+    logging.info("Downloading %s" % ctUrl)
+    ctText = httpReq(ctUrl)
+    # track^Iname='SC21_WIGW2_3_NA_occX'^Idescription='nucleoatac.occ - occupancy based on fragment size distribution - using modified code!'^Ivisibility='2'^Ipriority='0.001'^IaltColor='127,127,127'^IautoScale='OFF'^IbigDataUrl='http://bartzabel.ls.manchester.ac.uk/Ian/adP0erXwWr/SC_20240716/SC21_WIGW2_3_nucleoatac.occ.bw'^Idb='hg38'^IinputType='bigWig'^ImaxHeightPixels='10:32:128'^IorigTrackLine='track type=bigWig name="SC21_WIGW2_3_NA_occX" description="nucleoatac.occ - occupancy based on fragment size distribution - using modified code!" bigDataUrl=http://bartzabel.ls.manchester.ac.uk/Ian/adP0erXwWr/SC_20240716/SC21_WIGW2_3_nucleoatac.occ.bw visibility=full maxHeightPixels=10:32:128 viewLimits=0:2 autoScale=OFF'^ItdbType='bigWig'^Itype='bigWig'^IviewLimits='0:2'$
+    tdbs = []
+    for line in ctText.splitlines():
+        if not line.startswith("track"):
+            errAbort("Unexpected custom track line in file %s" % ctUrl)
+
+        parts = line.rstrip("\n").split("\t")[1:]
+        tdb = {}
+        for p in parts:
+            if p.startswith("origTrackLine"):
+                continue
+            key, val = p.split("=", maxsplit=1)
+            val = val.lstrip("'").rstrip("'")
+
+            if key=="description":
+                key = "longLabel"
+            if key=="name":
+                key = "track"
 
-def convCtUrl(url, outDir):
-    " given an hgTracks URL with an hgsid on it, get all custom track lines and create a hub file for it. Does not download custom tracks. "
+            if key in ["inputType", "tdbType"]:
+                continue
+
+            tdb[key] = val
+            if key=="track":
+                tdb["shortLabel"] = val
+
+        if "dbTableName" in tdb or "bigDataUrl" not in tdb:
+            logging.warn("Cannot import custom tracks without bigDataUrls, skipping %s" % tdb["track"])
+        else:
+            tdbs.append(tdb)
+
+    return tdbs
+
+def downloadTrackArchive(serverUrl, hgsid, ofh):
+    " download the track archive for a given hgsid and save it under tgzFname "
+
+    # https://hgwdev-max.gi.ucsc.edu/cgi-bin/hgSession?hgsid=425203018_AsoR0syMagMP0brh6a2Y2F7R2RNi&hgS_makeDownload_=Submit
+    logging.info("Getting track archive from server %s, hgsid %s" % (serverUrl, hgsid))
+    cgiUrl = serverUrl+f"/cgi-bin/hgSession"
+    params = {"hgsid":hgsid, "hgS_makeDownload_":"Submit"}
+
+    page = httpReq(cgiUrl, params=params)
+    statusToken = re.search(r"backgroundStatus=([^&]*)", page).group(1)
+    statusToken = unquote(statusToken)
+    logging.debug("Status token is %s" % statusToken)
+
+    # get the download token from the status page
+    keepGoing = True
+    tryCount = 0
+    while keepGoing:
+        waitSecs = 3
+        logging.info("Waiting for %d seconds for archive" % waitSecs)
+        time.sleep(waitSecs)
+
+        params = {"hgsid":hgsid, "backgroundStatus":statusToken }
+        page = httpReq(cgiUrl, params=params)
+        matchObj = re.search(r"hgS_doDownload_([^']*)", page)
+        if matchObj is None:
+            logging.info("No hgsid found, waiting some more...")
+            tryCount += 1
+            if tryCount > 10:
+                errAbort("Cannot find hgsid even after long wait. Giving up.")
+        else:
+            downloadToken = unquote(matchObj.group(1))
+            keepGoing = False
+
+    params = { "hgsid" : hgsid, "hgS_doDownload_"+downloadToken : "1", "hgS_saveLocalBackupFileName": "test" }
+    logging.info("Downloading track archive and saving to %s" % ofh.name)
+    binData = httpReq(cgiUrl, params=params, asBytes=True)
+
+    ofh.write(binData)
+    ofh.flush()
+
+def makeTempFile(suffix=None, dir=None, mode="w", prefix=None):
+    " make a temporary file. Do not delete in debug mode. always delete in normal mode, at the latest when program exits. "
+    if debugMode:
+        tmpFn = tempfile.mkstemp(suffix=suffix, dir=dir, prefix=prefix)[1] # does not remove file
+        fh = open(tmpFn, mode)
+    else:
+        fh = tempfile.NamedTemporaryFile(prefix=prefix, suffix=suffix, dir=dir, mode=mode) # removes file on destruction of fh variable
+    return fh
+
+def convCtUrlOrFile(url, inDir, outDir, doDownload):
+    """ given an hgTracks URL with an hgsid or a session link or .tar.gz local track archive tarball, get all
+    custom track lines and create a hub file for it. Try to convert BED custom tracks to bigBed.
+    Download bigDataUrls to outDir.
+    """
+
+    downDir = join(outDir, "archive.tmp")
+    makedirs(downDir)
+
+    if url.startswith("http"):
+        if "hgsid=" in url:
             serverUrl, hgsid = hgsidFromUrl(url)
-    cartDumpUrl = serverUrl+"/cgi-bin/cartDump?hgsid="+hgsid
+        else:
+            serverUrl, hgsid = hgsidFromPage(url)
+
+        tgzFh = makeTempFile(dir=downDir, suffix=".tar.gz", mode="wb")
+        downloadTrackArchive(serverUrl, hgsid, tgzFh)
+        tgzFname = tgzFh.name
+    else:
+        tgzFh = None
+        tgzFname = url
+
+    logging.info("Extracting %s to %s" % (tgzFname, downDir))
+    with tarfile.open(tgzFname, 'r:gz') as tar:
+        tar.extractall(path=downDir)
+
+    convArchDir(inDir, downDir, outDir, doDownload)
+
+    if not debugMode:
+        if tgzFh:
+            tgzFh.close() # = deletes temp file
+        logging.info("Removing %s" % downDir)
+        shutil.rmtree(downDir)
+
+def stanzaKey(stanza):
+    " return key of stanza, so track name or .hub or .genome "
+    if "track" in stanza:
+        return stanza["track"][2]
+    else:
+        for tdbType in ["hub", "genome"]:
+            if tdbType in stanza:
+                return "."+tdbType
+
+    errAbort("Got hub.txt file with a stanza that has neither a 'track', nor a 'hub', nor a 'genome' key: %s" % repr(stanza))
+
+def stanzaNew(pairs):
+    " given key,val pairs, make a new stanza "
+
+def stanzaMatchesRe(tdb, tags, pat):
+    " try to match pat (a compiled regex) against values of all tags listed in 'tags'. Never match the special stanzas .hub and .genome . "
+    for tag in tags:
+        if tag in tdb:
+            comments, indent, value = tdb[tag]
+            if pat.search(value):
+                return True
+    return False
+
+def stanzaMatchesTrack(tdb, searchName):
+    " True if track name is same as searchName. False for non-track stanzas."
+    tag = "track"
+    if not tag in tdb:
+        return False
+    comments, indent, value = tdb[tag]
+    if value==searchName:
+        return True
+    return False
+
+def stanzaEdit(tdb, indent=None, newTags=None, delTags=None):
+    " modify a stanza. newTags is a list of [key, val]. delTags is a list of keys. "
+    newTdb = OrderedDict()
+    for key, valTuple in tdb.items():
+        comments, lineIndent, val = valTuple
+        if indent is not None:
+            lineIndent = indent
+
+        if delTags is None or key not in delTags:
+            newTdb[key] = (comments, lineIndent, val)
+
+    if newTags is not None:
+        for key, val in newTags:
+            newTdb[key] = ([], lineIndent, val)
+
+    return newTdb
+
+def stanzaGetVal(tdb, tag, default=None):
+    if tag in tdb:
+        return tdb[tag][2]
+    else:
+        return default
+
+def tdbCommentsFromPairs(pairs, indent=0):
+    " make a new tdbComments data structure from a list of (key, val) pairs "
+    tdbs = OrderedDict()
+    newTdb = OrderedDict()
+    trackName = None
+    for key, val in pairs:
+        newTdb[key] = ([], indent, val)
+        if key=="track":
+            trackName = val
+    assert(trackName is not None) # can only make track stanzas for now
+    tdbs[trackName] = newTdb
+    return tdbs
+
+def tdbCommentsParse(fname):
+    """ like iterRaStanzas(), parse a hub.txt file, returns an ordered dict of trackName -> stanza BUT retains all comments
+        Stanza is an OrderedDict of key -> (beforeCommentLines, indent, value)
+        This system retains all comments of the input file and allows to restore the entire file identically
+        Exceptions:
+         - DOS/MAC newline characters become Unix NLs
+         - multiple spaces between key and val become a single space
+        Hub and genome stanzas are saved like tracks with the special tracknames ".hub" and ".genome"
+
+        XX This should probably become a class TdbComments with methods on it.
+    """
+    bakFname = fname+".bak"
+    shutil.copy(fname, bakFname)
+
+    logging.debug(f"Parsing {fname}")
+    tdbs = OrderedDict()
+    headerDone = False
+    headerLines = []
+
+    stanza = OrderedDict()
+    comments = []
+
+    for line in open(fname):
+        line = line.rstrip("\r\n")
+        if len(line)==0:
+            # empty line = new stanza
+            if len(stanza)!=0:
+                tdbKey = stanzaKey(stanza)
+                tdbs[tdbKey] = stanza
+                stanza = OrderedDict()
+                comments = []
+            # preserve empty lines at the file beginning or within comments
+            else:
+                comments.append("")
+
+        # a normal line can either be a comment or a "key<space>value" lines
+        else:
+            if line.lstrip(" ").startswith("#"):
+                comments.append(line)
+                continue
+            else:
+                nonWhite = line.lstrip()
+                indent = len(line) - len(nonWhite)
+                key, val = nonWhite.split(" ", 1) # preserve trailing whitespace
+                if key in stanza:
+                    logging.warn(f"TrackDb .ra format error: duplicated key {key} in stanza {stanza}")
+                stanza[key] = (comments, indent, val)
+                comments = []
+
+    # handle last stanza, most files don't end with a newline
+    if len(stanza)!=0:
+        tdbKey = stanzaKey(stanza)
+        tdbs[tdbKey] = stanza
+
+    tdbCount = len(tdbs)
+    logging.info(f"Read {fname}, {tdbCount} stanzas")
+    return tdbs
+
+def tdbCommentsWrite(tdbs, fname):
+    " write back the structure returned by tdbCommentsParse into fname "
+    ofh = open(fname, "wt")
+    for tdb in tdbs.values():
+        for key, (comments, indent, val) in tdb.items():
+            if len(comments)!=0:
+                ofh.write("\n".join(comments))
+                ofh.write("\n")
+            ofh.write("".join(indent*[" "]))
+            ofh.write("%s %s\n" % (key, val))
+
+        ofh.write("\n")
+    ofh.close()
+
+    tdbCount = len(tdbs)
+    logging.info(f"Wrote {fname}, {tdbCount} stanzas")
+
+def tdbCommentsAppendStanza(tdbs, tdb, indent=0):
+    " add a single OrderedDict() as a new track stanza to the end of the data structure from tdbCommentsParse "
+    tdbKey = stanzaKey(tdb)
+    tdbs[tdbKey] = tdb
+    return tdbs
+
+def tdbCommentsAppendOneList(tdbs, pairs, indent=0):
+    " add a single list of pairs as a new track stanza to the end of the data structure from tdbCommentsParse "
+    newTdb = OrderedDict()
+    trackName = None
+    for key, val in tdb:
+        newTdb[key] = ([], indent, val)
+        if key=="track":
+            trackName = val
+    assert(trackName is not None)
+    tdbs[trackName] = newTdb
+    return tdbs
+
+def tdbCommentsEdit(tdbs, indent=None, newTags=None, delTags=None):
+    " indent stanzas of a tdbCommentsParse data structure or add some keyVals "
+    newTdbs = OrderedDict()
+    for trackName, tdb in tdbs.items():
+        newTdb = stanzaEdit(tdb, indent=indent, newTags=newTags, delTags=delTags)
+        newTdbs[trackName] = newTdb
+
+    return newTdbs
+
+def tdbCommentsAppendAll(tdbs1, tdbs2):
+    " apend the second tdbCommentsParse data structure to the first "
+    for key, val in tdbs2.items():
+        tdbs1[key] = val
+    return tdbs1
 
+def tdbCommentsInsertAfter(tdbs, parentName, insertTdbs):
+    " search in tdbs for a track parentName, insert newTdbs, and return the result "
+    newTdbs = OrderedDict()
+    for name, tdb in tdbs.items():
+        tdbCommentsAppendStanza(newTdbs, tdb)
+        if stanzaMatchesTrack(tdb, parentName):
+            tdbCommentsAppendAll(newTdbs, insertTdbs)
+    return newTdbs
+
+def addView(hubFname, contType, contName, contLabel):
+    " add a view under a container "
+    logging.debug("Adding to %s: view of type %s with name %s and label %s" % (hubFname, contType, contName, contLabel))
+
+    if "/" not in contName:
+        errAbort("For views, the parent has to be specified in the name with slash, e.g. myComposite/myView")
+    if contName.count("/")!=1:
+        errAbort("View name must contain one single slash, but not more.")
+
+    parentName, viewTrackSuffix = contName.split("/")
+
+    tdbs = tdbCommentsParse(hubFname)
+
+    if parentName not in tdbs:
+        errAbort("Parent container track %s does not exist in %s" % (parentName, hubFname))
+
+    viewTrackName = parentName+"_view_"+viewTrackSuffix
+
+    viewName  = makeLegalTrackName(contLabel)
+
+    # just overwrite the old one
+    #if viewTrackName in tdbs:
+        #errAbort("Track with name %s already exists." % viewTrackName)
+
+    viewPairs = [
+        ( "track" , viewTrackName),
+        ( "shortLabel" , contLabel),
+        ( "parent" , parentName),
+        ( "view" , viewName),
+        ( "visibility" , "dense"),
+        ( "type" , contType),
+        ( "scoreFilter" , "off"),
+        ( "viewUi" , "on")
+    ]
+
+    insertTdbs = tdbCommentsFromPairs(viewPairs, indent=4)
+
+    newTdbs = tdbCommentsInsertAfter(tdbs, parentName, insertTdbs)
+
+    parentTdb = newTdbs[parentName]
+    subGroupVal = stanzaGetVal(parentTdb, "subGroup1")
+    stanzaAddVal(parentTdb, "subGroup1", "view Views PK=Peaks SIG=Signals")
+
+    tdbCommentsWrite(newTdbs, hubFname)
+    logging.info("New view added, name is %s" % viewTrackName)
+
+def addContainer(hubFname, cont, contType, contName, contLabel):
+    " add a new container track and save hub.txt "
+    logging.debug("Adding to %s: container of type %s with name %s and label %s" % (hubFname, cont, contName, contLabel))
+    tdbs = tdbCommentsParse(hubFname)
+
+    mustBeLegalTrackName(contName)
+
+    if contName in tdbs:
+        errAbort(f"A track with the name {contName} already exists in {hubFname}.")
+
+    indent = 4
+    if cont=="composite":
+        #tdb["compositeTrack"] = ([], 0, "on")
+        contKey = "compositeTrack"
+    elif cont=="superTrack":
+        #tdb["superTrack"] = ([], 0, "on")
+        contKey = "superTrack"
+    else:
+        errAbort("container track type must be either composite or superTrack or view, not %s" % repr(contType))
+
+    containerDef =  (
+        ('track', contName),
+        ('shortLabel', contLabel),
+        ('longLabel', contLabel),
+        ('visibility', "dense"),
+        ('type', contType),
+        ('autoScale', 'group'),
+        (contKey, 'on')
+    )
+
+    contTdbs = tdbCommentsFromPairs(containerDef)
+    newTdbs = tdbCommentsAppendAll(tdbs, contTdbs)
+
+    tdbCommentsWrite(newTdbs, hubFname)
+
+def nest(hubFname, parentName, trackPat):
+    " put all tracks matching trackPat under container contName and save hub.txt "
+    tdbs = tdbCommentsParse(hubFname)
+
+    pat = re.compile(trackPat)
+
+    if not parentName in tdbs:
+        errAbort("container track %s is not part of hub %s. Try the 'add' command to add it." % (parentName, hubFname))
+
+    isView = False
+    if "_view_" in parentName:
+        isView = True
+
+    matchTdbs = OrderedDict()
+    oldTdbs = OrderedDict()
+    for name, tdb in tdbs.items():
+        if name not in [".hub", ".genome"] and stanzaMatchesRe(tdb, ["track", "shortLabel"], pat) and not name==parentName: # never match the parent
+            matchTdbs[name] = tdb
+        else:
+            oldTdbs[name] = tdb
+
+    logging.info("Found %d tracks matching %s" % (len(matchTdbs), trackPat))
+
+    if len(matchTdbs)==0:
+        errAbort("No matching tracks, aborting.")
+
+    indent = 4
+    if isView:
+        indent = 8
+
+    matchTdbs = tdbCommentsEdit(matchTdbs, indent=indent, newTags=[["parent", parentName]] )
+
+    # copy over all the old stanzas to newTdbs, and inject the matching stanzas after the parent
+    newTdbs = tdbCommentsInsertAfter(oldTdbs, parentName, matchTdbs)
+
+
+    tdbCommentsWrite(newTdbs, hubFname)
+
+def unnest(hubFname, trackPat):
+    " remove parent attribute from all tracks matching trackPat, unindent them and save hub.txt. Does not change track order. "
+    tdbs = tdbCommentsParse(hubFname)
+
+    pat = re.compile(trackPat)
+
+    newTdbs = OrderedDict()
+    modCount = 0
+    for name, tdb in tdbs.items():
+        if name not in [".hub", ".genome"] and stanzaMatchesRe(tdb, ["track", "shortLabel"], pat):
+            tdb = stanzaEdit(tdb, indent=0, delTags=["parent"] )
+            modCount += 1
+        newTdbs[name] = tdb
+
+    logging.info("Modified %d stanzas" % modCount)
+
+    tdbCommentsWrite(newTdbs, hubFname)
 
 def hubtools(args, options):
     """ find files under dirName and create a trackDb.txt for them"""
 
     cmd = args[0]
 
     inDir = "."
     if options.inDir:
         inDir = options.inDir
 
     outDir = "."
     if options.outDir:
         outDir = options.outDir
 
     if cmd=="up":
@@ -1223,54 +2038,97 @@
         hubName = args[1]
         uploadFiles(inDir, hubName)
         return
 
     tdbDir = inDir
     if options.outDir:
         tdbDir = options.outDir
 
     if cmd=="jbrowse":
         importJbrowse(args[1], args[2], tdbDir)
     elif cmd == "tab":
         raToTab(args[1])
     elif cmd == "make":
         db = args[1]
 
-        meta = parseMeta(inDir)
+        meta = parseMeta([inDir])
         dirFiles = readDirs(inDir, meta)
 
         hubFname = join(tdbDir, "hub.txt")
         logging.info("Writing %s" % hubFname)
         ofh = open(hubFname, "w")
 
-        meta = parseMeta(inDir)
         writeHubGenome(ofh, db, meta)
 
-        writeTdb(inDir, dirFiles, "top", tdbDir, ofh)
-        writeTdb(inDir, dirFiles, "comps", tdbDir, ofh)
+        makeTrackDbEntries(inDir, dirFiles, "top", tdbDir, ofh)
+        makeTrackDbEntries(inDir, dirFiles, "comps", tdbDir, ofh)
 
         ofh.close()
 
     elif cmd=="conv":
+        if len(args)==1:
+            errAbort("First argument for conv command must be the database, e.g. hg19 or hg38")
         db = args[1]
 
         convTsvDir(inDir, db, outDir)
 
-    elif cmd=="archive":
-        convArchDir(inDir, outDir)
-
     elif cmd=="ct":
         url = args[1]
-        convCtUrl(url, outDir)
+        doDownload = options.doDownload
+        convCtUrlOrFile(url, inDir, outDir, doDownload)
+
+    elif cmd=="add":
+        if len(args)<6:
+            errAbort("Please provide in this order: hub.txt-Filename, 'composite' or 'superTrack' or 'view', containerType, nameOfContainer, labelOfContainer")
+        hubFname = args[1]
+        cont = args[2]
+        contType = args[3]
+        contName = args[4]
+        contLabel = args[5]
+
+        if cont=="view":
+            addView(hubFname, contType, contName, contLabel)
+        else:
+            addContainer(hubFname, cont, contType, contName, contLabel)
+
+
+    elif cmd=="nest":
+        if len(args)<4:
+            errAbort("Please provide in this order: hub.txt-Filename, containerName, trackRegex")
+        hubFname = args[1]
+        contName = args[2]
+        trackPat = args[3]
+
+        nest(hubFname, contName, trackPat)
+
+    elif cmd=="unnest":
+        if len(args)<3:
+            errAbort("Please provide in this order: hub.txt-Filename, trackRegex")
+        hubFname = args[1]
+        trackPat = args[2]
+
+        unnest(hubFname, trackPat)
+
+    #elif cmd=="view":
+        #if len(args)<3:
+            #errAbort("Please provide in this order: hub.txt-Filename, containerName, trackRegex")
+        #hubFname = args[1]
+        ##trackPat = args[2]
+
+        #unnest(hubFname, trackPat)
 
+        #trackPat = args[4]
+        #splitPat = None
+        #if len(args)>5:
+            #splitPat = args[5]
     else:
-        logging.error("Unknown command: '%s'" % args[1])
+        logging.error("Unknown command: '%s'" % args[0])
 
 
 # ----------- main --------------
 def main():
     args, options = parseArgs()
 
     hubtools(args, options)
 
 
 main()