2c8a873e7286d797e88549d5a2825f0ea6fe3f04 chmalee Fri May 15 15:07:52 2026 -0700 HubSpace now allows tracks hubs to be defined via hub.txt+genomes.txt+trackDb.txt files, refs Baihe email and #37566 diff --git src/hg/js/hgMyData.js src/hg/js/hgMyData.js index 8bd3e531a63..d4d5cd17757 100644 --- src/hg/js/hgMyData.js +++ src/hg/js/hgMyData.js @@ -1,16 +1,16 @@ -/* jshint esnext: true */ +/* jshint esversion: 8 */ var debugCartJson = true; function prettyFileSize(num) { if (!num) {return "0B";} if (num < (1024 * 1024)) { return `${(num/1024).toFixed(1)}KB`; } else if (num < (1024 * 1024 * 1024)) { return `${((num/1024)/1024).toFixed(1)}MB`; } else { return `${(((num/1024)/1024)/1024).toFixed(1)}GB`; } } function cgiEncode(value) { // copy of cheapgi.c:cgiEncode except we are explicitly leaving '/' characters, and @@ -111,31 +111,43 @@ document.getElementById("spinner").remove(); let generateDiv = document.getElementById("generateDiv"); generateDiv.style.display = "block"; let revokeDiv = document.getElementById("revokeDiv"); revokeDiv.style.display = "none"; }; let cartData = {revokeApiKey: {}}; cart.setCgiAndUrl(fileListEndpoint); cart.send(cartData, handleSuccess); cart.flush(); } const fileNameRegex = /[0-9a-zA-Z._]+/g; // allowed characters in file names const fileNameFixRegex = /[^0-9a-zA-Z_]+/g; // '.' get replaced to underbars in trackHub.c. Also any files uploaded from hubtools that may have weird chars need to be escaped -const parentDirRegex = /[0-9a-zA-Z._]+/g; // allowed characters in hub names +const parentDirSegmentRegex = /^[0-9a-zA-Z._]+$/; // allowed characters in each hub-path segment + +function isValidParentDir(parentDir) { + // Slash-separated path of segments matching parentDirSegmentRegex; no '..'. + if (!parentDir) return false; + if (parentDir.startsWith("/") || parentDir.endsWith("/")) return false; + let segments = parentDir.split("/"); + for (let seg of segments) { + if (!seg || seg === "." || seg === "..") return false; + if (!parentDirSegmentRegex.test(seg)) return false; + } + return true; +} function getTusdEndpoint() { // this variable is set by hgHubConnect and comes from hg.conf value return tusdEndpoint; } let uppyOptions = { trigger: ".uploadButton", showProgressDetails: true, note: "The UCSC Genome Browser is not a HIPAA compliant data store. Do not upload patient information or other sensitive data files here, as anyone with the URL can view them.", meta: {"genome": null, "fileType": null}, restricted: {requiredMetaFields: ["genome"]}, closeModalOnClickOutside: true, closeAfterFinish: true, theme: 'auto', @@ -290,85 +302,97 @@ doneButtonHandler: function() { uppy.clear(); }, }; // make our Uppy instance: const uppy = new Uppy.Uppy({ debug: true, allowMultipleUploadBatches: false, onBeforeUpload: (files) => { // set all the fileTypes and genomes from their selects let doUpload = true; let thisQuota = 0; let filesToOverwrite = []; // collect files that will overwrite existing ones - // Only one 2bit per hub is supported. propagateAssemblyHubMeta picks - // the first 2bit found and overwrites every sibling's genome with it, - // which silently orphans the second 2bit (lands on disk and in the - // hubSpace table but is not referenced from hub.txt). + // Split hubs (genomesFile= with multiple genomes) can carry multiple 2bits. + let cachedDescriptor = hubCreate.getLastHubBatchDescriptor(); + let isSplitHub = (cachedDescriptor && cachedDescriptor.isSplit) || + Object.values(files).some( + f => f.meta && f.meta.batchSplitHub === "true"); + + let hubTxtInBatch = Object.values(files).some(looksLikeHubTxt); + if (cachedDescriptor && cachedDescriptor.errors.length && + (isSplitHub || hubTxtInBatch)) { + for (let e of cachedDescriptor.errors) { + uppy.info(e, "error", 8000); + } + return false; + } + + // Single-file hubs synthesize one hub.txt for one genome. let twoBitsInBatch = Object.values(files).filter(looksLikeTwoBit); - if (twoBitsInBatch.length > 1) { + if (!isSplitHub && twoBitsInBatch.length > 1) { let names = twoBitsInBatch.map(f => f.name).join(", "); uppy.info(`Error: only one 2bit file per hub is supported. ` + `Found: ${names}. Upload one 2bit at a time, or split ` + `them into separate hubs.`, "error", 6000); return false; } // If a 2bit is in the batch, propagate its genome/hubType to siblings. let batchTwoBit = twoBitsInBatch[0]; - if (batchTwoBit) { + if (batchTwoBit && !isSplitHub) { let asmGenome = batchTwoBit.meta.genome; if (!asmGenome) { uppy.info(`Error: Genome name is required for ` + `${batchTwoBit.name}. Open the file card and enter ` + `a name for your assembly.`, "error", 5000); return false; } for (let f of Object.values(files)) { f.meta.genome = asmGenome; f.meta.genomeLabel = asmGenome; f.meta.hubType = "assemblyHub"; // fileType may also be stale; recompute from filename if missing if (!f.meta.fileType) { f.meta.fileType = hubCreate.detectFileType(f.name); } } } // Tag every file so pre-finish knows a user hub.txt is coming in // the same batch and can skip synthesizing its own. let hasHubTxt = Object.values(files).some(looksLikeHubTxt); for (let f of Object.values(files)) { f.meta.batchHasHubTxt = hasHubTxt ? "true" : "false"; } for (let [key, file] of Object.entries(files)) { let fileNameMatch = file.meta.name.match(fileNameRegex); - let parentDirMatch = file.meta.parentDir.match(parentDirRegex); if (!fileNameMatch || fileNameMatch[0] !== file.meta.name) { uppy.info(`Error: File name has special characters, please rename file: ${file.meta.name} to only include alpha-numeric characters, period, or underscore.`, 'error', 5000); doUpload = false; continue; } - if (!parentDirMatch || parentDirMatch[0] !== file.meta.parentDir) { - uppy.info(`Error: Hub name has special characters, please rename hub: ${file.meta.parentDir} for file: ${file.meta.name} to only include alpha-numeric characters, period, or underscore.`, 'error', 5000); + if (!isValidParentDir(file.meta.parentDir)) { + uppy.info(`Error: Hub path has special characters, please rename hub: ${file.meta.parentDir} for file: ${file.meta.name} to a path of alpha-numeric / period / underscore segments separated by '/'.`, 'error', 5000); doUpload = false; continue; } - if (!file.meta.genome) { + // Hub-level files in a split-hub batch intentionally carry empty genome. + if (!file.meta.genome && file.meta.batchSplitHub !== "true") { uppy.info(`Error: No genome selected for file ${file.meta.name}!`, 'error', 5000); doUpload = false; continue; } if (!file.meta.fileType) { uppy.info(`Error: File type not supported, file: ${file.meta.name}!`, 'error', 5000); doUpload = false; continue; } // check if this hub already exists and the genome is different from what was // just selected, if so, make the user create a new hub if (file.meta.parentDir in hubCreate.uiState.filesHash && hubCreate.uiState.filesHash[file.meta.parentDir].genome !== file.meta.genome) { let existing = hubCreate.uiState.filesHash[file.meta.parentDir]; // If the existing hub is an assembly hub, adopt its genome // automatically rather than erroring - the UI hid the picker @@ -424,98 +448,172 @@ // Our hooks exit 0 + RejectUpload=true, so the response body is the raw // errAbort message. tus-js-client still wraps error.message with // "tus: unexpected response while ..., response text: <ours>, request // id: n/a" when the status code is 4xx/5xx. if (response && response.body) return String(response.body).trim(); let body = null; try { body = error && error.originalResponse && error.originalResponse.getBody(); } catch (e) { /* ignore */ } if (body) return String(body).trim(); let msg = (error && error.message) || "Upload failed"; // Scrape the tus wrapping off if present. let m = msg.match(/response text:\s*([\s\S]*?)(?:,\s*request id:|$)/); return m ? m[1].trim() : msg; } +function parentDirFromRelativePath(file) { + // Return the directory portion of an Uppy folder-drop file's relative path, or null. + let rel = (file.data && file.data.webkitRelativePath) || + file.relativePath || ""; + if (!rel || !rel.includes("/")) return null; + let segments = rel.split("/"); + segments.pop(); // drop the filename + return segments.join("/"); +} + function looksLikeTwoBit(f) { return (f.name || "").toLowerCase().endsWith(".2bit"); } function looksLikeHubTxt(f) { // Accept exact "hub.txt" or any "*.hub.txt" (e.g. "araTha1.hub.txt"). let n = (f.name || "").toLowerCase(); return n === "hub.txt" || n.endsWith(".hub.txt"); } +let hubBatchParsesInFlight = 0; +function setUploadButtonEnabled(enabled) { + // Pauses uploads while parseHubBatch is running so pre-finish sees stamped meta. + let btn = document.querySelector(".uppy-StatusBar-actionBtn--upload"); + if (!btn) return; + btn.disabled = !enabled; + btn.style.opacity = enabled ? "" : "0.5"; + btn.style.cursor = enabled ? "" : "wait"; + btn.title = enabled ? "" : "Parsing hub definition..."; +} + +function applySplitHubDescriptor(uppyInstance, descriptor) { + // Stamp per-file genome from the descriptor and flag the batch as split. + let hubFile = descriptor.hubFile; + let hubParentDir = hubFile && hubFile.meta && hubFile.meta.parentDir; + // Nested layouts (per-genome subdirs) carry their own parentDir already. + let isNestedLayout = uppyInstance.getFiles().some( + f => f.meta && f.meta.parentDir && f.meta.parentDir.includes("/")); + for (let f of uppyInstance.getFiles()) { + let assignedGenome = descriptor.fileGenome.get(f.id); + let meta = { + batchSplitHub: "true", + hubType: descriptor.isAssemblyHub ? "assemblyHub" : "trackHub", + }; + if (hubParentDir && !isNestedLayout) meta.parentDir = hubParentDir; + if (assignedGenome) { + meta.genome = assignedGenome; + meta.genomeLabel = assignedGenome; + meta.genomeLocked = true; + } else if (descriptor.fileGenome.has(f.id)) { + // Hub-level files (hub.txt, genomes.txt): empty db. + meta.genome = ""; + meta.genomeLabel = ""; + meta.genomeLocked = true; + } + uppyInstance.setFileMeta(f.id, meta); + } + let names = descriptor.genomes.map(g => g.name).join(", "); + if (names) { + uppyInstance.info(`Split hub detected. Genomes: ${names}`, "info", 4000); + } +} + function propagateAssemblyHubMeta(uppyInstance) { // When a batch contains a 2bit (and/or an assembly-hub hub.txt), mirror the // custom genome name onto every file sharing that parentDir and mark every // file hubType=assemblyHub. hub.txt wins over the 2bit's default. // // We detect the hub-defining files by filename rather than by meta.fileType, // because setFileMeta updates Uppy's state immutably - file objects captured // from getFiles() earlier in this event may still carry old meta. let files = uppyInstance.getFiles(); let twoBit = files.find(looksLikeTwoBit); let hubTxt = files.find(looksLikeHubTxt); - if (!twoBit && !hubTxt) return; + if (!twoBit && !hubTxt) { + hubCreate.clearLastHubBatchDescriptor(); + return; + } function applyGenomeToSiblings(genome, alsoLockHubDefiners) { // Set genome/hubType on every file in the batch. Non-hub-defining // files (i.e. the sibling tracks) are always locked to this genome so // the user can't drift them. The hub-defining files (2bit, hub.txt) // are locked only when alsoLockHubDefiners is true - used by the // hub.txt path to pin the 2bit's editable field too. if (!genome) return; // All files in this batch belong to one new hub, so they must share // one parentDir. Take it from the hub-defining file - its parentDir // came from getDefaultHubName(), while a track that was added first // may have been pointed at an existing assembly hub. let hubDefiner = hubTxt || twoBit; let syncParentDir = hubDefiner && hubDefiner.meta && hubDefiner.meta.parentDir; + // Folder drops carry their own multi-segment parentDir; don't overwrite. + let isNestedLayout = uppyInstance.getFiles().some( + f => f.meta && f.meta.parentDir && f.meta.parentDir.includes("/")); for (let f of uppyInstance.getFiles()) { let isHubDefining = looksLikeTwoBit(f) || looksLikeHubTxt(f); let meta = { genome: genome, genomeLabel: genome, hubType: "assemblyHub", genomeLocked: !isHubDefining || alsoLockHubDefiners, }; - if (syncParentDir) meta.parentDir = syncParentDir; + if (syncParentDir && !isNestedLayout) meta.parentDir = syncParentDir; uppyInstance.setFileMeta(f.id, meta); } } if (hubTxt) { - hubCreate.readFileAsText(hubTxt.data).then((text) => { - let parsed = hubCreate.parseHubTxt(text); - // hub.txt is authoritative: lock the genome field so the user - // can't edit it and drift the stored db away from what hub.txt - // says. The user can always edit the hub.txt itself if they - // want a different name. + hubBatchParsesInFlight++; + setUploadButtonEnabled(false); + hubCreate.parseHubBatch(uppyInstance.getFiles()).then((descriptor) => { + // Skip stale parses; only the latest-completed one applies. + if (descriptor !== hubCreate.getLastHubBatchDescriptor()) return; + for (let e of descriptor.errors) { + uppyInstance.info(e, "error", 8000); + } + for (let w of descriptor.warnings) { + uppyInstance.info(w, "warning", 6000); + } + if (descriptor.isSplit) { + applySplitHubDescriptor(uppyInstance, descriptor); + return; + } + // Single-file hub: hub.txt is authoritative for the one genome + // it declares. Lock all siblings to that genome. + let parsed = descriptor.hubMeta || {}; if (parsed.isAssemblyHub && parsed.genome) { applyGenomeToSiblings(parsed.genome, true); uppyInstance.info(`Using genome "${parsed.genome}" from hub.txt`, "info", 4000); } else if (parsed.genome && twoBit) { let twoBitGenome = twoBit.meta.genome || hubCreate.sanitizeGenomeName(twoBit.name); if (parsed.genome !== twoBitGenome) { applyGenomeToSiblings(parsed.genome, true); uppyInstance.info(`Using genome "${parsed.genome}" from hub.txt (overrides 2bit default)`, "warning", 5000); } } }).catch((err) => { console.warn("Could not read hub.txt for genome detection:", err); + }).finally(() => { + hubBatchParsesInFlight--; + if (hubBatchParsesInFlight === 0) setUploadButtonEnabled(true); }); return; } let asmGenome = twoBit.meta.genome || hubCreate.sanitizeGenomeName(twoBit.name); applyGenomeToSiblings(asmGenome, false); } // create a custom uppy plugin to batch change the type and db fields class BatchChangePlugin extends Uppy.BasePlugin { constructor(uppy, opts) { super(uppy, opts); this.id = "BatchChangePlugin"; this.type = "progressindicator"; this.opts = opts; @@ -550,38 +648,46 @@ } removeBatchSelectsFromDashboard() { let batchSelectDiv = document.getElementById("batch-selector-div"); if (batchSelectDiv) { batchSelectDiv.remove(); } } addBatchSelectsToDashboard() { if (!document.getElementById("batch-selector-div")) { // If the batch contains a 2bit, the UCSC genome picker makes no // sense - show the custom genome name read-only instead. Detect by // filename rather than meta.hubType because setFileMeta updates // Uppy state immutably and the meta may not be visible on file - // objects captured from getFiles() earlier in this event. - let assemblyHubGenome = null; + // objects captured from getFiles() earlier in this event. A split + // assembly hub can declare more than one 2bit (one per genome); + // join all of them. + let assemblyHubGenomes = []; for (let f of this.uppy.getFiles()) { if (looksLikeTwoBit(f)) { - assemblyHubGenome = f.meta.genome || hubCreate.sanitizeGenomeName(f.name); - break; + let g = f.meta.genome || hubCreate.sanitizeGenomeName(f.name); + if (g && !assemblyHubGenomes.includes(g)) { + assemblyHubGenomes.push(g); + } } } + let assemblyHubGenome = null; + if (assemblyHubGenomes.length) { + assemblyHubGenome = assemblyHubGenomes.join(", "); + } let batchSelectDiv = document.createElement("div"); batchSelectDiv.id = "batch-selector-div"; batchSelectDiv.style.display = "grid"; batchSelectDiv.style.width = "80%"; // the grid syntax is 2 columns, 3 rows batchSelectDiv.style.gridTemplateColumns = "max-content minmax(0, 200px) max-content 1fr min-content"; batchSelectDiv.style.gridTemplateRows = "repest(3, auto)"; batchSelectDiv.style.margin = "10px auto"; // centers this div batchSelectDiv.style.fontSize = "14px"; batchSelectDiv.style.gap = "8px"; if (window.matchMedia("(prefers-color-scheme: dark)").matches) { batchSelectDiv.style.color = "#eaeaea"; } @@ -602,31 +708,35 @@ if (assemblyHubGenome) { // Assembly hub: show the custom genome name as a locked text // field, no UCSC picker or search. let locked = document.createElement("input"); locked.type = "text"; locked.id = "batchAsmHubGenome"; locked.value = assemblyHubGenome; locked.disabled = true; locked.classList.add("uppy-u-reset", "uppy-c-textInput"); locked.style.gridArea = "2 / 2 / 2 / 2"; locked.style.margin = "2px"; batchDbLabel.for = "batchAsmHubGenome"; let note = document.createElement("div"); + if (assemblyHubGenomes.length > 1) { + note.textContent = "(assembly hub - genome per file is set by genomes.txt; this list shows all genomes in the hub)"; + } else { note.textContent = "(assembly hub - genome locked; shared by all files in this batch)"; + } note.style.gridArea = "2 / 3 / 2 / 5"; note.style.margin = "auto 0"; note.style.fontStyle = "italic"; batchSelectDiv.appendChild(batchSelectText); batchSelectDiv.appendChild(batchDbLabel); batchSelectDiv.appendChild(locked); batchSelectDiv.appendChild(note); } else { // Track hub: the usual UCSC picker + autocomplete. batchDbSelect = document.createElement("select"); this.createOptsForSelect(batchDbSelect, hubCreate.makeGenomeSelectOptions()); batchDbSelect.id = "batchDbSelect"; batchDbSelect.style.gridArea = "2 / 2 / 2 / 2"; batchDbSelect.style.margin = "2px"; @@ -641,129 +751,156 @@ batchDbGenomeSearchBar.classList.add("uppy-u-reset", "uppy-c-textInput"); batchDbGenomeSearchBar.type = "text"; batchDbGenomeSearchBar.id = "batchDbSearchBar"; batchDbGenomeSearchBar.style.gridArea = "2 / 4 / 2 / 4"; batchDbGenomeSearchButton = document.createElement("input"); batchDbGenomeSearchButton.type = "button"; batchDbGenomeSearchButton.value = "search"; batchDbGenomeSearchButton.id = "batchDbSearchBarButton"; batchDbGenomeSearchButton.style.gridArea = "2 / 5 / 2 / 5"; batchDbSelect.addEventListener("change", (ev) => { let files = this.uppy.getFiles(); let val = ev.target.value; let label = ev.target.selectedOptions[0].label; let hub = hubCreate.assemblyHubByGenome(val); + let newRoot = hub ? hub.fileName : hubCreate.uiState.hubNameDefault; for (let [key, file] of Object.entries(files)) { + // Keep the file's subdirectory under whatever root the + // batch genome change implies; only the root segment + // moves. + let oldParent = (file.meta && file.meta.parentDir) || ""; + let segments = oldParent.split("/"); + let newParent; + if (segments.length > 1) { + newParent = newRoot + "/" + segments.slice(1).join("/"); + } else { + newParent = newRoot; + } let meta = { genome: val, genomeLabel: label, hubType: hub ? "assemblyHub" : "trackHub", - parentDir: hub ? hub.fileName : hubCreate.uiState.hubNameDefault, + parentDir: newParent, }; this.uppy.setFileMeta(file.id, meta); } }); batchSelectDiv.appendChild(batchSelectText); batchSelectDiv.appendChild(batchDbLabel); batchSelectDiv.appendChild(batchDbSelect); batchSelectDiv.appendChild(batchDbSearchBarLabel); batchSelectDiv.appendChild(batchDbGenomeSearchBar); batchSelectDiv.appendChild(batchDbGenomeSearchButton); } // the batch change hub name (shown in both modes) let batchParentDirLabel = document.createElement("label"); batchParentDirLabel.textContent = "Hub Name"; batchParentDirLabel.for = "batchParentDir"; batchParentDirLabel.style.gridArea = "3 / 1 / 3 / 1"; let batchParentDirInput = document.createElement("input"); batchParentDirInput.id = "batchParentDir"; batchParentDirInput.value = hubCreate.getDefaultHubName(); batchParentDirInput.style.gridArea = "3 / 2 / 3 / 2"; batchParentDirInput.style.margin= "1px 1px auto"; batchParentDirInput.classList.add("uppy-u-reset", "uppy-c-textInput"); batchParentDirInput.addEventListener("change", (ev) => { let files = this.uppy.getFiles(); - let val = ev.target.value; + let newRoot = ev.target.value; for (let [key, file] of Object.entries(files)) { - this.uppy.setFileMeta(file.id, {parentDir: val}); + // Swap only the root segment; preserve any per-genome + // subdirectory the user supplied via a folder drop. + let oldParent = (file.meta && file.meta.parentDir) || ""; + let segments = oldParent.split("/"); + let newParent; + if (segments.length > 1) { + newParent = newRoot + "/" + segments.slice(1).join("/"); + } else { + newParent = newRoot; + } + this.uppy.setFileMeta(file.id, {parentDir: newParent}); } }); batchSelectDiv.appendChild(batchParentDirLabel); batchSelectDiv.appendChild(batchParentDirInput); // append the batch changes to the bottom of the file list, for some reason // I can't append to the actual Dashboard-files, it must be getting emptied // and re-rendered or something let uppyFilesDiv = document.querySelector(".uppy-Dashboard-progressindicators"); if (uppyFilesDiv) { uppyFilesDiv.insertBefore(batchSelectDiv, uppyFilesDiv.firstChild); } // autocomplete only applies in the track-hub path if (batchDbSelect && batchDbGenomeSearchBar && batchDbGenomeSearchButton) { let justInitted = initAutocompleteForInput(batchDbGenomeSearchBar.id, batchDbSelect); if (justInitted) { batchDbGenomeSearchButton.addEventListener("click", (e) => { let inp = document.getElementById(batchDbGenomeSearchBar.id).value; let selector = "[id='"+batchDbGenomeSearchBar.id+"']"; $(selector).autocompleteCat("search", inp); }); } } } } install() { this.uppy.on("file-added", (file) => { - // Only one 2bit per hub is supported. If this batch already has - // a 2bit, reject the new one before any meta-setting runs (which - // would propagate the wrong genome to it). onBeforeUpload also - // enforces this; the duplicate check here keeps the per-file card - // UI from drifting. - if (looksLikeTwoBit(file)) { + // Reject a duplicate 2bit only when there's no hub.txt in the batch: + // a folder drop or a manual pick of hub.txt + multi-genome + // genomes.txt + several 2bits is legitimate; we can't know that + // synchronously here, so defer to the pre-finish hook (which has + // the parseHubBatch result). + let droppedFromFolder = !!parentDirFromRelativePath(file); + let batchHasHubTxt = this.uppy.getFiles().some(looksLikeHubTxt); + if (looksLikeTwoBit(file) && !droppedFromFolder && !batchHasHubTxt) { let existingTwoBits = this.uppy.getFiles().filter( - f => f.id !== file.id && looksLikeTwoBit(f)); + f => f.id !== file.id && looksLikeTwoBit(f) && + !parentDirFromRelativePath(f)); if (existingTwoBits.length > 0) { this.uppy.removeFile(file.id); // Close the file card if it auto-opened for the first 2bit; // otherwise it covers the error banner. const dash = this.uppy.getPlugin("Dashboard"); if (dash) dash.toggleFileCard(false); // Long duration so the user has time to read it; the // StatusBar (setState.error) truncates to "Upload failed" // and hides the message behind a "?" icon. this.uppy.info( `Only one 2bit file per hub is allowed. ` + `"${existingTwoBits[0].name}" was added; ` + `"${file.name}" was not. To create a separate ` + `hub for "${file.name}", upload it on its own.`, 'error', 15000); return; } } - // add default meta data for genome and fileType + // Default meta; folder drops preserve their subdirectory. let ftype = hubCreate.detectFileType(file.name); + let dropPath = parentDirFromRelativePath(file); + let defaultParentDir = dropPath || hubCreate.getDefaultHubName(); let defaultMeta = { "genome": hubCreate.defaultDb(), "fileType": ftype, - "parentDir": hubCreate.getDefaultHubName(), + "parentDir": defaultParentDir, "hubType": "trackHub", }; if (ftype === "2bit") { // This file defines an assembly hub. Default the genome to the // sanitized filename stem; the user can edit it in the file card. defaultMeta.genome = hubCreate.sanitizeGenomeName(file.name); defaultMeta.genomeLabel = defaultMeta.genome; defaultMeta.hubType = "assemblyHub"; } this.uppy.setFileMeta(file.id, defaultMeta); // When drilled into an assembly hub, inherit and lock its genome. let drilledIntoAsmHub = false; if (hubCreate.uiState.currentHub && hubCreate.uiState.currentHub === defaultMeta.parentDir) { @@ -777,31 +914,34 @@ }); drilledIntoAsmHub = true; } } // Top-level with no drilled-in hub: if the user already has an // assembly hub, default this file to target it so they don't have // to re-enter the genome + hub name. The user can still switch via // the dropdown. Skip this when the file itself is hub-defining // (2bit or hub.txt) or when any file in the batch is - in that // case the batch is creating a *new* hub, not adding to an // existing one, so the defaults should not point at the old hub. let fileIsHubDefining = ftype === "2bit" || ftype === "hub.txt"; let batchHasHubDefining = this.uppy.getFiles().some(f => looksLikeTwoBit(f) || looksLikeHubTxt(f)); - if (!drilledIntoAsmHub && !fileIsHubDefining && !batchHasHubDefining) { + // dropPath means the user dragged a folder; in that case parentDir + // already encodes the user's intended hub root, and redirecting to + // an existing assembly hub would discard the folder layout. + if (!drilledIntoAsmHub && !fileIsHubDefining && !batchHasHubDefining && !dropPath) { let firstHub = hubCreate.firstAssemblyHub(); if (firstHub) { this.uppy.setFileMeta(file.id, { genome: firstHub.genome, genomeLabel: firstHub.genome, parentDir: firstHub.fileName, hubType: "assemblyHub", // NOT genomeLocked - user may want a different hub/genome }); } } // If a 2bit is in the batch, every sibling file in the same parentDir // adopts its genome and gets hubType=assemblyHub. Also handle hub.txt: // parse it client-side and, if it declares an assembly hub, mirror @@ -809,30 +949,46 @@ propagateAssemblyHubMeta(this.uppy); if (this.uppy.getFiles().length > 1) { this.addBatchSelectsToDashboard(); } else { // only open the file editor when there is one file const dash = uppy.getPlugin("Dashboard"); dash.toggleFileCard(true, file.id); } }); this.uppy.on("file-removed", (file) => { // remove the batch change selects if now <2 files present if (this.uppy.getFiles().length < 2) { this.removeBatchSelectsFromDashboard(); } + // If a hub-definition file leaves the batch, the cached split-hub + // descriptor is no longer valid. Clear the cache and the per-file + // stamps so pre-finish re-evaluates from scratch. + if (looksLikeHubTxt(file) || + (file.meta && file.meta.fileName === "genomes.txt")) { + hubCreate.clearLastHubBatchDescriptor(); + for (let f of this.uppy.getFiles()) { + if (f.meta && f.meta.batchSplitHub === "true") { + this.uppy.setFileMeta(f.id, { + batchSplitHub: undefined, + genomeLocked: false, + }); + } + } + propagateAssemblyHubMeta(this.uppy); + } }); this.uppy.on("dashboard:modal-open", () => { // check if there were already files chosen from before: if (this.uppy.getFiles().length > 2) { this.addBatchSelectsToDashboard(); } if (this.uppy.getFiles().length < 2) { this.removeBatchSelectsFromDashboard(); } }); this.uppy.on("dashboard:modal-closed", () => { if (this.uppy.getFiles().length < 2) { this.removeBatchSelectsFromDashboard(); } @@ -841,37 +997,35 @@ if (allFiles.length === completeFiles.length) { this.uppy.clear(); } }); this.uppy.on("dashboard:file-edit-start", (file) => { autocompletes[`${file.name}DbInput`] = false; }); this.uppy.on("dashboard:file-edit-complete", (file) => { // check the filename and hubname metadata and warn the user // to edit them if they are wrong. unfortunately I cannot // figure out how to force the file card to re-toggle // and jump back into the editor from here if (file) { let fileNameMatch = file.meta.name.match(fileNameRegex); - let parentDirMatch = file.meta.parentDir.match(parentDirRegex); - const dash = uppy.getPlugin("Dashboard"); if (!fileNameMatch || fileNameMatch[0] !== file.meta.name) { uppy.info(`Error: File name has special characters, please rename file: '${file.meta.name}' to only include alpha-numeric characters, period, or underscore.`, 'error', 5000); } - if (!parentDirMatch || parentDirMatch[0] !== file.meta.parentDir) { - uppy.info(`Error: Hub name has special characters, please rename hub: '${file.meta.parentDir}' to only include alpha-numeric characters, period, or underscore.`, 'error', 5000); + if (!isValidParentDir(file.meta.parentDir)) { + uppy.info(`Error: Hub path '${file.meta.parentDir}' must be alpha-numeric / period / underscore segments separated by '/'.`, 'error', 5000); } } }); } uninstall() { // not really used because we aren't ever uninstalling the uppy instance this.uppy.off("file-added"); } } var hubCreate = (function() { let uiState = { // our object for keeping track of the current UI and what to do userUrl: "", // the web accesible path where the uploads are stored for this user hubNameDefault: "", currentHub: "", // if the user has a hub dir open, set the name here and use it as the default @@ -934,48 +1088,283 @@ function assemblyHubByGenome(genome) { // Return the dir row of the user's assembly hub whose genome matches, // or null. If genome is falsy, return the first assembly hub found. for (let fullPath in uiState.filesHash) { let fd = uiState.filesHash[fullPath]; if (fd.fileType !== "dir" || fd.hubType !== "assemblyHub" || !fd.genome) continue; if (!genome || fd.genome === genome) return fd; } return null; } function firstAssemblyHub() { return assemblyHubByGenome(null); } function genomeIsAssemblyHub(genome) { return !!genome && !!assemblyHubByGenome(genome); } + function parseRaSettings(text) { + // Parse stanza-style .ra text. Blank lines separate stanzas; #-comments are skipped. + let stanzas = []; + let current = null; + if (!text) return stanzas; + for (let raw of text.split(/\r?\n/)) { + let line = raw.replace(/^\s+/, ""); + if (line === "") { + if (current) { stanzas.push(current); current = null; } + continue; + } + if (line.startsWith("#")) continue; + let sp = line.indexOf(" "); + let tab = line.indexOf("\t"); + let split = (sp === -1) ? tab : (tab === -1 ? sp : Math.min(sp, tab)); + if (split === -1) continue; + let key = line.substring(0, split); + let value = line.substring(split + 1).trim(); + if (!current) current = {}; + if (!(key in current)) current[key] = value; + } + if (current) stanzas.push(current); + return stanzas; + } + function parseHubTxt(text) { - // Very small hub.txt parser: extracts `genome` and `twoBitPath` lines. - // Returns {genome, twoBitPath, isAssemblyHub} (values may be null/false). - let ret = {genome: null, twoBitPath: null, isAssemblyHub: false}; + // Returns {genome, twoBitPath, isAssemblyHub, genomesFile, useOneFile}. + let ret = {genome: null, twoBitPath: null, isAssemblyHub: false, + genomesFile: null, useOneFile: false}; if (!text) return ret; - let lines = text.split(/\r?\n/); - for (let line of lines) { - let trimmed = line.replace(/^\s+/, ""); - if (trimmed.startsWith("genome ") && !ret.genome) { - ret.genome = trimmed.substring(7).trim(); - } else if (trimmed.startsWith("twoBitPath ")) { - ret.twoBitPath = trimmed.substring(11).trim(); + let stanzas = parseRaSettings(text); + let hub = stanzas[0] || {}; + if (hub.genome) ret.genome = hub.genome; + if (hub.twoBitPath) { + ret.twoBitPath = hub.twoBitPath; + ret.isAssemblyHub = true; + } + if (hub.genomesFile) ret.genomesFile = hub.genomesFile; + if (hub.useOneFile && hub.useOneFile.toLowerCase() === "on") { + ret.useOneFile = true; + } + // useOneFile hubs put `genome` in later stanzas. + if (!ret.genome) { + for (let s of stanzas) { + if (s.genome) { ret.genome = s.genome; break; } + } + } + if (!ret.twoBitPath) { + for (let s of stanzas) { + if (s.twoBitPath) { + ret.twoBitPath = s.twoBitPath; ret.isAssemblyHub = true; + break; + } } } return ret; } + function basename(p) { + if (!p) return ""; + let i = Math.max(p.lastIndexOf("/"), p.lastIndexOf("\\")); + return i === -1 ? p : p.substring(i + 1); + } + + function findFileInBatch(files, refPath) { + // Match a hub.txt/genomes.txt path against the batch, by relativePath or basename. + let target = refPath.replace(/^\.\//, ""); + let targetBase = basename(target); + let exactPathMatch = null; + let baseMatch = null; + for (let f of files) { + let rel = (f.meta && f.meta.relativePath) || + (f.data && f.data.webkitRelativePath) || + f.relativePath || ""; + if (rel && (rel === target || rel.endsWith("/" + target))) { + exactPathMatch = f; + break; + } + if ((f.meta && f.meta.name === targetBase) || f.name === targetBase) { + if (!baseMatch) baseMatch = f; + } + } + return exactPathMatch || baseMatch || null; + } + + function parseTrackDbForDataFiles(text) { + // Return all bigDataUrl-style references in a trackDb.txt. + let refs = []; + if (!text) return refs; + for (let raw of text.split(/\r?\n/)) { + let line = raw.replace(/^\s+/, ""); + for (let key of ["bigDataUrl", "bigDataIndex", "bamIndex", + "indexUrl", "searchTrix"]) { + if (line.startsWith(key + " ") || line.startsWith(key + "\t")) { + refs.push(line.substring(key.length).trim()); + break; + } + } + } + return refs; + } + + let parseBatchSeq = 0; + let latestCompletedParseSeq = 0; + async function parseHubBatch(files) { + // Walk the upload batch and build a hub descriptor: + // { + // isSplit: bool, // hub.txt uses genomesFile= + // isAssemblyHub: bool, + // hubFile: file, // hub.txt file in batch (or null) + // genomesFile: file, // genomes.txt file in batch (split only) + // genomes: [ + // { name, twoBitFile, trackDbFile, dataFiles: [file, ...] } + // ], + // fileGenome: Map<fileId, genomeName>, // hub.txt/genomes.txt absent + // errors: [string, ...], + // parseSeq: number, // monotonic id of this parse + // } + // Single-file layouts (useOneFile / no genomesFile) return isSplit=false. + let mySeq = ++parseBatchSeq; + let descriptor = { + isSplit: false, + isAssemblyHub: false, + hubFile: null, + hubMeta: null, + genomesFile: null, + genomes: [], + fileGenome: new Map(), + errors: [], // upload-blocking: missing referenced files + warnings: [], // surfaced but don't block: orphans, parse hiccups + parseSeq: mySeq, + }; + function cacheAndReturn() { + if (mySeq > latestCompletedParseSeq) { + latestCompletedParseSeq = mySeq; + lastHubBatchDescriptor = descriptor; + } + return descriptor; + } + let hubTxt = files.find(looksLikeHubTxt); + if (!hubTxt) return cacheAndReturn(); + descriptor.hubFile = hubTxt; + + let hubText; + try { + hubText = await readFileAsText(hubTxt.data); + } catch (e) { + descriptor.errors.push("Could not read hub.txt: " + e); + return cacheAndReturn(); + } + + let hubParsed = parseHubTxt(hubText); + descriptor.hubMeta = hubParsed; + descriptor.isAssemblyHub = hubParsed.isAssemblyHub; + if (!hubParsed.genomesFile || hubParsed.useOneFile) { + // Not a split hub - existing single-file flow handles it. + return cacheAndReturn(); + } + + let genomesFile = findFileInBatch(files, hubParsed.genomesFile); + if (!genomesFile) { + descriptor.errors.push( + `hub.txt references genomesFile=${hubParsed.genomesFile}, but ` + + `that file is not in the upload batch. Add it and try again.`); + return cacheAndReturn(); + } + descriptor.genomesFile = genomesFile; + + let genomesText; + try { + genomesText = await readFileAsText(genomesFile.data); + } catch (e) { + descriptor.errors.push("Could not read genomes.txt: " + e); + return cacheAndReturn(); + } + descriptor.isSplit = true; + + let genomeStanzas = parseRaSettings(genomesText); + for (let stanza of genomeStanzas) { + if (!stanza.genome) continue; + let entry = { name: stanza.genome, twoBitFile: null, + trackDbFile: null, dataFiles: [] }; + if (stanza.trackDb) { + entry.trackDbFile = findFileInBatch(files, stanza.trackDb); + if (!entry.trackDbFile) { + descriptor.errors.push( + `genomes.txt references trackDb=${stanza.trackDb} for ` + + `genome ${stanza.genome}, but that file is not in the ` + + `upload batch.`); + } + } + if (stanza.twoBitPath) { + entry.twoBitFile = findFileInBatch(files, stanza.twoBitPath); + if (!entry.twoBitFile) { + descriptor.errors.push( + `genomes.txt references twoBitPath=${stanza.twoBitPath} ` + + `for genome ${stanza.genome}, but that file is not in ` + + `the upload batch.`); + } + descriptor.isAssemblyHub = true; + } + descriptor.genomes.push(entry); + } + + for (let g of descriptor.genomes) { + if (!g.trackDbFile) continue; + let trackDbText; + try { + trackDbText = await readFileAsText(g.trackDbFile.data); + } catch (e) { + descriptor.errors.push( + `Could not read trackDb for ${g.name}: ${e}`); + continue; + } + let refs = parseTrackDbForDataFiles(trackDbText); + for (let ref of refs) { + let dataFile = findFileInBatch(files, ref); + if (dataFile) { + g.dataFiles.push(dataFile); + descriptor.fileGenome.set(dataFile.id, g.name); + } + // bigDataUrl targets are allowed to be missing - data files + // can arrive in later batches. + } + descriptor.fileGenome.set(g.trackDbFile.id, g.name); + if (g.twoBitFile) { + descriptor.fileGenome.set(g.twoBitFile.id, g.name); + } + } + + // Mark hub.txt and genomes.txt as hub-level (null genome). + descriptor.fileGenome.set(hubTxt.id, null); + descriptor.fileGenome.set(genomesFile.id, null); + + // Flag orphans as warnings: the upload still works (the file lands + // on disk and in hubSpace) but trackDb won't reference it until the + // user adds a track stanza. + for (let f of files) { + if (descriptor.fileGenome.has(f.id)) continue; + descriptor.warnings.push( + `File ${f.name} is in the batch but is not referenced by any ` + + `trackDb in the hub definition. It will be uploaded as an ` + + `orphan; add a track stanza if you want it to display.`); + } + + return cacheAndReturn(); + } + + let lastHubBatchDescriptor = null; + function getLastHubBatchDescriptor() { return lastHubBatchDescriptor; } + function clearLastHubBatchDescriptor() { lastHubBatchDescriptor = null; } + function readFileAsText(fileObj) { // Return a Promise resolving to the file contents as text. return new Promise((resolve, reject) => { let reader = new FileReader(); reader.onload = () => resolve(reader.result); reader.onerror = () => reject(reader.error); reader.readAsText(fileObj); }); } function detectFileType(fileName) { let fileLower = fileName.toLowerCase(); for (let fileType in extensionMap) { for (let ext of extensionMap[fileType]) { if (fileLower.endsWith(ext)) { @@ -1053,30 +1442,61 @@ autoChoice.label = "Auto-detect from extension"; autoChoice.value = "Auto-detect from extension"; autoChoice.selected = true; ret.push(autoChoice); let choices = ["bigBed", "bam", "vcf", "vcf (bgzip or gzip compressed)", "bigWig", "hic", "cram", "bigBarChart", "bigGenePred", "bigMaf", "bigInteract", "bigPsl", "bigChain"]; choices.forEach( (e) => { let choice = {}; choice.id = e; choice.label = e; choice.value = e; ret.push(choice); }); return ret; } + function findHubGenome(hubName) { + // Walk the hub subtree for the first non-empty genome. Split-hub + // root dirs and hub-level files carry "" and need this fallback. + let dir = uiState.filesHash[hubName]; + if (!dir) return null; + if (dir.genome) return dir.genome; + if (!dir.children) return null; + for (let c of dir.children) { + if (c.genome) return c.genome; + if (c.fileType === "dir") { + let nested = findHubGenome(c.fullPath); + if (nested) return nested; + } + } + return null; + } + + function isAssemblyHub(hubName) { + // Hub-root dir's hubType can be "trackHub" if a hub-level file + // uploaded first; walk the subtree for any assemblyHub or 2bit child. + let dir = uiState.filesHash[hubName]; + if (!dir) return false; + if (dir.hubType === "assemblyHub") return true; + if (!dir.children) return false; + for (let c of dir.children) { + if (c.hubType === "assemblyHub" || c.fileType === "2bit") return true; + if (c.fileType === "dir" && isAssemblyHub(c.fullPath)) return true; + } + return false; + } + function viewInGenomeBrowser(fname, ftype, genome, hubName, hubType) { // redirect to hgTracks with this track open in the hub if (typeof uiState.userUrl !== "undefined" && uiState.userUrl.length > 0) { if (ftype in extensionMap) { // TODO: tusd should return this location in it's response after // uploading a file and then we can look it up somehow, the cgi can // write the links directly into the html directly for prev uploaded files maybe? let hubUrl = uiState.userUrl + cgiEncode(hubTxtPathForHub(hubName)); // Assembly hubs use the user-defined genome name, which isn't a // UCSC db - hgTracks needs 'genome=' (resolves via the hub) // rather than 'db=' (looks up a native assembly). let dbParam = hubType === "assemblyHub" ? "genome" : "db"; let url = "../cgi-bin/hgTracks?hgsid=" + getHgsid() + "&" + dbParam + "=" + genome + "&hubUrl=" + encodeURIComponent(hubUrl) + "&" + trackHubFixName(fname) + "=pack"; window.location.assign(url); return false; @@ -1085,32 +1505,33 @@ } function trackHubFixName(trackName) { // replace everything but alphanumeric and underscore with underscore return encodeURIComponent(trackName.replaceAll(fileNameFixRegex, "_")); } function viewHubInGenomeBrowser(hubName) { // connect the whole hub in hgTracks, without pack'ing any specific track if (typeof uiState.userUrl === "undefined" || uiState.userUrl.length === 0) { return; } let dirRow = uiState.filesHash[hubName]; if (!dirRow) return; let hubUrl = uiState.userUrl + cgiEncode(hubTxtPathForHub(hubName)); - let dbParam = dirRow.hubType === "assemblyHub" ? "genome" : "db"; - let url = "../cgi-bin/hgTracks?hgsid=" + getHgsid() + "&" + dbParam + "=" + dirRow.genome + "&hubUrl=" + encodeURIComponent(hubUrl); + let dbParam = isAssemblyHub(hubName) ? "genome" : "db"; + let genome = dirRow.genome || findHubGenome(hubName) || ""; + let url = "../cgi-bin/hgTracks?hgsid=" + getHgsid() + "&" + dbParam + "=" + genome + "&hubUrl=" + encodeURIComponent(hubUrl); window.location.assign(url); } function showHubBanner(hubName) { let banner = document.getElementById("hubBanner"); let nameSpan = document.getElementById("hubBannerName"); if (!banner || !nameSpan) return; nameSpan.textContent = hubName; banner.style.display = ""; } function hideHubBanner() { let banner = document.getElementById("hubBanner"); if (banner) banner.style.display = "none"; } @@ -1118,36 +1539,43 @@ // helper object so we don't need to use an AbortController to update // the data this function is using let selectedData = {}; // track which items the user directly selected (vs children of selected directories) let directlySelected = {}; function viewAllInGenomeBrowser(ev) { // redirect to hgTracks with these tracks/hubs open let data = selectedData; if (typeof uiState.userUrl !== "undefined" && uiState.userUrl.length > 0) { let url = "../cgi-bin/hgTracks?hgsid=" + getHgsid(); let genome; // may be multiple genomes in list, just redirect to the first one // TODO: this should probably raise an alert to click through let hubsAdded = {}; _.forEach(data, (d) => { if (!genome) { + // Hub-level rows carry empty db; fall back via the subtree. genome = d.genome; - // Assembly hubs use 'genome=' (hub-resolved name), native - // UCSC assemblies use 'db='. - let dbParam = d.hubType === "assemblyHub" ? "genome" : "db"; + let hubRoot = (d.fileType === "dir") ? d.fullPath : d.parentDir; + if (!genome && hubRoot) { + genome = findHubGenome(hubRoot); + } + if (genome) { + let isAsm = (d.hubType === "assemblyHub") || + (hubRoot && isAssemblyHub(hubRoot)); + let dbParam = isAsm ? "genome" : "db"; url += "&" + dbParam + "=" + genome; } + } if (d.fileType === "hub.txt") { url += "&hubUrl=" + encodeURIComponent(uiState.userUrl + cgiEncode(d.fullPath)); } else if (d.fileType in extensionMap) { // TODO: tusd should return this location in it's response after // uploading a file and then we can look it up somehow, the cgi can // write the links directly into the html directly for prev uploaded files maybe? if (!(d.parentDir in hubsAdded)) { // NOTE: hubUrls get added regardless of whether they are on this assembly // or not, because multiple genomes may have been requested. If this user // switches to another genome we want this hub to be connected already // Resolve the actual hub.txt filename - user may have // uploaded "<prefix>.hub.txt" rather than literal hub.txt. let hubDir = d.parentDir.replace(/\/$/, ""); url += "&hubUrl=" + encodeURIComponent(uiState.userUrl + cgiEncode(hubTxtPathForHub(hubDir))); @@ -1409,33 +1837,37 @@ rowNode = row.node(); } oldRowData = row.data(); // put the data in the header: let rowClone = rowNode.cloneNode(true); // match the background color of the normal rows: rowClone.style.backgroundColor = "#fff9d2"; let thead = document.querySelector(".dt-scroll-headInner > table:nth-child(1) > thead:nth-child(1)"); // remove the checkbox because it doesn't do anything, and replace it // with a back arrow 'button' let btn = document.createElement("button"); btn.id = "backButton"; $(btn).button({icon: "ui-icon-triangle-1-w"}); btn.addEventListener("click", (e) => { let parentDir = dirData.parentDir; - let parentDirPath = dirData.fullPath.slice(0,-dirData.fullPath.length); + // Walk one level up by stripping the leaf segment. + let pathParts = dirData.fullPath.split("/"); + let parentDirPath = pathParts.slice(0, -1).join("/"); if (parentDirPath.length) { + // Mirror the click-down path: filter, then move header row. dataTableShowDir(table, parentDir, parentDirPath); + dataTableCustomOrder(table, {fullPath: parentDirPath}); } else { dataTableShowTopLevel(table); dataTableCustomOrder(table); dataTableEmptyBreadcrumb(table); } table.draw(); }); let tdBtn = document.createElement("td"); tdBtn.appendChild(btn); rowClone.replaceChild(tdBtn, rowClone.childNodes[0]); if (thead.childNodes.length === 1) { thead.appendChild(rowClone); } else { thead.replaceChild(rowClone, thead.lastChild); } @@ -1954,82 +2386,100 @@ uppy.info(cleanMsg, 'error', 30000); // Genome-name collision is fixable in place by editing the 2bit's // genome field, so reopen the file card. if (file && cleanMsg && cleanMsg.includes(hubGenomeCollisionErrFrag)) { const dash = uppy.getPlugin("Dashboard"); if (dash) dash.toggleFileCard(true, file.id); } }); uppy.on('upload-success', (file, response) => { const metadata = file.meta; const d = new Date(metadata.lastModified); const pad = (num) => String(num).padStart(2, '0'); const dFormatted = `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`; const now = new Date(Date.now()); const nowFormatted = `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())} ${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}`; - let newReqObj, hubTxtObj, parentDirObj; + let newReqObj, hubTxtObj; let hubType = metadata.hubType || "trackHub"; + // Multi-segment parentDir (split-hub uploads) needs per-segment rows. + let parentSegments = metadata.parentDir.split("/"); + let parentLeaf = parentSegments[parentSegments.length - 1]; newReqObj = { "fileName": cgiEncode(metadata.fileName), "fileSize": metadata.fileSize, "fileType": metadata.fileType, "genome": metadata.genome, - "parentDir": cgiEncode(metadata.parentDir), + "parentDir": cgiEncode(parentLeaf), "lastModified": dFormatted, "uploadTime": nowFormatted, "fullPath": cgiEncode(metadata.parentDir) + "/" + cgiEncode(metadata.fileName), "hubType": hubType, }; // from what I can tell, any response we would create in the pre-finish hook // is completely ignored for some reason, so we have to fake the other files // we would have created with this one file and add them to the table if they // weren't already there: // Only fabricate a hub.txt row when the backend actually synthesized // one. Skip if the user supplied their own *.hub.txt (either already // in filesHash from a prior upload, or coming in this same batch - // upload-success order is arbitrary so the hub.txt row may not be // in filesHash yet when a sibling's upload-success fires). let dirHash = uiState.filesHash[cgiEncode(metadata.parentDir)]; let hubTxtExists = !!(dirHash && dirHash.children && dirHash.children.some(c => c.fileType === "hub.txt")); let batchHasHubTxt = metadata.batchHasHubTxt === "true"; if (metadata.fileType !== "hub.txt" && !hubTxtExists && !batchHasHubTxt) { hubTxtObj = { "uploadTime": nowFormatted, "lastModified": dFormatted, "fileName": "hub.txt", "fileSize": 0, "fileType": "hub.txt", "genome": metadata.genome, - "parentDir": cgiEncode(metadata.parentDir), + "parentDir": cgiEncode(parentLeaf), "fullPath": cgiEncode(metadata.parentDir) + "/hub.txt", "hubType": hubType, }; } - parentDirObj = { + // One dir row per path segment; leaf-only db, matching makeParentDirRows(). + // For split hubs the hub-root dir stays empty regardless of layout. + let isSplitHub = metadata.batchSplitHub === "true"; + let dirRows = []; + for (let i = 0; i < parentSegments.length; i++) { + let dirFullPath = parentSegments.slice(0, i + 1) + .map(cgiEncode).join("/"); + let dirParent = i > 0 ? cgiEncode(parentSegments[i - 1]) : ""; + let isLeaf = (i === parentSegments.length - 1); + let dirDb; + if (isSplitHub) { + dirDb = (isLeaf && parentSegments.length > 1) ? metadata.genome : ""; + } else { + dirDb = isLeaf ? metadata.genome : ""; + } + dirRows.push({ "uploadTime": nowFormatted, "lastModified": dFormatted, - "fileName": cgiEncode(metadata.parentDir), + "fileName": cgiEncode(parentSegments[i]), "fileSize": 0, "fileType": "dir", - "genome": metadata.genome, - "parentDir": "", - "fullPath": cgiEncode(metadata.parentDir), + "genome": dirDb, + "parentDir": dirParent, + "fullPath": dirFullPath, "hubType": hubType, - }; - // package the three objects together as one "hub" and display it - let hub = [parentDirObj, newReqObj]; + }); + } + let hub = dirRows.concat([newReqObj]); if (hubTxtObj) { hub.push(hubTxtObj); } addNewUploadedHubToTable(hub); updateQuota(metadata.fileSize); }); uppy.on('complete', (result) => { history.replaceState(uiState, "", document.location.href); console.log("replace history with uiState"); }); inited = true; } function checkJsonData(jsonData, callerName) { // Return true if jsonData isn't empty and doesn't contain an error; @@ -2080,20 +2530,23 @@ } else { showExistingFiles([]); } } } return { init: init, uiState: uiState, defaultDb: defaultDb, makeGenomeSelectOptions: makeGenomeSelectOptions, getDefaultHubName: getDefaultHubName, detectFileType: detectFileType, sanitizeGenomeName: sanitizeGenomeName, readFileAsText: readFileAsText, parseHubTxt: parseHubTxt, + parseHubBatch: parseHubBatch, + getLastHubBatchDescriptor: getLastHubBatchDescriptor, + clearLastHubBatchDescriptor: clearLastHubBatchDescriptor, firstAssemblyHub: firstAssemblyHub, genomeIsAssemblyHub: genomeIsAssemblyHub, assemblyHubByGenome: assemblyHubByGenome, }; }());