373b5c4187422a9584814dcfa324028fd2e50f14 chmalee Tue Aug 4 15:00:30 2026 -0700 Have the pre-finish hook return the rows it wrote, and fix the table display bugs that the real row data now lets us resolve, refs #37999 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> diff --git src/hg/js/hgMyData.js src/hg/js/hgMyData.js index cd292b3775f..8e1f0709b9d 100644 --- src/hg/js/hgMyData.js +++ src/hg/js/hgMyData.js @@ -1,29 +1,42 @@ /* jshint esversion: 8 */ var debugCartJson = true; function prettyFileSize(num) { if (!num) {return "0B";} if (num < (1024 * 1024)) { return `${(num/1024).toFixed(1)}KB`; } else if (num < (1024 * 1024 * 1024)) { return `${((num/1024)/1024).toFixed(1)}MB`; } else { return `${(((num/1024)/1024)/1024).toFixed(1)}GB`; } } +function renderTimeCell(data, type) { + // DataTables renderer for the two time columns. The server sends seconds since + // the epoch, so the reader sees their own timezone rather than the server's, + // while ordering stays on the number + if (type !== "display") { + return data; + } + if (!data) { + return ""; + } + return new Date(data * 1000).toLocaleString(); +} + function cgiEncode(value) { // copy of cheapgi.c:cgiEncode except we are explicitly leaving '/' characters, and // space becomes '+': let splitVal = value.split('/'); splitVal.forEach((ele, ix) => { if (ele == " ") { splitVal[ix] = '+'; } else { splitVal[ix] = encodeURIComponent(ele); } }); return splitVal.join('/'); } function cgiDecode(value) { @@ -1805,31 +1818,32 @@ }); } }); updateSelectedFileDiv(data, selectedRow.data().fileType === "dir"); } function createOneCrumb(table, dirName, dirFullPath, doAddEvent) { // make a new span that can be clicked to nav through the table let newSpan = document.createElement("span"); newSpan.id = dirName; newSpan.textContent = decodeURIComponent(dirName); newSpan.classList.add("breadcrumb"); if (doAddEvent) { newSpan.addEventListener("click", function(e) { dataTableShowDir(table, dirName, dirFullPath); - dataTableCustomOrder(table, {"fullPath": dirFullPath}); + // the whole row, so the back button this builds knows the parentDir + dataTableCustomOrder(table, uiState.filesHash[dirFullPath] || {"fullPath": dirFullPath}); table.draw(); }); } else { // can't click the final crumb so don't underline it newSpan.style.textDecoration = "unset"; } return newSpan; } function dataTableEmptyBreadcrumb(table) { let currBreadcrumb = document.getElementById("breadcrumb"); currBreadcrumb.replaceChildren(currBreadcrumb.firstChild); } function dataTableCreateBreadcrumb(table, dirName, dirFullPath) { @@ -1869,32 +1883,31 @@ table.rows({selected: true}).deselect(); table.search.fixed("showRoot", function(searchStr, rowData, rowIx) { return !rowData.parentDir; }); uiState.currentHub = ""; uiState.currentHubPath = ""; hideHubBanner(); updateSelectedFileDiv(null); } function dataTableShowDir(table, dirName, dirFullPath) { // show the directory and all immediate children of the directory clearSearch(table); // deselect any selected rows like Finder et al when moving into/upto a directory table.rows({selected: true}).deselect(); - // Callers must call table.draw() after this so filter + order changes - // from showDir/customOrder render in a single redraw. + // Callers must call table.draw() after this to render the new filter. table.search.fixed("oneHub", function(searchStr, rowData, rowIx) { // calculate the fullPath of this rows parentDir in case the dirName passed // to this function has the same name as a parentDir further up in the // listing. For example, consider a test/test/tmp.txt layout, where "test" // is the parentDir of tmp.txt and the test subdirectory let parentDirFull = rowData.fullPath.split("/").slice(0,-1).join("/"); if (rowData.parentDir === dirName && parentDirFull === dirFullPath) { return true; } else if (rowData.fullPath === dirFullPath) { // also return the directory itself return true; } else { return false; } }); @@ -1916,66 +1929,75 @@ // in uploadTime order if (!dirData) { // make sure the old row can show up again in the table let thead = document.querySelector(".dt-scroll-headInner > table:nth-child(1) > thead:nth-child(1)"); if (thead.childNodes.length > 1) { let old = thead.removeChild(thead.lastChild); if (oldRowData) { table.row.add(oldRowData); oldRowData = null; } } table.order([{name: "uploadTime", dir: "desc"}]); } else { // move the dirName row into the header, then the other files can // sort normally - let row = table.row((idx,data) => data.fullPath === dirData.fullPath); - let rowNode = row.node(); if (oldRowData) { // restore the previous row, which will be not displayed by the search anyways: table.row.add(oldRowData); oldRowData = null; } + // A row only has a node while it is on the page being displayed, and + // deferRender means the rows of other pages have none at all. Order by + // fullPath so this directory sorts first, its path being a prefix of every + // row the filter leaves visible, and draw to return to the first page. + // Without this a directory holding more than one page of files sorts onto + // a later page by uploadTime, and has no node to move into the header + table.order([{name: "fullPath", dir: "asc"}]).draw(); + let row = table.row((idx,data) => data.fullPath === dirData.fullPath); + let rowNode = row.node(); if (!rowNode) { - // if we are using the breadcrumb to jump back 2 directories or doing an upload - // while a subdirectory is opened, we won't have a rowNode because the row will - // not have been rendered yet. So draw the table with the oldRowData restored - table.draw(); - // and now we can try again - row = table.row((idx,data) => data.fullPath === dirData.fullPath); - rowNode = row.node(); + // no row for this directory, so take out whatever directory the + // header is still showing rather than leave it naming another place + let staleHead = document.querySelector(".dt-scroll-headInner > table:nth-child(1) > thead:nth-child(1)"); + if (staleHead.childNodes.length > 1) { + staleHead.removeChild(staleHead.lastChild); + } + table.order([{name: "uploadTime", dir: "desc"}]); + return; } oldRowData = row.data(); // put the data in the header: let rowClone = rowNode.cloneNode(true); // match the background color of the normal rows: rowClone.style.backgroundColor = "#fff9d2"; let thead = document.querySelector(".dt-scroll-headInner > table:nth-child(1) > thead:nth-child(1)"); // remove the checkbox because it doesn't do anything, and replace it // with a back arrow 'button' let btn = document.createElement("button"); btn.id = "backButton"; $(btn).button({icon: "ui-icon-triangle-1-w"}); btn.addEventListener("click", (e) => { let parentDir = dirData.parentDir; // Walk one level up by stripping the leaf segment. let pathParts = dirData.fullPath.split("/"); let parentDirPath = pathParts.slice(0, -1).join("/"); if (parentDirPath.length) { // Mirror the click-down path: filter, then move header row. dataTableShowDir(table, parentDir, parentDirPath); - dataTableCustomOrder(table, {fullPath: parentDirPath}); + // the whole row, so going back again knows this directory's parent + dataTableCustomOrder(table, uiState.filesHash[parentDirPath] || {fullPath: parentDirPath}); } else { dataTableShowTopLevel(table); dataTableCustomOrder(table); dataTableEmptyBreadcrumb(table); } table.draw(); }); let tdBtn = document.createElement("td"); tdBtn.appendChild(btn); rowClone.replaceChild(tdBtn, rowClone.childNodes[0]); if (thead.childNodes.length === 1) { thead.appendChild(rowClone); } else { thead.replaceChild(rowClone, thead.lastChild); } @@ -2123,80 +2145,124 @@ // frontend wise: move the file row into a 'child' of the hub row console.log(`sending addToHub req for ${rowData.fileName} to `); cart.setCgiAndUrl(fileListEndpoint); cart.send({addToHub: {hubName: "", dataFile: ""}}); cart.flush(); } function updateQuota(newFileSize) { // Change the quota displayed to the user, pass in newFileSize as a negative number // when deleting files let container = document.getElementById("quotaDiv"); uiState.userQuota += newFileSize; container.textContent = `Using ${prettyFileSize(uiState.userQuota)} of ${prettyFileSize(uiState.maxQuota)}`; } + // Response bodies from tus, keyed by upload URL. Uppy's tus plugin aborts the + // request before it emits upload-success, and aborting an XMLHttpRequest clears + // its status and its responseText, so the body has to be read while the request + // is still live + let tusResponseBodies = {}; + + function rememberTusResponseBody(req, res) { + // tus onAfterResponse hook, called for every request an upload makes. Only the + // PATCH that finishes the upload carries the file list from the pre-finish hook + if (req.getMethod() !== "PATCH") { + return; + } + let body = res.getBody(); + if (body) { + tusResponseBodies[req.getURL()] = body; + } + } + + function uploadedHubFromResponse(response) { + // Return the hubSpace rows the pre-finish hook reported for this upload, or + // null. tusd forwards the hook's response body on the request that completes + // the upload, which rememberTusResponseBody saved under this upload's URL + let url = response ? response.uploadURL : null; + if (!url) { + return null; + } + let text = tusResponseBodies[url]; + delete tusResponseBodies[url]; + if (!text) { + return null; + } + try { + let parsed = JSON.parse(text); + return parsed.fileList && parsed.fileList.length > 0 ? parsed.fileList : null; + } catch (e) { + console.error(`could not parse upload response: ${e}`); + return null; + } + } + function addNewUploadedHubToTable(hub) { - // hub is a list of objects representing the file just uploaded, the associated - // hub.txt, and directory. Make a new row for each in the filesTable, except for - // maybe the hub directory row and hub.txt which we may have already seen before + // hub is the list of rows the server holds for the hub this upload went into: + // the file itself, the hub.txt, and a row per directory. Add the ones the table + // has not seen and refresh the ones it has let table = $("#filesTable").DataTable(); - let justUploaded = {}; // hash of contents of hub but keyed by fullPath let hubDirData = {}; // the data for the parentDir of the uploaded file + // index the table once: hub carries every row of the hub, so looking each one + // up by scanning the table would be quadratic on a hub with many files + let rowIndexByPath = {}; + table.rows().every(function() { + rowIndexByPath[this.data().fullPath] = this.index(); + }); for (let obj of hub) { if (!obj.parentDir) { hubDirData = obj; } - let rowObj; if (!(obj.fullPath in uiState.filesHash)) { - justUploaded[obj.fullPath] = obj; - rowObj = table.row.add(obj); + table.row.add(obj); uiState.fileList.push(obj); // NOTE: we don't add the obj to the filesHash until after we're done // so we don't need to reparse all files each time we add one } else { - // File already exists - update the existing row with new data (for overwrites) - let existingObj = uiState.filesHash[obj.fullPath]; - existingObj.fileSize = obj.fileSize; - existingObj.lastModified = obj.lastModified; - existingObj.uploadTime = obj.uploadTime; - // Find and invalidate the row in DataTable to refresh display - let allRows = table.rows().indexes(); - for (let j = 0; j < allRows.length; j++) { - let rowData = table.row(allRows[j]).data(); - if (rowData.fullPath === obj.fullPath) { - table.row(allRows[j]).invalidate(); - break; - } + // Row already in the table, take the server's values for it. An upload + // changes more than its own row: a 2bit flips every row in the hub to + // assemblyHub, and a re-upload changes size, md5sum and times + Object.assign(uiState.filesHash[obj.fullPath], obj); + if (obj.fullPath in rowIndexByPath) { + table.row(rowIndexByPath[obj.fullPath]).invalidate(); } } } // show all the new rows we just added, note the double draw, we need // to have the new rows rendered to do the order because the order // will copy the actual DOM node parseFileListIntoHash(uiState.fileList); // stay in the directory the user has open, the upload may have gone into a // subdirectory of the hub and would not be listed at the hub level. Both calls // have to name the same directory, or the row moved into the header and the row // dropped from the table are different ones let showDirData = hubDirData; if (uiState.currentHubPath && uiState.currentHubPath in uiState.filesHash) { showDirData = uiState.filesHash[uiState.currentHubPath]; } + if (showDirData.fullPath) { dataTableShowDir(table, showDirData.fileName, showDirData.fullPath); dataTableCustomOrder(table, showDirData); + } else { + // no directory to open, so show everything rather than filter on a + // path we do not have + dataTableShowTopLevel(table); + dataTableCustomOrder(table); + dataTableEmptyBreadcrumb(table); + } table.draw(); } function doRowSelect(evtype, table, indexes) { let selectedRow = table.row(indexes); let rowTr = selectedRow.node(); if (rowTr) { handleCheckboxSelect(evtype, table, selectedRow); } } function indentActionButton(rowTr, rowData) { let numIndents = "0px"; //data.parentDir !== "" ? data.fullPath.split('/').length - 1: 0; if (rowData.fileType !== "dir") { numIndents = "10px"; @@ -2317,32 +2383,32 @@ { targets: 9, visible: false, searchable: false, orderable: true, } ], columns: [ {data: "", }, {data: "", }, {data: "fileName", title: "File name"}, {data: "fileSize", title: "File size"}, {data: "fileType", title: "File type"}, {data: "genome", title: "Genome"}, {data: "parentDir", title: "Hubs"}, - {data: "lastModified", title: "File Last Modified"}, - {data: "uploadTime", title: "Upload Time", name: "uploadTime"}, + {data: "lastModified", title: "File Last Modified", render: renderTimeCell}, + {data: "uploadTime", title: "Upload Time", name: "uploadTime", render: renderTimeCell}, {data: "fullPath", title: "fullPath", name: "fullPath"}, ], drawCallback: function(settings) { console.log("table draw"); }, rowCallback: function(row, data, displayNum, displayIndex, dataIndex) { // row is a tr element, data is the td values // a row can represent one of three things: // a 'folder', with no parents, but with children // a folder with parents and children (can only come from hubtools // a 'file' with no children, but with parentDir // we assign the appropriate classes which are used later to // collapse/expand and select rows for viewing or deletion if (!data.parentDir) { row.className = "topLevelRow"; @@ -2450,31 +2516,31 @@ return; } if (e.target.closest && e.target.closest(".fileLink")) { e.stopPropagation(); return; } if (e.target.className !== "dt-select-checkbox") { e.stopPropagation(); // we've clicked somewhere not on the checkbox itself, we need to: // 1. open the directory if the clicked row is a directory // 2. select the file if the clicked row is a regular file let row = table.row(e.target); let data = row.data(); if (data.children && data.children.length > 0) { dataTableShowDir(table, data.fileName, data.fullPath); - dataTableCustomOrder(table, {"fullPath": data.fullPath}); + dataTableCustomOrder(table, data); table.draw(); } else { if (row.selected()) { row.deselect(); doRowSelect("deselect", table, row.index()); } else { row.select(); doRowSelect("select", table, row.index()); } } } }); return table; } @@ -2483,141 +2549,79 @@ if (uiState.fileList) { parseFileListIntoHash(uiState.fileList); } // first add the top level directories/files let table = showExistingFiles(uiState.fileList); uppy.use(Uppy.Dashboard, uppyOptions); // define this in init so globals are available at runtime let tusOptions = { endpoint: getTusdEndpoint(), withCredentials: true, retryDelays: null, removeFingerprintOnSuccess: true, // clean up localStorage after successful upload + onAfterResponse: rememberTusResponseBody, }; uppy.use(Uppy.Tus, tusOptions); uppy.use(BatchChangePlugin, {target: Uppy.Dashboard}); uppy.on('upload-error', (file, error, response) => { // Replace tus's verbose default ("tus: unexpected response while // uploading chunk, originated from request (method: PATCH, ...)") // with the message our hook actually sent. Overwrite per-file // state, global state.error (read by the StatusBar), and the // info[] array (transient banner) - Uppy core populates all three // with the wrapped message before this handler runs. let cleanMsg = extractHookErrorMessage(error, response); if (file) { uppy.setFileState(file.id, {error: cleanMsg}); } uppy.setState({error: cleanMsg, info: []}); // Long-duration banner so the user has time to read the message; // the StatusBar truncates to "Upload failed" and hides the rest // behind a "?" icon. uppy.info(cleanMsg, 'error', 30000); // Genome-name collision is fixable in place by editing the 2bit's // genome field, so reopen the file card. if (file && cleanMsg && cleanMsg.includes(hubGenomeCollisionErrFrag)) { const dash = uppy.getPlugin("Dashboard"); if (dash) dash.toggleFileCard(true, file.id); } }); uppy.on('upload-success', (file, response) => { - const metadata = file.meta; - const d = new Date(metadata.lastModified); - const pad = (num) => String(num).padStart(2, '0'); - const dFormatted = `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`; - const now = new Date(Date.now()); - const nowFormatted = `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())} ${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}`; - let newReqObj, hubTxtObj; - let hubType = metadata.hubType || "trackHub"; - // Multi-segment parentDir (split-hub uploads) needs per-segment rows. - let parentSegments = metadata.parentDir.split("/"); - let parentLeaf = parentSegments[parentSegments.length - 1]; - // the hub.txt lives at the hub, not in the subdirectory the file went into - let hubRoot = parentSegments[0]; - newReqObj = { - "fileName": cgiEncode(metadata.fileName), - "fileSize": metadata.fileSize, - "fileType": metadata.fileType, - "genome": metadata.genome, - "parentDir": cgiEncode(parentLeaf), - "lastModified": dFormatted, - "uploadTime": nowFormatted, - "fullPath": cgiEncode(metadata.parentDir) + "/" + cgiEncode(metadata.fileName), - "hubType": hubType, - }; - // from what I can tell, any response we would create in the pre-finish hook - // is completely ignored for some reason, so we have to fake the other files - // we would have created with this one file and add them to the table if they - // weren't already there: - // Only fabricate a hub.txt row when the backend actually synthesized - // one. Skip if the user supplied their own *.hub.txt (either already - // in filesHash from a prior upload, or coming in this same batch - - // upload-success order is arbitrary so the hub.txt row may not be - // in filesHash yet when a sibling's upload-success fires). - let dirHash = uiState.filesHash[cgiEncode(hubRoot)]; - let hubTxtExists = !!(dirHash && dirHash.children && - dirHash.children.some(c => c.fileType === "hub.txt")); - let batchHasHubTxt = metadata.batchHasHubTxt === "true"; - if (metadata.fileType !== "hub.txt" && !hubTxtExists && !batchHasHubTxt) { - hubTxtObj = { - "uploadTime": nowFormatted, - "lastModified": dFormatted, - "fileName": "hub.txt", - "fileSize": 0, - "fileType": "hub.txt", - "genome": metadata.genome, - "parentDir": cgiEncode(hubRoot), - "fullPath": cgiEncode(hubRoot) + "/hub.txt", - "hubType": hubType, - }; - } - // One dir row per path segment; leaf-only db, matching makeParentDirRows(). - // For split hubs the hub-root dir stays empty regardless of layout. - let isSplitHub = metadata.batchSplitHub === "true"; - let dirRows = []; - for (let i = 0; i < parentSegments.length; i++) { - let dirFullPath = parentSegments.slice(0, i + 1) - .map(cgiEncode).join("/"); - let dirParent = i > 0 ? cgiEncode(parentSegments[i - 1]) : ""; - let isLeaf = (i === parentSegments.length - 1); - let dirDb; - if (isSplitHub) { - dirDb = (isLeaf && parentSegments.length > 1) ? metadata.genome : ""; + // the file is on the server whatever the table does with it + updateQuota(file.meta.fileSize); + // uppy resolves this file's upload only after every upload-success listener + // has returned, so an error thrown here leaves the batch unfinished and the + // dialog open. The upload itself has already succeeded, keep it that way + try { + let hub = uploadedHubFromResponse(response); + if (hub) { + addNewUploadedHubToTable(hub); } else { - dirDb = isLeaf ? metadata.genome : ""; - } - dirRows.push({ - "uploadTime": nowFormatted, - "lastModified": dFormatted, - "fileName": cgiEncode(parentSegments[i]), - "fileSize": 0, - "fileType": "dir", - "genome": dirDb, - "parentDir": dirParent, - "fullPath": dirFullPath, - "hubType": hubType, - }); + // the hook reports the rows it wrote, so an empty body means the + // table cannot be updated without asking the server again + console.error(`upload of '${file.meta.fileName}' returned no file list`); + uppy.info(`'${file.meta.fileName}' uploaded, but this page could not ` + + `be updated to show it. Reload the page to see your files.`, + 'warning', 10000); } - let hub = dirRows.concat([newReqObj]); - if (hubTxtObj) { - hub.push(hubTxtObj); + } catch (e) { + console.error(`could not show '${file.meta.fileName}' in the table:`, e); } - addNewUploadedHubToTable(hub); - updateQuota(metadata.fileSize); }); uppy.on('complete', (result) => { history.replaceState(uiState, "", document.location.href); console.log("replace history with uiState"); }); inited = true; } function checkJsonData(jsonData, callerName) { // Return true if jsonData isn't empty and doesn't contain an error; // otherwise complain on behalf of caller. if (! jsonData) { alert(callerName + ': empty response from server'); } else if (jsonData.error) { console.error(jsonData.error);