7ba5c812bda048b28ade940e0030b8000a02ae4b chmalee Mon Aug 10 11:58:51 2026 -0700 hubSpace: key rows on location so two hubs can hold the same file name, refs #37964 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> diff --git src/hg/lib/userdata.c src/hg/lib/userdata.c index 8f158a0d918..0bc28371523 100644 --- src/hg/lib/userdata.c +++ src/hg/lib/userdata.c @@ -225,120 +225,126 @@ struct dyString *ret = dyStringCreate("%s%s%s%s", pathPrefix, parentDir, lastChar(parentDir) == '/' ? "" : "/", fname); path = dyStringCannibalize(&ret); } else path = catTwoStrings(pathPrefix, fname); char canonicalPath[PATH_MAX]; realpath(path, canonicalPath); // after canonicalizing the path, make sure it starts with the userDataDir, to prevent // deleting files like blah/../../../../systemFile.text if (startsWith(pathPrefix, canonicalPath)) return cloneString(canonicalPath); } return NULL; } -static boolean checkHubSpaceRowExists(struct hubSpace *row) -/* Return TRUE if row already exists */ -{ -struct sqlConnection *conn = hConnectCentral(); -struct dyString *queryCheck = sqlDyStringCreate("select count(*) from hubSpace where userName='%s' and fileName='%s' and parentDir='%s'", row->userName, row->fileName, row->parentDir); -int ret = sqlQuickNum(conn, dyStringCannibalize(&queryCheck)); -hDisconnectCentral(&conn); -return ret > 0; -} - static boolean checkHubSpaceLocationExists(char *userName, char *location) -/* Return TRUE if location exists for userName and has exactly one row */ +/* Return TRUE if this user already has a row for location. location is the row's + * identity: a file name plus its immediate parent directory is not unique, two hubs + * can each hold a 'sub/test.bb' */ { struct sqlConnection *conn = hConnectCentral(); struct dyString *queryCheck = sqlDyStringCreate("select count(*) from hubSpace where userName='%s' and location='%s'", userName, location); int ret = sqlQuickNum(conn, dyStringCannibalize(&queryCheck)); hDisconnectCentral(&conn); -return ret == 1; +return ret > 0; } -boolean userHasOwnNamedHubTxtInDir(char *userName, char *parentDir) +boolean userHasOwnNamedHubTxtInDir(char *userName, char *hubName, char *hubDir) /* Return TRUE if the user uploaded a *.hub.txt file NOT literally named 'hub.txt' - * (e.g. 'araTha1.hub.txt') in parentDir. Distinguishes "user's own authoritative - * hub.txt" from "backend-synthesized hub.txt that we're free to modify". */ + * (e.g. 'araTha1.hub.txt') at the top level of hubDir. Distinguishes "user's own + * authoritative hub.txt" from "backend-synthesized hub.txt that we're free to modify". + * parentDir alone would also match a *.hub.txt sitting in some other hub's + * subdirectory that happens to be named hubName, so pin it to hubDir as well */ { -if (!userName || !parentDir || !parentDir[0]) return FALSE; +if (isEmpty(userName) || isEmpty(hubName) || isEmpty(hubDir)) return FALSE; +struct dyString *prefix = dyStringCreate("%s/", hubDir); struct sqlConnection *conn = hConnectCentral(); struct dyString *q = sqlDyStringCreate( "select count(*) from hubSpace where userName='%s' and parentDir='%s' " - "and fileType='hub.txt' and fileName<>'hub.txt'", - userName, parentDir); + "and left(location,%d)='%s' and fileType='hub.txt' and fileName<>'hub.txt'", + userName, hubName, (int)dyStringLen(prefix), dyStringContents(prefix)); int ret = sqlQuickNum(conn, dyStringCannibalize(&q)); hDisconnectCentral(&conn); +dyStringFree(&prefix); return ret > 0; } char *existingHubTypeForDir(char *userName, char *hubName) /* Return the hubType of this user's hub dir row (hubName with parentDir=''), * or NULL if no such row exists. The returned string is heap-allocated; * pre-finish is a short-lived hook process so it does not bother to free. */ { if (!userName || !hubName || !hubName[0]) return NULL; struct sqlConnection *conn = hConnectCentral(); struct dyString *q = sqlDyStringCreate( "select hubType from hubSpace where userName='%s' and fileName='%s' and parentDir=''", userName, hubName); char *ret = sqlQuickString(conn, dyStringCannibalize(&q)); hDisconnectCentral(&conn); return ret; } -char *hubNameFromPath(char *path) -/* Return the last directory component of path. Assume that a '.' char in the last component - * means that component is a filename and go back further */ +char *hubLeafFromPath(char *path) +/* Return the last '/' separated component of path, ignoring a trailing '/'. Callers + * pass a directory, so there is no filename to guess at: a '.' in the last component + * is part of a directory name, which isValidParentDir allows */ { char *copy = cloneString(path); -if (endsWith(copy, "/")) +while (endsWith(copy, "/")) trimLastChar(copy); -char *ptr = strrchr(copy, '/'); -// check to see if we're in a file name, like /blah/blah/name/hub.txt -if (ptr) - { - if (strchr(ptr, '.')) - { - *ptr = 0; - ptr = strrchr(copy, '/'); - } - if (ptr) +char *lastSlash = strrchr(copy, '/'); +if (lastSlash) { - ++ptr; - return cloneString(ptr); - } + char *leaf = cloneString(lastSlash + 1); + freeMem(copy); + return leaf; } return copy; } void addHubSpaceRowForFile(struct hubSpace *row) /* We created a file for a user, now add an entry to the hubSpace table for it */ { struct sqlConnection *conn = hConnectCentral(); // now write out row to hubSpace table if (!sqlTableExistsOnMain(conn, "hubSpace")) { errAbort("No hubSpace MySQL table is present. Please send an email to genome-www@soe.ucsc.edu describing the exact steps you took just before you got this error"); } hubSpaceSaveToDb(conn, row, "hubSpace", 0); hDisconnectCentral(&conn); } +static void fillEmptyDirRowDb(char *userName, char *location, char *db) +/* Give a directory row a genome if it does not have one yet. The rows above a file + * are created without one, so a hub whose first upload landed in a subdirectory has + * no genome on its own row, and the UI then reads it as a hub for genome "" and + * refuses to add anything to it. Only ever fills an empty value, so a hub that + * already has a genome keeps it */ +{ +if (isEmpty(userName) || isEmpty(location) || isEmpty(db)) + return; +struct sqlConnection *conn = hConnectCentral(); +struct dyString *q = sqlDyStringCreate( + "update hubSpace set db='%s' where userName='%s' and location='%s' and db=''", + db, userName, location); +sqlUpdate(conn, dyStringCannibalize(&q)); +hDisconnectCentral(&conn); +} + void makeParentDirRows(char *userName, time_t lastModified, char *db, char *parentDirStr, char *userDataDir, char *hubType) /* For each '/' separated component of parentDirStr, create a row in hubSpace. Return the * final subdirectory component of parentDirStr */ { int i, slashCount = countChars(parentDirStr, '/'); char *components[256]; struct dyString *currLocation = dyStringCreate("%s", userDataDir); int foundSlashes = chopByChar(cloneString(parentDirStr), '/', components, slashCount); if (foundSlashes > 256) errAbort("parentDir setting '%s' too long", parentDirStr); for (i = 0; i < foundSlashes; i++) { char *subdir = components[i]; if (sameString(subdir, ".")) continue; @@ -349,33 +355,35 @@ dyStringAppend(currLocation, subdir); struct hubSpace *row = NULL; AllocVar(row); row->userName = userName; row->fileName = subdir; row->fileSize = 0; row->fileType = "dir"; row->creationTime = NULL; row->lastModified = sqlUnixTimeToDate(&lastModified, TRUE); // Leaf-only db; ancestors sit above per-genome subdirs. row->db = (i == foundSlashes - 1) ? db : ""; row->location = cloneString(dyStringContents(currLocation)); row->md5sum = ""; row->parentDir = i > 0 ? components[i-1] : ""; row->hubType = hubType ? hubType : "trackHub"; - // only insert a row for this parentDir if it's unique to the table - if (!checkHubSpaceRowExists(row)) + // only insert a row for this directory if it's not in the table yet + if (!checkHubSpaceLocationExists(row->userName, row->location)) addHubSpaceRowForFile(row); + else + fillEmptyDirRowDb(row->userName, row->location, db); } } static char *defaultPosFromTwoBit(char *twoBitPath) /* Open the 2bit, pick the first sequence and return "chrom:1-min(size,1000)". * Returns NULL if the 2bit cannot be opened, has no sequences, or the first * sequence name would inject content into hub.txt. */ { struct errCatch *errCatch = errCatchNew(); char *result = NULL; if (errCatchStart(errCatch)) { struct twoBitFile *tbf = twoBitOpen(twoBitPath); if (tbf && tbf->indexList && trackHubIsValidSeqName(tbf->indexList->name)) { @@ -384,31 +392,31 @@ int end = (size < 1000) ? size : 1000; struct dyString *ds = dyStringCreate("%s:1-%d", firstName, end); result = dyStringCannibalize(&ds); } if (tbf) twoBitClose(&tbf); } errCatchEnd(errCatch); errCatchFree(&errCatch); return result; } char *hubRootFromParentDir(char *parentDir) /* Return the first '/' separated component of parentDir, which is the hub itself. * The hub.txt and the hubSpace dir row for a hub both live at that level, while - * hubNameFromPath gives the immediately containing directory, which for a nested + * hubLeafFromPath gives the immediately containing directory, which for a nested * parentDir like 'myHub/hg38' is a subdirectory of the hub */ { char *copy = cloneString(parentDir); char *firstSlash = strchr(copy, '/'); if (firstSlash) *firstSlash = 0; return copy; } char *hubPathFromParentDir(char *parentDir, char *userDataDir) /* Return the directory holding this hub's hub.txt, that is the user's directory * plus the hub component of parentDir */ { char *hubRoot = hubRootFromParentDir(parentDir); char *hubPath = catTwoStrings(userDataDir, hubRoot); @@ -529,61 +537,68 @@ // Write to a sibling temp file and rename into place so a partial write // (ENOSPC, SIGKILL, etc.) cannot leave the user's hub.txt truncated. char *tmpFile = cloneString(rTempName(hubDir, "hub", ".txt")); FILE *f = mustOpen(tmpFile, "w"); mustWrite(f, out->string, out->stringSize); carefulClose(&f); mustRename(tmpFile, hubFile); freeMem(tmpFile); freeMem(hubDir); freez(&defaultPos); dyStringFree(&out); slNameFreeList(&lines); } -static void refreshHubTextRow(char *userName, char *hubFile, char *hubName) +static void refreshHubTextRow(char *userName, char *hubFile) /* Bring the hubSpace row for a hub.txt back in line with the file on disk. Called * after rewriting a hub.txt that already has a row, so My Data and the quota do not - * keep reporting the size and checksum from before the rewrite */ + * keep reporting the size and checksum from before the rewrite. Keyed on location: + * a user can have a subdirectory whose name is another hub's name, which would give + * two hub.txt rows the same fileName and parentDir */ { -if (!fileExists(hubFile)) +if (isEmpty(userName) || !fileExists(hubFile)) return; time_t modTime = fileModTime(hubFile); struct sqlConnection *conn = hConnectCentral(); struct dyString *q = sqlDyStringCreate( "update hubSpace set fileSize=%lld, md5sum='%s', lastModified='%s' " - "where userName='%s' and fileName='hub.txt' and parentDir='%s'", + "where userName='%s' and location='%s'", (long long)fileSize(hubFile), md5HexForFile(hubFile), - sqlUnixTimeToDate(&modTime, TRUE), userName, hubName); + sqlUnixTimeToDate(&modTime, TRUE), userName, hubFile); sqlUpdate(conn, dyStringCannibalize(&q)); hDisconnectCentral(&conn); } -static void setAssemblyHubTypeForDir(char *userName, char *parentDir) -/* Flip this user's hub (dir row + direct-child files) to hubType='assemblyHub'. - * Does not recurse into nested parentDirs like hubName/tracks; only the - * hubtools-then-UI promotion flow can produce those. */ +static void setAssemblyHubTypeForDir(char *userName, char *hubDir) +/* Flip every row of this user's hub to hubType='assemblyHub': the hub's own directory + * row and everything below it, however deeply nested. Matches on a location prefix + * with left() rather than LIKE, since a hub name may contain '_' and LIKE would read + * that as a wildcard. left() counts characters, which is the same as bytes here + * because cgiEncodeFull leaves only ASCII in a path. Callers run in the tusd hook, + * where getDataDir has already canonicalized the prefix the rows were written with */ { -if (!userName || !parentDir || parentDir[0] == '\0') return; +if (isEmpty(userName) || isEmpty(hubDir)) return; +struct dyString *prefix = dyStringCreate("%s/", hubDir); struct sqlConnection *conn = hConnectCentral(); struct dyString *q = sqlDyStringCreate( "update hubSpace set hubType='assemblyHub' " - "where userName='%s' and (parentDir='%s' or (fileName='%s' and parentDir=''))", - userName, parentDir, parentDir); + "where userName='%s' and (location='%s' or left(location,%d)='%s')", + userName, hubDir, (int)dyStringLen(prefix), dyStringContents(prefix)); sqlUpdate(conn, dyStringCannibalize(&q)); hDisconnectCentral(&conn); +dyStringFree(&prefix); } int lockHubDir(char *hubDir) /* Acquire an exclusive flock on hubDir/.hub.lock, creating the lock file * if necessary. Returns a file descriptor; pass to unlockHubDir to release. * Serializes hub.txt read-modify-write across parallel pre-finish processes. */ { struct dyString *lockPath = dyStringCreate("%s%s.hub.lock", hubDir, endsWith(hubDir, "/") ? "" : "/"); // O_CLOEXEC so the md5sum children we fork under this lock do not inherit it. // An inherited fd outliving a killed hook would hold the lock with no owner left // to release it, and every later upload to the hub would time out int fd = open(dyStringContents(lockPath), O_RDWR | O_CREAT | O_CLOEXEC, 0666); if (fd < 0) errnoAbort("could not open hub lock %s", dyStringContents(lockPath)); @@ -605,80 +620,74 @@ sleep1000(HUB_LOCK_POLL_MS); waitedMs += HUB_LOCK_POLL_MS; } dyStringFree(&lockPath); return fd; } void unlockHubDir(int fd) /* Release an exclusive hub lock acquired by lockHubDir. Closing the fd * releases the flock automatically on Linux. */ { if (fd >= 0) close(fd); } -void upgradeExistingHubToAssembly(struct hubSpace *rowForFile, char *userDataDir, char *encodedParentDir) +void upgradeExistingHubToAssembly(struct hubSpace *rowForFile, char *userDataDir) /* When a 2bit lands in a hub, add the assembly stanza to hub.txt (if the * backend owns it) and flip every row for this hub to hubType='assemblyHub'. * No-op unless rowForFile is a 2bit. */ { if (!sameOk(rowForFile->fileType, "2bit")) return; char *hubDir = hubPathFromParentDir(rowForFile->parentDir, userDataDir); struct dyString *hubFileDy = dyStringCreate("%s%shub.txt", hubDir, endsWith(hubDir, "/") ? "" : "/"); char *hubFile = dyStringCannibalize(&hubFileDy); upgradeHubTxtForAssembly(hubFile, rowForFile->db, rowForFile->location); -// setAssemblyHubTypeForDir looks up the hub's row at the top level, so give it the -// hub component of parentDir rather than a nested subdirectory -char *hubNameOnly = encodedParentDir ? hubRootFromParentDir(encodedParentDir) : NULL; -if (hubNameOnly && hubNameOnly[0]) - { - setAssemblyHubTypeForDir(rowForFile->userName, hubNameOnly); +setAssemblyHubTypeForDir(rowForFile->userName, hubDir); // hub.txt just changed on disk, so the row's size and md5 are out of date - refreshHubTextRow(rowForFile->userName, hubFile, hubNameOnly); - } +refreshHubTextRow(rowForFile->userName, hubFile); freeMem(hubFile); } char *writeHubText(char *path, char *userName, char *db, char *twoBitFileName) /* Create a hub.txt file, optionally creating the directory holding it. * If twoBitFileName is non-NULL, write an assembly hub stanza referencing it * (with stub organism / scientificName / description / defaultPos derived from * the 2bit). For convenience, return the file name of the created hub, which * can be freed. */ { // an empty path would put the hub.txt at the root of the filesystem if (isEmpty(path)) errAbort("no directory given for the hub, cannot create hub.txt"); int oldUmask = 00; oldUmask = umask(0); makeDirsOnPath(path); // restore umask umask(oldUmask); // now make the hub.txt with some basic information char *hubFile = NULL; struct dyString *hubFileDy = dyStringCreate("%s%shub.txt", path, endsWith(path, "/") ? "" : "/"); hubFile = dyStringCannibalize(&hubFileDy); if (fileExists(hubFile)) return hubFile; -char *hubName = hubNameFromPath(path); +char *hubName = hubLeafFromPath(path); FILE *f = mustOpen(hubFile, "w"); fprintf(f, "hub %s\n" "email %s\n" "shortLabel %s\n" "longLabel %s\n" "useOneFile on\n" "\n" "genome %s\n", hubName, emailForUserName(userName), hubName, hubName, db); if (twoBitFileName) { // Assembly hub: write twoBitPath plus stub fields the user can edit later. // The bigDataUrl/twoBitPath is relative to the hub.txt location. char *twoBitBase = pathRelativeToHubDir(twoBitFileName, path); @@ -814,33 +823,35 @@ char *hubPath = writeHubStanzasForFile(rowForFile, userDataDir); // update the mysql table with a record of the hub.txt: struct hubSpace *hubTextRow = NULL; AllocVar(hubTextRow); hubTextRow->userName = rowForFile->userName; hubTextRow->fileName = "hub.txt"; hubTextRow->fileSize = fileSize(hubPath); hubTextRow->fileType = "hub.txt"; hubTextRow->creationTime = NULL; time_t lastModTime = fileModTime(hubPath); hubTextRow->lastModified = sqlUnixTimeToDate(&lastModTime, TRUE); hubTextRow->db = rowForFile->db; hubTextRow->location = hubPath; hubTextRow->md5sum = md5HexForFile(hubPath); -hubTextRow->parentDir = hubNameFromPath(hubPath); +// the hub.txt sits at the hub root, so take the hub name from the file's parentDir +// rather than picking it back out of the hub.txt path +hubTextRow->parentDir = hubRootFromParentDir(rowForFile->parentDir); hubTextRow->hubType = rowForFile->hubType ? rowForFile->hubType : "trackHub"; -if (!checkHubSpaceRowExists(hubTextRow)) +if (!checkHubSpaceLocationExists(hubTextRow->userName, hubTextRow->location)) addHubSpaceRowForFile(hubTextRow); } static void deleteHubSpaceRow(char *fname, char *userName) /* Deletes a row from the hubspace table for a given fname */ { struct sqlConnection *conn = hConnectCentral(); struct dyString *deleteQuery = sqlDyStringCreate("delete from hubSpace where location='%s' and userName='%s'", fname, userName); sqlUpdate(conn, dyStringCannibalize(&deleteQuery)); hDisconnectCentral(&conn); } void removeFileForUser(char *fname, char *userName) /* Remove a file (or recursively, a directory) for this user if it exists */ {