310ed9324362faacfdf0ca7d0cd011605ce2e750
galt
  Tue Nov 28 03:05:06 2023 -0800
Improving ability to restore symlinked files back into their wrangle or submit directories so they are not lost. Includes special files such as 2bit under the path "". Handles either one dataset at a time, or can with great care be used to restore all files for all sets. Great care must be used with this command, and observe many caviots as well. We have no real way to preserve our symlinks other than this.

diff --git src/hg/cirm/cdw/lib/cdwLib.c src/hg/cirm/cdw/lib/cdwLib.c
index 66161b2..f2e9d5e 100644
--- src/hg/cirm/cdw/lib/cdwLib.c
+++ src/hg/cirm/cdw/lib/cdwLib.c
@@ -1616,74 +1616,94 @@
     {
     warn("Too few symlinks followed: %d symlinks. Where is the symlink created by cdwSubmit?", symlinkLevels);
     return NULL;
     }
 if (!sameString(path, oldPath))
     {
     warn("Found symlinks point to %s, expecting to find symlink pointing to old path %s", path, oldPath);
     return NULL;
     }
 
 freeMem(path);
 return lastPath;
 }
 
 
+void safeCopyFile(char *source, char *dest)
+/* Copy file from source to dest using temp file. Perform move at the end only when the file is complete.
+ * Prevents partial copies. */
+{
+char tempFileName[1024];
+safef(tempFileName, sizeof tempFileName, "%s.tempCdwCopy", dest);
+copyFile(source, tempFileName);
+mustRename(tempFileName, dest);
+}
+
+
 void cdwReallyRemoveFile(struct sqlConnection *conn, char *submitDir, long long fileId, boolean unSymlinkOnly, boolean really)
 /* If unSymlinkOnly is NOT specified, removes all records of file from database and from Unix file system if 
  * the really flag is set.  Otherwise just print some info on the file.
  * Tries to find original submitdir and replace symlink with real file to restore it. */
 {
+
 struct cdwFile *ef = cdwFileFromId(conn, fileId);
 char *path = cdwPathForFileId(conn, fileId);
 verbose(1, "%s id=%u, submitFileName=%s, path=%s\n", 
     unSymlinkOnly ? "unlocking" : "removing", ef->id, ef->submitFileName, path);
+struct cdwSubmit *es = cdwSubmitFromId(conn, ef->submitId);
 if (really)
     {
     char query[1024];
-    struct cdwSubmit *es = cdwSubmitFromId(conn, ef->submitId);
 
     if (!unSymlinkOnly)
 	{
 	cdwRemoveQaRecords(conn, fileId);
 	sqlSafef(query, sizeof(query),
 	    "delete from cdwGroupFile where fileId=%lld", fileId);
 	sqlUpdate(conn, query);
 	sqlSafef(query, sizeof(query), "delete from cdwValidFile where fileId=%lld", fileId);
 	sqlUpdate(conn, query);
 	sqlSafef(query, sizeof(query), "delete from cdwFile where id=%lld", fileId);
 	sqlUpdate(conn, query);
 	}
 
+    }
+
 char *lastPath = NULL;
 // skip symlink check if meta or manifest which do not get validated or license plate or symlink
 if (!((fileId == es->manifestFileId) || (fileId == es->metaFileId)))
     lastPath = findSubmitSymlink(ef->submitFileName, submitDir, path);
 if (lastPath)
     {
     verbose(3, "lastPath=%s path=%s\n", lastPath, path);
-	if (unlink(lastPath) == -1)  // drop about to be invalid symlink
-	    errnoAbort("unlink failure %s", lastPath);
-	copyFile(path, lastPath);
+    }
+
+if (really)
+    {
+    if (lastPath)
+	{
+        // safer copy protection for the symlink across
+	safeCopyFile(path, lastPath);
+
 	touchFileFromFile(path, lastPath);
 	chmod(lastPath, 0664);
 	freeMem(lastPath);
-	}
 
 	if (!unSymlinkOnly)
 	    mustRemove(path);
 	}
+    }
 freez(&path);
 cdwFileFree(&ef);
 }
 
 void cdwFileResetTags(struct sqlConnection *conn, struct cdwFile *ef, char *newTags, 
     boolean revalidate, int submitId)
 /* Reset tags on file, strip out old validation and QA,  schedule new validation and QA. */
 /* Remove existing QA records and rerun QA agent on given file.   */
 {
 long long fileId = ef->id;
 /* Update database to let people know format revalidation is in progress. */
 char query[4*1024];
 
 /* Update tags for file in cdwFile table. */
 sqlSafef(query, sizeof(query), "update cdwFile set tags='%s' where id=%lld", newTags, fileId);