469949aeb805fd5ad78395c248fe62bc583ccbf9
hiram
  Fri Dec 4 23:28:46 2020 -0800
use MD5() from openssl instead of md5HexForString() hundreds of times faster refs #23891

diff --git src/utils/twoBitDup/twoBitDup.c src/utils/twoBitDup/twoBitDup.c
index ab5f858..dca16dd 100644
--- src/utils/twoBitDup/twoBitDup.c
+++ src/utils/twoBitDup/twoBitDup.c
@@ -1,28 +1,28 @@
 /* twoBitDup - check to see if a twobit file has any identical sequences in it. */
 
 /* Copyright (C) 2013 The Regents of the University of California 
  * See README in this or parent directory for licensing information. */
 #include "common.h"
 #include "linefile.h"
 #include "hash.h"
 #include "options.h"
 #include "twoBit.h"
 #include "dnaseq.h"
 #include "math.h"
 #include "udc.h"
-#include "md5.h"
+#include <openssl/md5.h>
 
 // static char const rcsid[] = "$Id: newProg.c,v 1.30 2010/03/24 21:18:33 hiram Exp $";
 
 void usage()
 /* Explain usage and exit. */
 {
 errAbort(
   "twoBitDup - check to see if a twobit file has any identical sequences in it\n"
   "usage:\n"
   "   twoBitDup file.2bit\n"
   "options:\n"
   "  -keyList=file - file to write a key list, two columns: md5sum and sequenceName\n"
   "                   NOTE: use of keyList is very time expensive for 2bit files\n"
   "                   with a large number of sequences (> 5,000).  Better to\n"
   "                   use a cluster run with the doIdKeys.pl automation script.\n"
@@ -58,34 +58,49 @@
     verbose(2, "writing key list to %s\n", keyList);
     keyListFile = mustOpen(keyList, "w");
     }
 
 for (index = tbf->indexList; index != NULL; index = index->next)
     {
     verbose(2,"grabbing seq %s\n", index->name);
     int size;
     struct dnaSeq *seq = twoBitReadSeqFragExt(tbf, index->name,
 	0, 0, FALSE, &size);
     struct hashEl *hel;
     if ((hel = hashLookup(seqHash, seq->dna)) != NULL)
 	printf("%s and %s are identical\n", index->name, (char *)hel->val);
     else
 	hel = hashAdd(seqHash, seq->dna, index->name);
-    if (keyListFile) {
+    if (keyListFile)
+	{
+/* This used to be extremely slow:
+#include "md5.h"
        char *md5Sum = md5HexForString(seq->dna);
        fprintf(keyListFile, "%s\t%s\n", md5Sum, index->name);
        freeMem(md5Sum);
+ * changed to use MD5() in openssl 2020-12-04:
+ */
+	unsigned char md5Result[MD5_DIGEST_LENGTH];
+	MD5((unsigned char *)seq->dna, strlen(seq->dna), md5Result);
+	int i;
+	struct dyString *ds = newDyString(MD5_DIGEST_LENGTH);
+	for(i = 0; i < MD5_DIGEST_LENGTH; i++)
+	    {
+	    dyStringPrintf(ds, "%02x", md5Result[i]);
+	    }
+	fprintf(keyListFile, "%s\t%s\n", ds->string, index->name);
+	freeDyString(&ds);
 	}
     freeDnaSeq(&seq);
     }
 }
 
 int main(int argc, char *argv[])
 /* Process command line. */
 {
 optionInit(&argc, argv, options);
 if (argc != 2)
     usage();
 keyList = optionVal("keyList", NULL);
 udcSetDefaultDir(optionVal("udcDir", udcDefaultDir()));
 twoBitDup(argv[1]);
 return 0;