c8c1b1a9adac6dfa7a869326ca3f774a724a50db
lrnassar
  Fri Sep 4 17:51:48 2026 -0700
Document the RefSeq Historical protein table build and add a reminder to the notifier.

The historical rna.gbff we already download carries a /translation for every
coding record, and gbProcess has a -pepFa flag that writes them out, so this is
one extra pass over a file we have rather than a new download. Adds the build
steps to the RS_2024_08 makedoc section, and two lines to the weekly notifier so
the next refresh does not leave the protein table behind. refs #38248

diff --git src/hg/makeDb/doc/hg38/ncbiRefSeq.txt src/hg/makeDb/doc/hg38/ncbiRefSeq.txt
index e8e294eae5c..aef09fce51b 100644
--- src/hg/makeDb/doc/hg38/ncbiRefSeq.txt
+++ src/hg/makeDb/doc/hg38/ncbiRefSeq.txt
@@ -538,30 +538,59 @@
 ln -sf `pwd`/refSeqHistorical.deDuped.fa /gbdb/hg38/ncbiRefSeq/refSeqHistorical.fa
 hgLoadSeq -drop -seqTbl=seqNcbiRefSeqHistorical -extFileTbl=extNcbiRefSeqHistorical hg38 /gbdb/hg38/ncbiRefSeq/refSeqHistorical.fa
 
 # link metadata table for the hgvs xref search
 zcat ${REL}_knownrefseq_rna.gbff.gz | (grep ' :: ' || true) \
   | perl -wpe 's/\s+::.*//; s/^\s+//;' | sort -u > pragmaLabels.txt
 /hive/data/outside/genbank/bin/x86_64/gbProcess /dev/null raFile.txt ${REL}_knownrefseq_rna.gbff.gz 2>gbProcess.err
 cat pragmaLabels.txt | tr -s '[:blank:]' | cut -d':' -f1 | sort -u | cut -d' ' -f2- > pragmaLabels.cleaned.txt
 ~/kent/src/hg/utils/automation/gff3ToRefLink.pl \
     raFile.txt ${REL}_genomic.gff.gz pragmaLabels.cleaned.txt continueOnParentErrors \
     2> refLink.stderr | LC_ALL=C sort > refLink.tab
 cut -f1 refSeqHistorical.gp | LC_ALL=C sort -u > name.list
 LC_ALL=C join -t$'\t' name.list refLink.tab > ncbiRefSeqLinkOld.tab
 hgLoadSqlTab hg38 ncbiRefSeqLinkHistorical ~/kent/src/hg/lib/ncbiRefSeqLink.sql ncbiRefSeqLinkOld.tab
 
+# protein sequences for deprecated NP_ accessions (DONE - Lou - 2026-09-04)
+#############################################################################
+# ncbiRefSeqLinkHistorical names the protein for every deprecated transcript but
+# we stored no sequences, so HGVS p. terms and bare NP_ searches on a deprecated
+# protein version did not resolve.  NCBI publishes no protein FASTA for the
+# historical set; the translations are in the rna.gbff downloaded above and
+# gbProcess -pepFa writes them out.  Redo this step on every historical refresh.
+# refs #38248
+cd /hive/data/outside/refSeqHistorical/RS_2024_08
+
+# second gbProcess pass, so the link-table commands above stay exactly as run.
+/hive/data/outside/genbank/bin/x86_64/gbProcess -pepFa=refSeqHistoricalPep.fa \
+    /dev/null /dev/null ${REL}_knownrefseq_rna.gbff.gz 2>gbProcessPep.err
+
+# sort -u is safe here: no accession in this file carries two different
+# sequences.  protAcc is field 6 of ncbiRefSeqLink.sql
+# (id, status, name, product, mrnaAcc, protAcc, locusLinkId, omimId, ...).
+# Restrict to the accessions the link table references; nothing can look up the
+# rest.  LC_ALL=C on both sides of the join, as above.
+faToTab -type=protein -keepAccSuffix refSeqHistoricalPep.fa stdout \
+    | LC_ALL=C sort -u > refSeqHistoricalPep.tab
+cut -f6 ncbiRefSeqLinkOld.tab | grep -v '^$' | LC_ALL=C sort -u > histProtAcc.list
+LC_ALL=C join -t$'\t' histProtAcc.list refSeqHistoricalPep.tab \
+    > ncbiRefSeqPepTableHistorical.tab
+
+# for RS_2024_08: 162041 fasta records, 87191 unique, 66119 after the join
+hgLoadSqlTab hg38 ncbiRefSeqPepTableHistorical ~/kent/src/hg/lib/pepPred.sql \
+    ncbiRefSeqPepTableHistorical.tab
+
 # version string for the track (DONE - Matt - 2026-08-27)
 #############################################################################
 # This subtrack has its own dataVersion, separate from the composite's
 # /gbdb/hg38/ncbiRefSeq/ncbiRefSeqVersion.txt, which tracks the current RefSeq
 # annotation release and is years ahead of the historical data. Without this
 # file the Historical track advertises the current release in its long label,
 # on its track settings page and on its details pages.
 # Redo this step on every historical refresh, with the release NCBI published.
 cd /hive/data/outside/refSeqHistorical/RS_2024_08
 echo "NCBI RefSeq ${REL} (2024-10-22)" > ncbiRefSeqHistoricalVersion.txt
 ln -sf `pwd`/ncbiRefSeqHistoricalVersion.txt \
     /gbdb/hg38/ncbiRefSeq/ncbiRefSeqHistoricalVersion.txt
 
 # trackDb: dataVersion on the ncbiRefSeqHistorical stanza in
 # kent/src/hg/makeDb/trackDb/human/trackDb.ra