77a819d426026e8a6ac3d7965af8f44fbb9a0272 hiram Wed Jan 10 12:42:07 2024 -0800 better manage large genome construction and add in RepeatModeler track refs #29545 diff --git src/hg/utils/automation/asmHubRmodelJoinAlign.pl src/hg/utils/automation/asmHubRmodelJoinAlign.pl new file mode 100755 index 0000000..035aaf9 --- /dev/null +++ src/hg/utils/automation/asmHubRmodelJoinAlign.pl @@ -0,0 +1,338 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use FindBin qw($Bin); +use lib "$Bin"; +use AsmHub; +use File::Basename; + +my $argc = scalar(@ARGV); + +if ($argc != 2) { + printf STDERR "usage: asmHubRmodelJoinAlign.pl asmId buildDir > asmId.repeatModeler.html\n"; + printf STDERR "where asmId is the assembly identifier,\n"; + printf STDERR "expecting to find buildDir/html/asmId.names.tab naming file for this assembly,\n"; + printf STDERR "and buildDir/trackData/repeatModeler/asmId.rmsk.class.profile counts of rmsk categories.\n"; + exit 255; +} + +my $asmId = shift; +my $buildDir = shift; +my $namesFile = "$buildDir/html/$asmId.names.tab"; +my $rModelClassProfile = "$buildDir/trackData/repeatMasker/$asmId.rmsk.class.profile.txt"; +my $faSizeFile = "$buildDir/trackData/repeatModeler/faSize.rmsk.txt"; +my $rmskVersion = "$buildDir/trackData/repeatModeler/versionInfo.txt"; +my $rModelVersion = "$buildDir/trackData/repeatModeler/modelerVersion.txt"; + +my $maskingPercent = ""; +if ( -s "${faSizeFile}" ) { + $maskingPercent=`grep -w masked "${faSizeFile}" | cut -d' ' -f1`; + chomp $maskingPercent; +} + +my $errOut = 0; +if ( ! -s $rModelClassProfile ) { + printf STDERR "ERROR: can not find rmsk class profile file:\n\t'%s'\n", $rModelClassProfile; + $errOut = 255; +} + +if ( ! -s $namesFile ) { + printf STDERR "ERROR: can not find rmsk class profile file:\n\t'%s'\n", $rModelClassProfile; + $errOut = 255; +} + +if ($errOut) { + exit $errOut; +} + +my $em = ""; +my $noEm = ""; +my $assemblyDate = `grep -v "^#" $namesFile | cut -f9`; +chomp $assemblyDate; +my $ncbiAssemblyId = `grep -v "^#" $namesFile | cut -f10`; +chomp $ncbiAssemblyId; +my $organism = `grep -v "^#" $namesFile | cut -f5`; +chomp $organism; + +print <<_EOF_ +

Description

+

+This track shows the Repeat Masker annotations on the $assemblyDate $em${organism}$noEm/$asmId genome assembly as calculated from a custom library produced by RepeatModeler. +

+ +

+This track was created by using Arian Smit's +RepeatMasker +program, which screens DNA sequences +for interspersed repeats and low complexity DNA sequences. The program +outputs a detailed annotation of the repeats that are present in the +query sequence (represented by this track), as well as a modified version +of the query sequence in which all the annotated repeats have been masked +(generally available on the +Downloads page). RepeatMasker uses the +Repbase Update library of repeats from the +Genetic +Information Research Institute (GIRI). +Repbase Update is described in Jurka (2000) in the References section below.

+_EOF_ +; + +if ( length($maskingPercent) ) { + printf "

Percent masking of sequence: %s

\n", $maskingPercent; + my $asmSize=`grep -w bases "${faSizeFile}" | cut -d' ' -f1`; + chomp $asmSize; + my $maskedBases=`grep -w bases "${faSizeFile}" | cut -d' ' -f9`; + chomp $maskedBases; + printf "

Assembly size: %s bases
\n", &AsmHub::commify($asmSize); + printf "Sequence masked: %s bases\n", &AsmHub::commify($maskedBases); + printf "

\n"; +} + +if ( -s "$rModelVersion" ) { + my $modelerVersion = `cat $rModelVersion`; + chomp $modelerVersion; + $modelerVersion =~ s/version/version:/; + printf "

%s

\n", $modelerVersion; +} + +if ( -s "$rmskVersion" ) { + +print <<_EOF_ +

RepeatMasker and libraries version

+

+

+_EOF_
+;
+print `cat $rmskVersion`;
+print <<_EOF_
+
+

+_EOF_ +; + +} + +print <<_EOF_ +

Display Conventions and Configuration

+

Context Sensitive Zooming

+

+This track employs a technique which chooses the appropriate visual representation for the data based on the +zoom scale, and or the number of annotations currently in view. The track will automatically switch from the +most detailed visualization ('Full' mode) to the denser view ('Pack' mode) when the window size is greater +than 45kb of sequence. It will further switch to the even denser single line view ('Dense' mode) if more than +500 annotations are present in the current view. +

+

Dense Mode Visualization

+

+In dense display mode, a single line is displayed denoting the coverage of repeats using a series +of colored boxes. The boxes are colored based on the classification of the repeat (see below for legend). +
+
+ + +

+

Pack Mode Visualization

+

+In pack mode, repeats are represented as sets of joined features. These are color coded as above based on the +class of the repeat, and the further details such as orientation (denoted by chevrons) and a family label are provided. +This family label may be optionally turned off in the track configuration. +
+
+ + +
+
+The pack display mode may also be configured to resemble the original UCSC repeat track. In this visualization +repeat features are grouped by classes (see below), and displayed on seperate track lines. The repeat ranges are +denoted as grayscale boxes, reflecting both the size of the repeat and +the amount of base mismatch, base deletion, and base insertion associated with a repeat element. +The higher the combined number of these, the lighter the shading. +
+
+ + +

+

Full Mode Visualization

+

+In the most detailed visualization repeats are displayed as chevron boxes, indicating the size and orientation of +the repeat. The interior grayscale shading represents the divergence of the repeat (see above) while the outline color +represents the class of the repeat. Dotted lines above the repeat and extending left or right +indicate the length of unaligned repeat model sequence and provide context for where a repeat fragment originates in its +consensus or pHMM model. If the length of the unaligned sequence +is large, an iterruption line and bp size is indicated instead of drawing the extension to scale. +
+
+ + +

+

+For example, the following repeat is a SINE element in the forward orientation with average +divergence. Only the 5' proximal fragment of the consensus sequence is aligned to the genome. +The 3' unaligned length (384bp) is not drawn to scale and is instead displayed using a set of +interruption lines along with the length of the unaligned sequence. +

+ +

+Repeats that have been fragmented by insertions or large internal deletions are now represented +by join lines. In the example below, a LINE element is found as two fragments. The solid +connection lines indicate that there are no unaligned consensus bases between the two fragments. +Also note these fragments form the 3' extremity of the repeat, as there is no unaligned consensus +sequence following the last fragment. +

+ +

+In cases where there is unaligned consensus sequence between the fragments, the repeat will look like +the following. The dotted line indicates the length of the unaligned sequence between the two +fragments. In this case the unaligned consensus is longer than the actual genomic distance between +these two fragments. +

+ +

+If there is consensus overlap between the two fragments, the joining lines will be drawn to indicate +how much of the left fragment is repeated in the right fragment. +

+ +

+The following table lists the repeat class colors: +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ColorRepeat Class
SINE - Short Interspersed Nuclear Element
LINE - Long Interspersed Nuclear Element
LTR - Long Terminal Repeat
DNA - DNA Transposon
Simple - Single Nucleotide Stretches and Tandem Repeats
Low_complexity - Low Complexity DNA
Satellite - Satellite Repeats
RNA - RNA Repeats (including RNA, tRNA, rRNA, snRNA, scRNA, srpRNA)
Other - Other Repeats (including class RC - Rolling Circle)
Unknown - Unknown Classification
+

+ +

+A "?" at the end of the "Family" or "Class" (for example, DNA?) +signifies that the curator was unsure of the classification. At some point in the future, +either the "?" will be removed or the classification will be changed.

+ +

Methods

+ +

+The RepeatMasker (www.repeatmasker.org) tool was used to generate the datasets found on this track hub. +

+ +

Class profiles

+

+

\n

\n

Detail class profiles

\n

\n

+

+

Credits

+ +

+Thanks to Arian Smit, Robert Hubley and GIRI for providing the tools and +repeat libraries used to generate this track. +

+ +

References

+ +

+Smit AFA, Hubley R, Green P. ${em}RepeatMasker Open-3.0${noEm}. + +http://www.repeatmasker.org. 1996-2010. +

+ +

+Repbase Update is described in: +

+ +

+Jurka J. + +Repbase Update: a database and an electronic journal of repetitive elements. +${em}Trends Genet${noEm}. 2000 Sep;16(9):418-420. +PMID: 10973072 +

+ +

+For a discussion of repeats in mammalian genomes, see: +

+ +

+Smit AF. + +Interspersed repeats and other mementos of transposable elements in mammalian genomes. +${em}Curr Opin Genet Dev${noEm}. 1999 Dec;9(6):657-63. +PMID: 10607616 +

+ +

+Smit AF. + +The origin of interspersed repeats in the human genome. +${em}Curr Opin Genet Dev${noEm}. 1996 Dec;6(6):743-8. +PMID: 8994846 +

+_EOF_ + ; +