Commit 91dbb878 authored by nchernia's avatar nchernia
Browse files

Merged changes from master

parents 8283baba d696dd12

SLURM/scripts/chimeric_blacklist.awk

+1 −1
Changes for SLURM/scripts/chimeric_blacklist.awk: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -74,7 +74,7 @@ BEGIN{ 
  OFS="\t";

  tottot = -1; # will count first non-group

}

{

$0 !~ /^@/{

  # input file is sorted by read name.  Look at read name to group 

  # appropriately

  split($1,a,"/");

SLURM/scripts/countligations.sh

+3 −3
Changes for SLURM/scripts/countligations.sh: 3 added lines, 3 removed lines.
Original line number Diff line number Diff line
@@ -27,10 +27,10 @@ 
# Juicer version 1.5

if [ "$usegzip" -eq 1 ]

then 

    num1=$(paste <(zcat $name1$ext) <(zcat $name2$ext) | grep -c $ligation)

    num2=$(zcat ${name1}${ext} | wc -l | awk '{print $1}')

    num1=$(paste <(gunzip -c $name1$ext) <(gunzip -c $name2$ext) | grep -cE $ligation)

    num2=$(gunzip -c ${name1}${ext} | wc -l | awk '{print $1}')

else

    num1=$(paste $name1$ext $name2$ext | grep -c $ligation)

    num1=$(paste $name1$ext $name2$ext | grep -cE $ligation)

    num2=$(wc -l ${name1}${ext} | awk '{print $1}')

fi

echo -ne "$num1 " > ${name}${ext}_norm.txt.res.txt

SLURM/scripts/dups.awk

+6 −0
Changes for SLURM/scripts/dups.awk: 6 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -50,8 +50,14 @@ function optcheck(tile1,tile2,x1,x2,y1,y2) { 
# Executed once, before beginning the file read

BEGIN {

	i=0;

  if (length(nowobble)==0) {

      wobble1=4;

      wobble2=4;

  }

  else {

      wobble1=0;

      wobble2=0;

  }

	# names of output files

	# the variable "name" can be set via the -v flag

	dupname=name"dups.txt";

SLURM/scripts/juicer.sh

+52 −32
Changes for SLURM/scripts/juicer.sh: 52 added lines, 32 removed lines.
Original line number Diff line number Diff line
@@ -65,7 +65,7 @@ 
#             match any files with the read1str.   

# Juicer version 1.5.6

shopt -s extglob

juicer_version="1.5.6"

juicer_version="1.5.7"

## Set the following variables to work with your system



# Aiden Lab specific check
@@ -113,7 +113,7 @@ else 
    queue_time="1440"

    # default long queue, can also be set in options

    long_queue="long"

    long_queue_time="10080"

    long_queue_time="1440"

fi



# size to split fastqs. adjust to match your needs. 4000000=1M reads per split
@@ -141,10 +141,13 @@ genomeID="hg19" 
shortreadend=0

# description, default empty

about=""

nofrag=0

# do not include fragment delimited maps by default

nofrag=1

# use wobble for dedupping by default (not just exact matches)

justexact=0



## Read arguments                                                     

usageHelp="Usage: ${0##*/} [-g genomeID] [-d topDir] [-q queue] [-l long queue] [-s site]\n                 [-a about] [-R end] [-S stage] [-p chrom.sizes path]\n                 [-y restriction site file] [-z reference genome file]\n                 [-C chunk size] [-D Juicer scripts directory]\n                 [-Q queue time limit] [-L long queue time limit] [-b ligation] [-t threads]\n                 [-A account name] [-r] [-h] [-x]"

usageHelp="Usage: ${0##*/} [-g genomeID] [-d topDir] [-q queue] [-l long queue] [-s site]\n                 [-a about] [-R end] [-S stage] [-p chrom.sizes path]\n                 [-y restriction site file] [-z reference genome file]\n                 [-C chunk size] [-D Juicer scripts directory]\n                 [-Q queue time limit] [-L long queue time limit] [-b ligation] [-t threads]\n                 [-A account name] [-r] [-h] [-f] [-j]"

genomeHelp="* [genomeID] must be defined in the script, e.g. \"hg19\" or \"mm10\" (default \n  \"$genomeID\"); alternatively, it can be defined using the -z command"

dirHelp="* [topDir] is the top level directory (default\n  \"$topDir\")\n     [topDir]/fastq must contain the fastq files\n     [topDir]/splits will be created to contain the temporary split files\n     [topDir]/aligned will be created for the final alignment"

queueHelp="* [queue] is the queue for running alignments (default \"$queue\")"
@@ -164,7 +167,8 @@ longQueueTimeHelp="* [long queue time limit]: time limit for long queue, i.e. -W 
ligationHelp="* [ligation junction]: use this string when counting ligation junctions"

threadsHelp="* [threads]: number of threads when running BWA alignment"

userHelp="* [account name]: user account name on cluster"

excludeHelp="* -x: exclude fragment-delimited maps from hic file creation"

excludeHelp="* -f: include fragment-delimited maps in hic file creation"

justHelp="* -j: just exact duplicates excluded at dedupping step"

helpHelp="* -h: print this help and exit"



printHelpAndExit() {
@@ -193,7 +197,7 @@ printHelpAndExit() { 
    exit "$1"

}



while getopts "d:g:R:a:hrq:s:p:l:y:z:S:C:D:Q:L:b:A:t:x" opt; do

while getopts "d:g:R:a:hrq:s:p:l:y:z:S:C:D:Q:L:b:A:t:jf" opt; do

    case $opt in

	g) genomeID=$OPTARG ;;

	h) printHelpAndExit 0;;
@@ -212,10 +216,11 @@ while getopts "d:g:R:a:hrq:s:p:l:y:z:S:C:D:Q:L:b:A:t:x" opt; do 
	D) juiceDir=$OPTARG ;;

	Q) queue_time=$OPTARG ;;

	L) long_queue_time=$OPTARG ;;

	x) nofrag=1 ;;

	f) nofrag=0 ;;

	b) ligation=$OPTARG ;;

	t) threads=$OPTARG ;;

	A) user=$OPTARG ;;

	j) justexact=1 ;;

	[?]) printHelpAndExit 1;;

    esac

done
@@ -252,7 +257,7 @@ else 
    ## file to be properly created

    if [ -z "$genomePath" ]

    then

        echo "***! You must define a chrom.sizes file via the \"-p\" flag that delineates the lengths of the chromosomes in the genome at $refSeq";

        echo "***! You must define a chrom.sizes file or a standard genome ID via the \"-p\" flag that delineates the lengths of the chromosomes in the genome at $refSeq; you may use \"-p hg19\" or other standard genomes";

        exit 1;

    fi

fi
@@ -273,6 +278,7 @@ fi 
if [ -z "$ligation" ]; then

    case $site in

	HindIII) ligation="AAGCTAGCTT";;

	MseI)  ligation="TTATAA";;

	DpnII) ligation="GATCGATC";;

	MboI) ligation="GATCGATC";;

	NcoI) ligation="CCATGCATGG";;
@@ -318,20 +324,25 @@ then 
    if [ $isRice -eq 1 ]

    then

	threads=8

	threadstring="-t $threads"

    elif [ $isBCM -eq 1 ]

    then

	threads=24

	threadstring="-t $threads"

    else

	threads=16 # VOLTRON; may need to make this separate and specific for Voltron

	threads=10 # VOLTRON; may need to make this separate and specific for Voltron

	## On voltron wiht 8 thread per core Power8 CPU bwa can use more threads

	threadstring="-t \$SLURM_JOB_CPUS_PER_NODE"

    fi

else

    threadstring="-t $threads"

fi



threadstring="-t $threads"

alloc_mem=$(($threads * 5000))



if [ $alloc_mem -gt 40000 ]

if [ $alloc_mem -gt 50000 ]

then

    alloc_mem=40000

    alloc_mem=50000

fi



## Directories to be created and regex strings for listing files
@@ -495,6 +506,7 @@ then 
                        #SBATCH -p $queue

			#SBATCH -t $queue_time

			#SBATCH -c 1

			#SBATCH --mem=5G

			#SBATCH -o $debugdir/split-%j.out

			#SBATCH -e $debugdir/split-%j.err

			#SBATCH -J "${groupname}_split_${i}"
@@ -510,6 +522,7 @@ SPLITEND` 
			#SBATCH -p $queue

			#SBATCH -t $queue_time

			#SBATCH -c 1

			#SBATCH --mem=5G

			#SBATCH -o $debugdir/split-%j.out

			#SBATCH -e $debugdir/split-%j.err

			#SBATCH -J "${groupname}_split_${i}"
@@ -584,6 +597,7 @@ SPLITEND` 
		#SBATCH -o $debugdir/count_ligation-%j.out

		#SBATCH -e $debugdir/count_ligation-%j.err

		#SBATCH -J "${groupname}${jname}_Count_Ligation"

		#SBATCH --mem=5G

                $userstring			



		date
@@ -603,7 +617,7 @@ CNTLIG` 
		#SBATCH -n 1

		#SBATCH -c $threads

		#SBATCH --ntasks=1

		#SBATCH --mem-per-cpu=$alloc_mem

		#SBATCH --mem=$alloc_mem

		#SBATCH -J "${groupname}_align1_${jname}"

		#SBATCH --threads-per-core=1		

                $userstring
@@ -650,7 +664,7 @@ ALGNR1` 
		#SBATCH -n 1

		#SBATCH -c $threads

		#SBATCH --ntasks=1

		#SBATCH --mem-per-cpu=$alloc_mem

		#SBATCH --mem=$alloc_mem

		#SBATCH -J "${groupname}_align2_${jname}"

		#SBATCH --threads-per-core=1		

                $userstring
@@ -694,7 +708,7 @@ ALGNR2` 
		#SBATCH -p $long_queue

		#SBATCH -o $debugdir/merge-%j.out

		#SBATCH -e $debugdir/merge-%j.err

		#SBATCH --mem-per-cpu=14G

		#SBATCH --mem=55G

		#SBATCH -t $queue_time

		#SBATCH -c 8 

		#SBATCH --ntasks=1
@@ -712,7 +726,7 @@ ALGNR2` 
			exit 1

		fi

		# sort read 1 aligned file by readname

		sort --parallel=8 -S 14G -T $tmpdir -k1,1 $name1$ext.sam > $name1${ext}_sort.sam

		sort --parallel=$threads -S 14G -T $tmpdir -k1,1f $name1$ext.sam > $name1${ext}_sort.sam

		#sort -T $tmpdir -k1,1 ${name1}${ext}.sam > ${name1}${ext}_sort.sam

		if [ \$? -ne 0 ]

		then
@@ -724,7 +738,7 @@ ALGNR2` 
		fi

		

		# sort read 2 aligned file by readname 

		sort --parallel=8 -S 14G -T $tmpdir -k1,1 $name2$ext.sam > $name2${ext}_sort.sam

		sort --parallel=$threads -S 14G -T $tmpdir -k1,1f $name2$ext.sam > $name2${ext}_sort.sam

		#sort -T $tmpdir -k1,1 $name2$ext.sam > $name2${ext}_sort.sam

		if [ \$? -ne 0 ]

		then
@@ -740,7 +754,7 @@ ALGNR2` 
		awk 'NF >= 11{\\\$1 = \\\$1"/2";print}' ${name2}${ext}_sort.sam > ${name2}${ext}_sort1.sam



		# merge the two sorted read end files

		sort --parallel=8 -S 14G -T $tmpdir -k1,1 -m $name1${ext}_sort1.sam $name2${ext}_sort1.sam > $name$ext.sam

		sort --parallel=$threads -S 14G -T $tmpdir -k1,1f -m $name1${ext}_sort1.sam $name2${ext}_sort1.sam > $name$ext.sam

		#sort -T $tmpdir -k1,1 -m $name1${ext}_sort1.sam $name2${ext}_sort1.sam > $name$ext.sam

		if [ \$? -ne 0 ]

		then
@@ -858,8 +872,7 @@ then 
        sbatch_wait=""

    fi

    

    # merge the sorted files into one giant file that is also sorted.      

    #jid=`sbatch <<- MRGSRT | egrep -o -e "\b[0-9]+$"

    # merge the sorted files into one giant file that is also sorted. jid=`sbatch <<- MRGSRT | egrep -o -e "\b[0-9]+$"

    

    if [ $isVoltron -eq 1 ]

    then
@@ -867,17 +880,24 @@ then 
    else

	sbatch_time="#SBATCH -t 1440"

    fi



    if [ $isBCM -eq 1 ]

    then

	sbatch_cpu_alloc="#SBATCH -c 1"

	sbatch_mem_alloc="#SBATCH --mem=8G"

    else

	sbatch_cpu_alloc="#SBATCH -c 8"

	sbatch_mem_alloc="#SBATCH --mem=64G"

    fi





    jid=`sbatch <<- EOF

		#!/usr/bin/bash

		#SBATCH -o $debugdir/fragmerge-%j.out

		#SBATCH -e $debugdir/fragmerge-%j.err

		#SBATCH --mem 256000

		${sbatch_mem_alloc}

		${sbatch_time}

		#SBATCH -p $long_queue

		#SBATCH -c 8

		${sbatch_cpu_alloc}

		#SBATCH -J "${groupname}_fragmerge"

		${sbatch_wait}

                $userstring
@@ -895,7 +915,7 @@ then 
		fi

		if [ $isRice -eq 1 ]

		then

			if ! ${juiceDir}/scripts/sort --parallel=48 -S8G -T ${tmpdir} -m -k2,2d -k6,6d -k4,4n -k8,8n -k1,1n -k5,5n -k3,3n $splitdir/*.sort.txt > $outputdir/merged_sort.txt

			if ! ${juiceDir}/scripts/sort --parallel=48 -S 32G -T ${tmpdir} -m -k2,2d -k6,6d -k4,4n -k8,8n -k1,1n -k5,5n -k3,3n $splitdir/*.sort.txt > $outputdir/merged_sort.txt

			then

				echo "***! Some problems occurred somewhere in creating sorted align files."

				touch $errorfile
@@ -904,7 +924,7 @@ then 
				echo "(-: Finished sorting all sorted files into a single merge."

			fi

		else

			if ! sort --parallel=48 -S8G -T ${tmpdir} -m -k2,2d -k6,6d -k4,4n -k8,8n -k1,1n -k5,5n -k3,3n $splitdir/*.sort.txt > $outputdir/merged_sort.txt

			if ! sort --parallel=48 -S 32G -T ${tmpdir} -m -k2,2d -k6,6d -k4,4n -k8,8n -k1,1n -k5,5n -k3,3n $splitdir/*.sort.txt > $outputdir/merged_sort.txt

			then

				echo "***! Some problems occurred somewhere in creating sorted align files."

				touch $errorfile
@@ -971,7 +991,7 @@ DEDUPGUARD` 
            exit 1 

        fi 

	squeue -u $USER -o "%A %T %j %E %R" | column -t

	awk -v queue=$long_queue -v groupname=$groupname -v debugdir=$debugdir -v dir=$outputdir -v topDir=$topDir -v juicedir=$juiceDir -v site=$site -v genomeID=$genomeID -v genomePath=$genomePath -v user=$USER -v guardjid=$guardjid -f $juiceDir/scripts/split_rmdups.awk $outputdir/merged_sort.txt

	awk -v queue=$long_queue -v groupname=$groupname -v debugdir=$debugdir -v dir=$outputdir -v topDir=$topDir -v juicedir=$juiceDir -v site=$site -v genomeID=$genomeID -v genomePath=$genomePath -v user=$USER -v guardjid=$guardjid -v justexact=$justexact -f $juiceDir/scripts/split_rmdups.awk $outputdir/merged_sort.txt

	##Schedule new job to run after last dedup part:

	##Push guard to run after last dedup is completed:

	##srun --ntasks=1 -c 1 -p "$queue" -t 1 -o ${debugdir}/dedup_requeue-%j.out -e ${debugdir}/dedup-requeue-%j.err -J "$groupname_msplit0" -d singleton echo ID: $ echo "\${!SLURM_JOB_ID}"; scontrol update JobID=$guardjid dependency=afterok:\$SLURM_JOB_ID
@@ -1075,7 +1095,7 @@ DUPCHECK` 
		#SBATCH -t $long_queue_time

		#SBATCH -c 1

		#SBATCH --ntasks=1

		#SBATCH --mem-per-cpu=6G

		#SBATCH --mem=25G

		#SBATCH -J "${groupname}_stats"

		${sbatch_wait}

                $userstring
@@ -1117,13 +1137,13 @@ STATS` 
	#SBATCH -t $long_queue_time

	#SBATCH -c 1

	#SBATCH --ntasks=1

	#SBATCH --mem-per-cpu=32G

	#SBATCH --mem=49G

	#SBATCH -J "${groupname}_hic"

	#SBATCH -d $dependstats

        $userstring			



	${load_java}

	export IBM_JAVA_OPTIONS="-Xmx48192m -Xgcthreads1"

	export IBM_JAVA_OPTIONS="-Xmx49152m -Xgcthreads1"

	date

	if [ -f "${errorfile}" ]

	then
@@ -1149,13 +1169,13 @@ HIC` 
	#SBATCH -t $long_queue_time

	#SBATCH -c 1

	#SBATCH --ntasks=1

	#SBATCH --mem-per-cpu=32G

	#SBATCH --mem=49G

	#SBATCH -J "${groupname}_hic30"

	#SBATCH -d ${dependstats}

        $userstring			



	${load_java}

	export IBM_JAVA_OPTIONS="-Xmx48192m -Xgcthreads1"

	export IBM_JAVA_OPTIONS="-Xmx49152m -Xgcthreads1"

	date

        if [ -f "${errorfile}" ]

        then
@@ -1186,7 +1206,7 @@ then 
    jid=`sbatch <<- HICCUPS | egrep -o -e "\b[0-9]+$"

	#!/bin/bash

	#SBATCH -p $queue

	#SBATCH --mem-per-cpu=2G

	#SBATCH --mem-per-cpu=4G

	${sbatch_req}

	#SBATCH -o $debugdir/hiccups_wrap-%j.out

	#SBATCH -e $debugdir/hiccups_wrap-%j.err

SLURM/scripts/juicer_arrowhead.sh

+15 −0
Changes for SLURM/scripts/juicer_arrowhead.sh: 15 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -37,7 +37,22 @@ printHelpAndExit() { 
#set defaults

genomeID="hg19"

hic_file_path="$(pwd)/aligned/inter_30.hic"



# Aiden Lab specific check

isRice=$(hostname | awk '{if ($1~/rice/){print 1}else {print 0}}')

isBCM=$(hostname | awk '{if ($1~/bcm/){print 1}else {print 0}}')

isVoltron=0

# Set default appropriately

if [ $isRice -eq 1 ] 

then

    juicer_tools_path="/projects/ea14/juicer/scripts/juicer_tools"

elif [ $isBCM -eq 1 ]

then    

    juicer_tools_path="/storage/aiden/juicer/scripts/juicer_tools"

else

    isVoltron=1

    juicer_tools_path="/gpfs0/juicer/scripts/juicer_tools"

fi



while getopts "h:j:i:" opt; do

    case $opt in

CRA Git | Maintained and supported by SUSTech CRA and CCSE