Commit 8283baba authored by nchernia's avatar nchernia
Browse files

adding user flag #106

parent 44752230

SLURM/scripts/juicer.sh

+64 −19
Changes for SLURM/scripts/juicer.sh: 64 added lines, 19 removed lines.
Original line number Diff line number Diff line
@@ -144,7 +144,7 @@ about="" 
nofrag=0



## Read arguments                                                     

usageHelp="Usage: ${0##*/} [-g genomeID] [-d topDir] [-q queue] [-l long queue] [-s site]\n                 [-a about] [-R end] [-S stage] [-p chrom.sizes path]\n                 [-y restriction site file] [-z reference genome file]\n                 [-C chunk size] [-D Juicer scripts directory]\n                 [-Q queue time limit] [-L long queue time limit] [-b ligation] [-t threads]\n                 [-r] [-h] [-x]"

usageHelp="Usage: ${0##*/} [-g genomeID] [-d topDir] [-q queue] [-l long queue] [-s site]\n                 [-a about] [-R end] [-S stage] [-p chrom.sizes path]\n                 [-y restriction site file] [-z reference genome file]\n                 [-C chunk size] [-D Juicer scripts directory]\n                 [-Q queue time limit] [-L long queue time limit] [-b ligation] [-t threads]\n                 [-A account name] [-r] [-h] [-x]"

genomeHelp="* [genomeID] must be defined in the script, e.g. \"hg19\" or \"mm10\" (default \n  \"$genomeID\"); alternatively, it can be defined using the -z command"

dirHelp="* [topDir] is the top level directory (default\n  \"$topDir\")\n     [topDir]/fastq must contain the fastq files\n     [topDir]/splits will be created to contain the temporary split files\n     [topDir]/aligned will be created for the final alignment"

queueHelp="* [queue] is the queue for running alignments (default \"$queue\")"
@@ -163,6 +163,7 @@ queueTimeHelp="* [queue time limit]: time limit for queue, i.e. -W 12:00 is 12 h 
longQueueTimeHelp="* [long queue time limit]: time limit for long queue, i.e. -W 168:00 is one week\n  (default ${long_queue_time})"

ligationHelp="* [ligation junction]: use this string when counting ligation junctions"

threadsHelp="* [threads]: number of threads when running BWA alignment"

userHelp="* [account name]: user account name on cluster"

excludeHelp="* -x: exclude fragment-delimited maps from hic file creation"

helpHelp="* -h: print this help and exit"
@@ -186,12 +187,13 @@ printHelpAndExit() { 
    echo -e "$longQueueTimeHelp"

    echo -e "$ligationHelp"

    echo -e "$threadsHelp"

    echo -e "$userHelp"

    echo "$excludeHelp"

    echo "$helpHelp"

    exit "$1"

}



while getopts "d:g:R:a:hrq:s:p:l:y:z:S:C:D:Q:L:b:t:x" opt; do

while getopts "d:g:R:a:hrq:s:p:l:y:z:S:C:D:Q:L:b:A:t:x" opt; do

    case $opt in

	g) genomeID=$OPTARG ;;

	h) printHelpAndExit 0;;
@@ -213,6 +215,7 @@ while getopts "d:g:R:a:hrq:s:p:l:y:z:S:C:D:Q:L:b:t:x" opt; do 
	x) nofrag=1 ;;

	b) ligation=$OPTARG ;;

	t) threads=$OPTARG ;;

	A) user=$OPTARG ;;

	[?]) printHelpAndExit 1;;

    esac

done
@@ -410,9 +413,17 @@ else 
    read1=${splitdir}"/*${read1str}*.fastq"

fi



if [ -z "$user" ]

then

    userstring=""

else

    userstring="#SBATCH -A $user"

fi



# Add header containing command executed and timestamp:

jid=`sbatch <<- HEADER | egrep -o -e "\b[0-9]+$"

	#!/bin/bash -l

	#!/bin/bash 

        $userstring

	#SBATCH -p $queue

	#SBATCH -t 2

	#SBATCH -c 1
@@ -480,13 +491,14 @@ then 
                if [ -z "$gzipped" ]

                then	

		    jid=`sbatch <<- SPLITEND | egrep -o -e "\b[0-9]+$"

			#!/bin/bash -l

			#!/bin/bash

                        #SBATCH -p $queue

			#SBATCH -t $queue_time

			#SBATCH -c 1

			#SBATCH -o $debugdir/split-%j.out

			#SBATCH -e $debugdir/split-%j.err

			#SBATCH -J "${groupname}_split_${i}"

                        $userstring			

			date

			echo "Split file: $filename"

			split -a 3 -l $splitsize -d --additional-suffix=.fastq $i $splitdir/$filename
@@ -494,13 +506,14 @@ then 
SPLITEND`

		else

		    jid=`sbatch <<- SPLITEND | egrep -o -e "\b[0-9]+$"

			#!/bin/bash -l

			#!/bin/bash

			#SBATCH -p $queue

			#SBATCH -t $queue_time

			#SBATCH -c 1

			#SBATCH -o $debugdir/split-%j.out

			#SBATCH -e $debugdir/split-%j.err

			#SBATCH -J "${groupname}_split_${i}"

                        $userstring			

			date

			echo "Split file: $filename"

			zcat $i | split -a 3 -l $splitsize -d --additional-suffix=.fastq - $splitdir/$filename
@@ -564,13 +577,15 @@ SPLITEND` 


	# count ligations

	jid=`sbatch <<- CNTLIG |  egrep -o -e "\b[0-9]+$"

		#!/bin/bash -l

		#!/bin/bash

		#SBATCH -p $queue

		#SBATCH -t $queue_time

		#SBATCH -c 1

		#SBATCH -o $debugdir/count_ligation-%j.out

		#SBATCH -e $debugdir/count_ligation-%j.err

		#SBATCH -J "${groupname}${jname}_Count_Ligation"

                $userstring			



		date

		export usegzip=${usegzip}; export name=${name}; export name1=${name1}; export name2=${name2}; export ext=${ext}; export ligation=${ligation}; ${juiceDir}/scripts/countligations.sh

		date
@@ -580,7 +595,7 @@ CNTLIG` 
	touchfile1=${tmpdir}/${jname}1



	jid=`sbatch <<- ALGNR1 | egrep -o -e "\b[0-9]+$"

		#!/bin/bash -l

		#!/bin/bash

		#SBATCH -p $queue

		#SBATCH -o $debugdir/align1-%j.out

		#SBATCH -e $debugdir/align1-%j.err
@@ -591,6 +606,8 @@ CNTLIG` 
		#SBATCH --mem-per-cpu=$alloc_mem

		#SBATCH -J "${groupname}_align1_${jname}"

		#SBATCH --threads-per-core=1		

                $userstring			



		${load_bwa}

		# Align read1

		date
@@ -625,7 +642,7 @@ ALGNR1` 
	# align read2 fastq

	touchfile2=${tmpdir}/${jname}2

	jid=`sbatch <<- ALGNR2 | egrep -o -e "\b[0-9]+$"

		#!/bin/bash -l

		#!/bin/bash

		#SBATCH -p $queue

		#SBATCH -o $debugdir/align2-%j.out

		#SBATCH -e $debugdir/align2-%j.err
@@ -636,6 +653,8 @@ ALGNR1` 
		#SBATCH --mem-per-cpu=$alloc_mem

		#SBATCH -J "${groupname}_align2_${jname}"

		#SBATCH --threads-per-core=1		

                $userstring			



		${load_bwa}

		date

		# Align read2
@@ -671,7 +690,7 @@ ALGNR2` 
	touchfile3=${tmpdir}/${jname}3

	# wait for top two, merge

	jid=`sbatch <<- MRGALL | egrep -o -e "\b[0-9]+$"

		#!/bin/bash -l

		#!/bin/bash

		#SBATCH -p $long_queue

		#SBATCH -o $debugdir/merge-%j.out

		#SBATCH -e $debugdir/merge-%j.err
@@ -682,6 +701,8 @@ ALGNR2` 
		#SBATCH -d $dependalign

		#SBATCH -J "${groupname}_merge_${jname}"

		#SBATCH --threads-per-core=1

                $userstring			



		export LC_COLLATE=C

		date

		if [ ! -f "${touchfile1}" ] || [ ! -f "${touchfile2}" ]
@@ -811,6 +832,8 @@ MRGALL` 
		#SBATCH -p $queue

		#SBATCH -J "${groupname}_check"

		#SBATCH -d $dependmerge

                $userstring			



		date

		echo "Checking $f"

		if [ ! -e $f ]
@@ -857,6 +880,8 @@ then 
		#SBATCH -c 8

		#SBATCH -J "${groupname}_fragmerge"

		${sbatch_wait}

                $userstring			



		date

		if [ -f "${errorfile}" ]

		then
@@ -908,7 +933,7 @@ then 
    # We keep the ID of this guard, so we can later alter dependencies of inner dedupping phase.

    # After dedup is done, this job will be released. 

    guardjid=`sbatch <<- DEDUPGUARD | egrep -o -e "\b[0-9]+$"

	#!/bin/bash -l

	#!/bin/bash

	#SBATCH -p $queue

	#SBATCH -o $debugdir/dedupguard-%j.out

	#SBATCH -e $debugdir/dedupguard-%j.err
@@ -918,6 +943,8 @@ then 
	#SBATCH --ntasks=1

	#SBATCH -J "${groupname}_dedup_guard"

	${sbatch_wait}

        $userstring			



	date

DEDUPGUARD`
@@ -925,7 +952,7 @@ DEDUPGUARD` 


    # if jobs succeeded, kill the cleanup job, remove the duplicates from the big sorted file

    jid=`sbatch <<- DEDUP | egrep -o -e "\b[0-9]+$"

	#!/bin/bash -l

	#!/bin/bash

	#SBATCH -p $queue

	#SBATCH --mem-per-cpu=2G

	#SBATCH -o $debugdir/dedup-%j.out
@@ -935,6 +962,8 @@ DEDUPGUARD` 
	#SBATCH --ntasks=1

	#SBATCH -J "${groupname}_dedup"

	${sbatch_wait}

        $userstring			



	date

        if [ -f "${errorfile}" ]

        then
@@ -958,7 +987,7 @@ DEDUP` 


    #Wait for all parts of split_rmdups to complete:

    jid=`sbatch <<- MSPLITWAIT | egrep -o -e "\b[0-9]+$"

	#!/bin/bash -l

	#!/bin/bash

	#SBATCH -p $queue

	#SBATCH -o $debugdir/post_dedup-%j.out

	#SBATCH -e $debugdir/post_dedup-%j.err
@@ -967,6 +996,8 @@ DEDUP` 
	#SBATCH --ntasks=1

	#SBATCH -J "${groupname}_post_dedup"

	#SBATCH -d ${dependguard}

        $userstring			



	date

	rm -Rf $tmpdir;

	find $debugdir -type f -size 0 | xargs rm
@@ -1000,6 +1031,8 @@ then 
	#SBATCH --ntasks=1

	#SBATCH -J "${groupname}_prep_done"     

	${sbatch_wait}

        $userstring			



	date

	export splitdir=${splitdir}; export outputdir=${outputdir}; export early=1; ${juiceDir}/scripts/check.sh

	date
@@ -1013,9 +1046,9 @@ if [ -z $postproc ] 
    then

    # Check that dedupping worked properly

    # in ideal world, we would check this in split_rmdups and not remove before we know they are correct

    awkscript='BEGIN{sscriptname = sprintf("%s/.%s_rmsplit.slurm", debugdir, groupname);}NR==1{if (NF == 2 && $1 == $2 ){print "Sorted and dups/no dups files add up"; printf("#!/bin/bash -l\n#SBATCH -o %s/dup-rm.out\n#SBATCH -e %s/dup-rm.err\n#SBATCH -p %s\n#SBATCH -J %s_msplit0\n#SBATCH -d singleton\n#SBATCH -t 1440\n#SBATCH -c 1\n#SBATCH --ntasks=1\ndate;\nrm %s/*_msplit*_optdups.txt; rm %s/*_msplit*_dups.txt; rm %s/*_msplit*_merged_nodups.txt;rm %s/split*;\ndate\n", debugdir, debugdir, queue, groupname, dir, dir, dir, dir) > sscriptname; sysstring = sprintf("sbatch %s", sscriptname); system(sysstring);close(sscriptname); }else{print "Problem"; print "***! Error! The sorted file and dups/no dups files do not add up, or were empty."}}'

    awkscript='BEGIN{sscriptname = sprintf("%s/.%s_rmsplit.slurm", debugdir, groupname);}NR==1{if (NF == 2 && $1 == $2 ){print "Sorted and dups/no dups files add up"; printf("#!/bin/bash\n#SBATCH -o %s/dup-rm.out\n#SBATCH -e %s/dup-rm.err\n#SBATCH -p %s\n#SBATCH -J %s_msplit0\n#SBATCH -d singleton\n#SBATCH -t 1440\n#SBATCH -c 1\n#SBATCH --ntasks=1\ndate;\nrm %s/*_msplit*_optdups.txt; rm %s/*_msplit*_dups.txt; rm %s/*_msplit*_merged_nodups.txt;rm %s/split*;\ndate\n", debugdir, debugdir, queue, groupname, dir, dir, dir, dir) > sscriptname; sysstring = sprintf("sbatch %s", sscriptname); system(sysstring);close(sscriptname); }else{print "Problem"; print "***! Error! The sorted file and dups/no dups files do not add up, or were empty."}}'

    jid=`sbatch <<- DUPCHECK | egrep -o -e "\b[0-9]+$"

	#!/bin/bash -l

	#!/bin/bash

	#SBATCH -p $queue

	#SBATCH -o $debugdir/dupcheck-%j.out

	#SBATCH -e $debugdir/dupcheck-%j.err
@@ -1025,6 +1058,7 @@ if [ -z $postproc ] 
	#SBATCH --mem-per-cpu=1G

	#SBATCH -J "${groupname}_dupcheck"

	${sbatch_wait}

        $userstring			



	date      

	ls -l ${outputdir}/merged_sort.txt | awk '{printf("%s ", \\\$5)}' > $debugdir/dupcheck-${groupname}
@@ -1034,7 +1068,7 @@ if [ -z $postproc ] 
DUPCHECK`

    sbatch_wait="#SBATCH -d afterok:$jid"

    jid=`sbatch <<- STATS | egrep -o -e "\b[0-9]+$"

		#!/bin/bash -l

		#!/bin/bash

		#SBATCH -p $long_queue

		#SBATCH -o $debugdir/stats-%j.out

		#SBATCH -e $debugdir/stats-%j.err
@@ -1044,6 +1078,7 @@ DUPCHECK` 
		#SBATCH --mem-per-cpu=6G

		#SBATCH -J "${groupname}_stats"

		${sbatch_wait}

                $userstring			



		date

		if [ -f "${errorfile}" ]
@@ -1075,7 +1110,7 @@ STATS` 
    dependstats="afterok:$jid"



    jid=`sbatch <<- HIC | egrep -o -e "\b[0-9]+$"

	#!/bin/bash -l

	#!/bin/bash

	#SBATCH -p $long_queue

	#SBATCH -o $debugdir/hic-%j.out

	#SBATCH -e $debugdir/hic-%j.err
@@ -1085,6 +1120,8 @@ STATS` 
	#SBATCH --mem-per-cpu=32G

	#SBATCH -J "${groupname}_hic"

	#SBATCH -d $dependstats

        $userstring			



	${load_java}

	export IBM_JAVA_OPTIONS="-Xmx48192m -Xgcthreads1"

	date
@@ -1105,7 +1142,7 @@ HIC` 
    dependhic="afterok:$jid"



    jid=`sbatch <<- HIC30 | egrep -o -e "\b[0-9]+$"

	#!/bin/bash -l

	#!/bin/bash

	#SBATCH -p $long_queue

	#SBATCH -o $debugdir/hic30-%j.out

	#SBATCH -e $debugdir/hic30-%j.err
@@ -1115,6 +1152,8 @@ HIC` 
	#SBATCH --mem-per-cpu=32G

	#SBATCH -J "${groupname}_hic30"

	#SBATCH -d ${dependstats}

        $userstring			



	${load_java}

	export IBM_JAVA_OPTIONS="-Xmx48192m -Xgcthreads1"

	date
@@ -1145,7 +1184,7 @@ then 
	sbatch_req="#SBATCH --gres=gpu:kepler:1"

    fi

    jid=`sbatch <<- HICCUPS | egrep -o -e "\b[0-9]+$"

	#!/bin/bash -l

	#!/bin/bash

	#SBATCH -p $queue

	#SBATCH --mem-per-cpu=2G

	${sbatch_req}
@@ -1155,6 +1194,8 @@ then 
	#SBATCH --ntasks=1

	#SBATCH -J "${groupname}_hiccups_wrap"

	${sbatch_wait}

        $userstring			



	${load_gpu}

	echo "load: $load_gpu"

	${load_java}
@@ -1174,7 +1215,7 @@ else 
fi



jid=`sbatch <<- ARROWS | egrep -o -e "\b[0-9]+$"

	#!/bin/bash -l

	#!/bin/bash

	#SBATCH -p $queue

	#SBATCH --mem-per-cpu=8G

	#SBATCH -o $debugdir/arrowhead_wrap-%j.out
@@ -1183,6 +1224,8 @@ jid=`sbatch <<- ARROWS | egrep -o -e "\b[0-9]+$" 
	#SBATCH --ntasks=1

	#SBATCH -J "${groupname}_arrowhead_wrap"

	${sbatch_wait}

        $userstring			



	${load_java}

	date

        if [ -f "${errorfile}" ]
@@ -1206,6 +1249,8 @@ jid=`sbatch <<- FINCLN1 | egrep -o -e "\b[0-9]+$" 
	#SBATCH --ntasks=1

	#SBATCH -J "${groupname}_prep_done"

	#SBATCH -d $dependarrows

        $userstring			



	date

	export splitdir=${splitdir}; export outputdir=${outputdir}; ${juiceDir}/scripts/check.sh

	date

CRA Git | Maintained and supported by SUSTech CRA and CCSE