Commit 89fbc4e6 authored by Aiden Lab's avatar Aiden Lab Committed by GitHub
Browse files

Merge pull request #37 from mzhibo/master 

bug fix: potential job dependency failure caused by jobID
parents d1c95feb 3eb7ecb1

PBS/scripts/juicer.sh→PBS/juicer.sh

+17 −19
Changes for PBS/juicer.sh: 17 added lines, 19 removed lines.
Original line number Diff line number Diff line
@@ -99,7 +99,7 @@ long_walltime="walltime=120:00:00" 
# give your email address to be used in #PBS -M to receive notifications when job error occurs.

# Must be either set with an email address or skipped

# This email is not included in the launch stat and postprocessing steps, add manually if needed

EMAIL='#PBS -M mzhibo@gmail.com'

EMAIL='#PBS -M xxx@gmail.com'

splitsize=90000000

# fastq files should look like filename_R1.fastq and filename_R2.fastq 

# if your fastq files look different, change this value
@@ -510,7 +510,7 @@ SPLITMV 
        export ligation=${ligation}

        ${juiceDir}/scripts/countligations.sh

CNTLIG

        jID_cntlig=\$(qstat | grep "CtLig\${countjobs}${groupname}" | cut -c 1-7 )

        jID_cntlig=\$(qstat | grep "CtLig\${countjobs}${groupname}" | cut -d ' ' -f 1 )

        echo "jID_cntlig \${countjobs} id is \${jID_cntlig}"

        ## Align read1        

        # align read1 fastq
@@ -573,7 +573,7 @@ CNTLIG 
ALGNR1

        wait

        # Get the jobID from qstat ouput by searching job specific string,"read1\${countjobs}" , in job Name.

        jID_1=\$( qstat | grep "ALN1\${countjobs}${groupname}" |cut -c 1-7 )

        jID_1=\$( qstat | grep "ALN1\${countjobs}${groupname}" |cut -d ' ' -f 1 )

        echo "align1 \$countjobs id is: \${jID_1}"

        # Align read2

        echo "starting read2 alignment"
@@ -625,7 +625,7 @@ ALGNR2 
        wait



        # Get the jobID of job above from qstat ouput using job specific string,"read2\${countjobs}" , in jobName.

        jID_2=\$(qstat | grep "ALN2\${countjobs}${groupname}" |cut -c 1-7 )

        jID_2=\$(qstat | grep "ALN2\${countjobs}${groupname}" |cut -d ' ' -f 1 )

        echo "align2 \${countjobs} id is: \${jID_2}"

        echo "starting merging from read1 and read2"

        # wait for align1 and align2 jobs finish,then merge
@@ -702,7 +702,7 @@ ALGNR2 
MRGALL

    wait



    jID_3=\$(qstat | grep "Mrg\${countjobs}${groupname}" | cut -c 1-7 )

    jID_3=\$(qstat | grep "Mrg\${countjobs}${groupname}" | cut -d ' ' -f 1 )

    echo "merging align1 and align2 \${coutjobs} id is \${jID_3}"

    echo "starting chimeric step after alignment"

    timestamp=\$(date +"%s" | cut -c 4-10)
@@ -764,7 +764,7 @@ MRGALL 
CHIMERIC

    wait



    jID_4=\$(qstat | grep "Chmr\${countjobs}${groupname}" | cut -c 1-7)

    jID_4=\$(qstat | grep "Chmr\${countjobs}${groupname}" | cut -d ' ' -f 1)

    echo "chimeric \$countjobs id is \$jID_4"

    exitstatus=\$(qstat -f \${jID_4} |grep "exit_status" )

    echo "the exit status of \{jID_4} is \${exitstatus}"
@@ -822,7 +822,7 @@ ALIGNWRAP 
    

fi

wait    

jID_alignwrap=$( qstat | grep AlnWrp${groupname} | cut -c 1-7 )

jID_alignwrap=$( qstat | grep AlnWrp${groupname} | cut -d ' ' -f 1 )

if [ -z $merge ]

then

    waitstring_mrgsrtwrp="#PBS -W depend=afterok:${jID_alignwrap}"
@@ -846,11 +846,10 @@ then 
    #PBS -o ${logdir}/${timestamp}_mergesortwrap_${groupname}.log

    #PBS -j oe

    #PBS -N MStWrp${groupname}

#    #PBS -W depend=afterok:${jID_alignwrap}

    ${waitstring_mrgsrtwrp}

    date +"%Y-%m-%d %H:%M:%S"

    echo "all alignment done, all aplitting and alignment jobs succeeded!" 

    jID_alnOK=\$( qstat | grep AlnOK_${groupname} | cut -c 1-7 )

    jID_alnOK=\$( qstat | grep AlnOK_${groupname} | cut -d ' ' -f 1 )

    echo "jID_aln-OK job id is \$jID_alnOK "

    timestamp=\$(date +"%s" | cut -c 4-10)

    if [ -z $merge ]
@@ -873,7 +872,6 @@ then 
        #PBS -o ${logdir}/\${timestamp}_fragmerge_${groupname}.log

        #PBS -j oe

        #PBS -N frgmrg${groupname}

#        #PBS -W depend=afterok:\${jID_alnOK}

        \${waitstring_alnOK}

        date +"%Y-%m-%d %H:%M:%S"

        export LC_ALL=C
@@ -889,7 +887,7 @@ then 
            rm -r ${tmpdir}        

        fi

MRGSRT

        jID_mrgsrt=\$( qstat | grep frgmrg${groupname} | cut -c 1-7 )

        jID_mrgsrt=\$( qstat | grep frgmrg${groupname} | cut -d ' ' -f 1 )



        ##kill all remaining jobs if previous mergesort step exited with error

        timestamp=\$(date +"%s" | cut -c 4-10)
@@ -908,13 +906,13 @@ MRGSRT 


        date +"%Y-%m-%d %H:%M:%S"

        echo "Error with merging sorted files job, ${jID_mrgsort}, deleting all remaining jobs of this pipeline."

        RemJob1=\$(qstat |grep "$groupname" |grep " Q \| H \| R " | awk 'BEGIN{FS=" "}{print $1}'| cut -c 1-7)

        RemJob1=\$(qstat |grep "$groupname" |grep " Q \| H \| R " | awk 'BEGIN{FS=" "}{print $1}'| cut -d ' ' -f 1)

        qdel \${RemJob1}

MRGSRTFAILCK

MRGSRTWRAP

fi

wait

jID_mrgsrtwrap=$( qstat| grep MStWrp${groupname} | cut -c 1-7 )

jID_mrgsrtwrap=$( qstat| grep MStWrp${groupname} | cut -d ' ' -f 1 )

if [ -z $dedup ]

then

    waitstring_RDpWrp="#PBS -W depend=afterok:${jID_mrgsrtwrap}"
@@ -940,7 +938,7 @@ then 
    ${waitstring_RDpWrp}

    date +"%Y-%m-%d %H:%M:%S"

    echo ${waitstring_RDpWrp}

    jID_mrgsrt=\$( qstat | grep frgmrg${groupname} | cut -c 1-7 )

    jID_mrgsrt=\$( qstat | grep frgmrg${groupname} | cut -d ' ' -f 1 )

    if [ -z $dedup ]

    then

        waitstring_osplit="#PBS -W depend=afterok:\${jID_mrgsrt}"
@@ -970,7 +968,7 @@ RMDUPLICATE 
RMDUPWRAP

fi



jID_rmdupwrap=$( qstat | grep RDpWrp${groupname} | cut -c 1-7 )

jID_rmdupwrap=$( qstat | grep RDpWrp${groupname} | cut -d ' ' -f 1 )

echo "jID_rmdupwrap ID: $jID_rmdupwrap"

wait

if [ -z "$genomePath" ]
@@ -1028,7 +1026,7 @@ then 
        ${juiceDir}/scripts/launch_stats.sh

SUPERWRAP1

    fi

    jID_superwrap1="$( qstat | grep SpWrp1${groupname} | cut -c 1-7 )"

    jID_superwrap1="$( qstat | grep SpWrp1${groupname} | cut -d ' ' -f 1 )"

    wait

    if [ -z $postproc ]

    then
@@ -1063,7 +1061,7 @@ SUPERWRAP1 
    export walltime=$walltime

    export long_walltime=$long_walltime

    export postproc=$postproc

    jID_launch=\$(qstat | grep Lnch_${groupname} | cut -c 1-7)

    jID_launch=\$(qstat | grep Lnch_${groupname} | cut -d ' ' -f 1)

    echo \$jID_launch

    echo "waitstring3 is \${waitstring3}"

    ${juiceDir}/scripts/postprocessing.sh
@@ -1089,8 +1087,8 @@ else 
    #PBS -W depend=afterok:${jID_rmdupwrap}

    date +"%Y-%m-%d %H:%M:%S"



    jID_osplit=\$( qstat | grep osplit${groupname} | cut -c 1-7 )

    jID_rmsplit=\$( qstat | grep RmSplt${groupname} | cut -c 1-7)        

    jID_osplit=\$( qstat | grep osplit${groupname} | cut -d ' ' -f 1 )

    jID_rmsplit=\$( qstat | grep RmSplt${groupname} | cut -d ' ' -f 1)        

    wait

    timestamp=\$(date +"%s" | cut -c 4-10)

    qsub <<PREPDONE

PBS/scripts/launch_stats.sh→PBS/launch_stats.sh

+5 −5
Changes for PBS/launch_stats.sh: 5 added lines, 5 removed lines.
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ echo "resolutions: $resolutions" 
echo "walltime: $walltime"

echo "long_walltime: $long_walltime"

echo $load_java

jID_osplit=$( qstat | grep osplit${groupname} | cut -c 1-7 )

jID_osplit=$( qstat | grep osplit${groupname} | cut -d ' ' -f 1 )



if [ -z $final ]

then
@@ -41,8 +41,8 @@ ${waitstring2} 


date +"%Y-%m-%d %H:%M:%S"

echo "Alignment and merge done, launching the stats job"

#jID_osplit=\$( qstat | grep osplit${groupname} | cut -c 1-7 )

jID_rmsplit=\$( qstat | grep RmSplt${groupname} | cut -c 1-7)

#jID_osplit=\$( qstat | grep osplit${groupname} | cut -d ' ' -f 1 )

jID_rmsplit=\$( qstat | grep RmSplt${groupname} | cut -d ' ' -f 1)



if [ -z $final ]

then
@@ -118,7 +118,7 @@ qsub <<- ABNORMAL 
	awk -f ${juiceDir}/scripts/collisions.awk $outputdir/abnormal.sam > $outputdir/collisions.txt

ABNORMAL



jID_stats0=\$( qstat | grep stats0${groupname} | cut -c 1-7)

jID_stats0=\$( qstat | grep stats0${groupname} | cut -d ' ' -f 1)



wait

echo "this is the value of jID_stats: \${jID_stats}"
@@ -146,7 +146,7 @@ qsub <<- HICWORK 
		${juiceDir}/scripts/juicer_tools pre -f $site_file -s $outputdir/inter.txt -g $outputdir/inter_hists.m -q 1 $outputdir/merged_nodups.txt $outputdir/inter.hic $genomePath

	fi

HICWORK

jID_stats30=\$( qstat | grep stats30${groupname} | cut -c 1-7)

jID_stats30=\$( qstat | grep stats30${groupname} | cut -d ' ' -f 1)

timestamp=\$(date +"%s" | cut -c 4-10)

echo "start submitting hic30 job"

qsub <<- HIC30WORK

PBS/scripts/postprocessing.sh→PBS/postprocessing.sh

+8 −8
Changes for PBS/postprocessing.sh: 8 added lines, 8 removed lines.
Original line number Diff line number Diff line 
#!/bin/bash

jID_launch=$( qstat | grep Lnch_${groupname} | cut -c 1-7 )

jID_launch=$( qstat | grep Lnch_${groupname} | cut -d ' ' -f 1 )

if [ -z $postproc ]

then

    waitstring3="#PBS -W depend=afterok:${jID_launch}"
@@ -20,7 +20,7 @@ ${waitstring3} 


date +"%Y-%m-%d %H:%M:%S"



jID_hic30=\$(qstat | grep hic30_${groupname} |cut -c 1-7)

jID_hic30=\$(qstat | grep hic30_${groupname} |cut -d ' ' -f 1)



if [ -z $postproc ]

then
@@ -52,7 +52,7 @@ qsub <<POSTPROCESS 
POSTPROCESS

POSTPROCWRAP



jID_postprocwrap=$( qstat |grep PPrWrp${groupname} | cut -c 1-7 )

jID_postprocwrap=$( qstat |grep PPrWrp${groupname} | cut -d ' ' -f 1 )

echo $jID_postprowrap

wait

timestamp=$(date +"%s" | cut -c 4-10)
@@ -66,11 +66,11 @@ qsub <<- FINCK 
#PBS -W depend=afterok:${jID_postprocwrap}



date +"%Y-%m-%d %H:%M:%S"    

jID_hic30=\$(qstat | grep hic30_${groupname} |cut -c 1-7)

jID_stats0=\$(qstat | grep stats0${groupname} |cut -c 1-7)

jID_stats30=\$(qstat | grep stats30${groupname} |cut -c 1-7)

jID_hic=\$(qstat | grep hic0_${groupname} |cut -c 1-7)

jID_postproc=\$(qstat | grep PProc_${groupname} |cut -c 1-7)

jID_hic30=\$(qstat | grep hic30_${groupname} |cut -d ' ' -f 1)

jID_stats0=\$(qstat | grep stats0${groupname} |cut -d ' ' -f 1)

jID_stats30=\$(qstat | grep stats30${groupname} |cut -d ' ' -f 1)

jID_hic=\$(qstat | grep hic0_${groupname} |cut -d ' ' -f 1)

jID_postproc=\$(qstat | grep PProc_${groupname} |cut -d ' ' -f 1)



waitstring5="#PBS -W depend=afterok:\${jID_postproc}"

if [ -z $postproc ]

CRA Git | Maintained and supported by SUSTech CRA and CCSE