trying parallelization of dup removal

# Juicer directory, contains scripts/, references/, and restriction_sites/

# can also be set in options via -D

juiceDir="/opt/juicer"

# defaults to where juicer.sh is located

juiceDir=/aidenlab

# top level directory, can also be set in options

topDir=$(pwd)

# restriction enzyme, can also be set in options

    touch ${outputdir}/dups.txt

    touch ${outputdir}/optdups.txt

    touch ${outputdir}/merged_nodups.txt

    if ! parallel --version >& /dev/null

    then

	awk -f ${juiceDir}/scripts/common/dups.awk -v name=${outputdir}/ ${outputdir}/merged_sort.txt

	# for consistency with cluster naming in split_rmdups

	mv ${outputdir}/optdups.txt ${outputdir}/opt_dups.txt 

    else # run in parallel if possible using GNU parallel

	rm -f ${outputdir}/cmds.txt #remove old commands file if exists

	if awk -v juicedir=${juiceDir} -v dir=$outputdir -v groupname=$groupname -v cmds=${outputdir}/cmds.txt -f ${juiceDir}/scripts/split_rmdups.awk $outputdir/merged_sort.txt

	then

	    if parallel < ${outputdir}/cmds.txt >& /dev/null

	    then

		cat ${outputdir}/*_msplit*_optdups.txt > ${outputdir}/opt_dups.txt

		cat ${outputdir}/*_msplit*_dups.txt > ${outputdir}/dups.txt

		cat ${outputdir}/*_msplit*_merged_nodups.txt > ${outputdir}/merged_nodups.txt

		sorttotal=`ls -l ${outputdir}/merged_sort.txt | awk '{print $5}'`

		nodupstotal=`ls -l ${outputdir}/merged_nodups.txt ${outputdir}/dups.txt ${outputdir}/opt_dups.txt | awk '{sum = sum + $5}END{print sum}'`

		if [ -z $sorttotal ] || [ -z $nodupstotal ] || [ $sorttotal -ne $nodupstotal ]

		then

		    echo "***! Error! The sorted file and dups/no dups files do not add up, or were empty. Merge or dedupping likely failed, restart pipeline with -S merge or -S dedup"

		    echo "Dups don't add up.  Check ${outputdir} for results"

		    exit 1

		else

		    rm ${outputdir}/*_msplit*_optdups.txt 

		    rm ${outputdir}/*_msplit*_dups.txt

		    rm ${outputdir}/*_msplit*_merged_nodups.txt

		    rm ${outputdir}/split*

		    echo "(-: Finished removing duplicates."

		fi

	    else

		echo "***! Some problems occurred somewhere in deduping the merged_sort file."

		exit 1

	    fi

	else

            echo "***! Some problems occurred somewhere in deduping the merged_sort file."

            exit 1

	fi

    fi

fi

if [ -z "$genomePath" ]

then

    #If no path to genome is give, use genome ID as default.

#    awk -f ${juiceDir}/scripts/common/dups.awk -v name=${outputdir}/ ${outputdir}/merged_sort.txt

    # for consistency with cluster naming in split_rmdups

    genomePath=$genomeID

fi

#CREATE HIC FILES

##########

#The MIT License (MIT)

#

# Copyright (c) 2015 Aiden Lab

#

# Permission is hereby granted, free of charge, to any person obtaining a copy

# of this software and associated documentation files (the "Software"), to deal

# in the Software without restriction, including without limitation the rights

# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

# copies of the Software, and to permit persons to whom the Software is

# furnished to do so, subject to the following conditions:

#

# The above copyright notice and this permission notice shall be included in

# all copies or substantial portions of the Software.

#

#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN

#  THE SOFTWARE.

##########

# Dedup script that submits deduping jobs after splitting at known 

# non-duplicates 

# Juicer version 1.5

BEGIN{

    tot=0;

    name=0;

    waitstring="";

}

{

    if (tot >= 1000000) {

	if (p1 != $1 || p2 != $2 || p4 != $4 || p5 != $5 || p8 != $8) {

	    sname=sprintf("%s_msplit%04d_", groupname, name);

	    dupname=dir"/"sname"dups.txt";

	    nodupname=dir"/"sname"merged_nodups.txt";

	    optname=dir"/"sname"optdups.txt";

	    sysstring = sprintf("touch %s %s %s", dupname,nodupname,optname);

	    system(sysstring);

	    printf("awk -f %s/scripts/common/dups.awk -v name=%s/%s %s/split%04d;\n", juicedir, dir, sname, dir, name, dir, name) >> cmds;

	    name++;

	    tot=0;

	}

    }

    outname = sprintf("%s/split%04d", dir, name);

    print > outname;

    p1=$1;p2=$2;p4=$4;p5=$5;p6=$6;p8=$8;

    tot++;

}

END {

    sname=sprintf("%s_msplit%04d_", groupname, name);

    dupname=dir"/"sname"dups.txt";

    nodupname=dir"/"sname"merged_nodups.txt";

    optname=dir"/"sname"optdups.txt";

    sysstring = sprintf("touch %s %s %s", dupname,nodupname,optname);

    system(sysstring);

    printf("awk -f %s/scripts/common/dups.awk -v name=%s/%s %s/split%04d;\n", juicedir, dir, sname, dir, name, dir, name) >> cmds;

}