#!/usr/bin/env perl

# This code derived from the MOCAT analysis pipeline
# Code is (c) Copyright EMBL and BGI, 2012-2013
# This code is released under GNU GPL v3.

use strict;
use warnings;
use Getopt::Long;
use File::Basename;
use File::Path qw(make_path);
use Cwd 'abs_path';
use MIME::Base64 'decode_base64';
use Pod::Usage 'pod2usage';

#use Carp 'verbose';
#$SIG{__DIE__} = sub { Carp::confess( @_ ) };


my @samples = ();
my $sample_file;
my $temp_dir = '.';
chomp(my $basedir = `pwd`);
my $cwd = $basedir;
my $INPUT_FORMAT = 'auto';
my $processors = 1;
my $calculateTaxonomy_previous_calc_coverage_stats_file = 0;
my $version = '1.1.1';
my $length_cutoff = 45;
my $identity_cutoff = 97;
my $quality_cutoff = 20;
my $prevalence_minimum = 2;
my $verbose = 0;
my $OUTPUT_FOLDER = 'RESULTS';

my $src_dir = './motus_data/src';
my $bin_dir = './motus_data/bin';
my $data_dir = './motus_data/data';

my %conf = ();
$conf{MOCAT_data_type} = 'solexaqa';
$conf{MOCAT_mapping_mode} = 'allbest';
$conf{screen_soap_cmd} = '-M 4';
$conf{screen_soap_seed_length} = 30;
$conf{screen_soap_max_mm} = 10;
$conf{filter_psort_buffer} = "2G";
$conf{MOCAT_rownames_dir} = './rownames/';

my $IS_PAIRED_END = -1;


my $systemType = `uname -s`;
chomp $systemType;

my $zcat = 'zcat';
if ( $systemType =~ m/Darwin/ ) { $zcat = 'gzcat'; }

# CORE
sub expand_data {
    my $basedir = shift;
    print("$0 will expand needed dependencies into ./motus_data....\n");
    open EXPAND, "|tar xzf - --directory $basedir"
        or die("Cannot open tar process");
    while (<DATA>) {
        print EXPAND decode_base64($_);
    }
    close(EXPAND);
}


sub mkdir_or_die {
    my $dir = shift;
    (-d $dir)
        or make_path($dir)
        or die("Could not create $dir: $!");
}

sub system_ {
    my $cmd = shift;
    if ($verbose) {
        print "Will execute \n\n$cmd\n\n";
    }
    (system($cmd) == 0) or die("Command '$cmd' failed: $!");
}

sub check_pairs {
    my $sample = shift;
    my @fqs = glob "$basedir/$sample/*.fq $basedir/$sample/*.fq.gz";
    foreach (@fqs) {
        if (/\.[12]\.fq(\.gz)$/) {
            return 1;
        }
    }
    return 0;
}

sub read_samples {
    if (!$sample_file) {
        my @inputs = ();
        if ($ARGV[0]) {
            (-f $ARGV[0])
                or usage(1);
            (!$ARGV[1]) or (-f $ARGV[1])
                or usage(1);
            for my $i (1..32768) {
                my $dir = "motus.processing.$i";
                if (-d $dir) { next }
                $sample_file = $dir;
                my $gz = '';
                if ($ARGV[0] =~ /\.gz$/) { $gz = ".gz"; }
                my $dot1 = '';
                my $dot2 = '';
                if ($ARGV[1]) {
                    $dot1 = ".1";
                    $dot2 = ".2";
                }
                mkdir_or_die($dir);
                @samples = ($dir);
                my $arg0abs = abs_path($ARGV[0]);
                system_("ln -s $arg0abs $dir/lane1$dot1.fq$gz");
                $IS_PAIRED_END = 0;
                if ($ARGV[1]) {
                    $IS_PAIRED_END = 1;
                    my $arg1abs = abs_path($ARGV[1]);
                    system_("ln -s $arg1abs $dir/lane1$dot2.fq$gz");
                }
                last;
            }
        } else {
            usage(1);
        }
    } else {
        (-s $sample_file)
            or die("ERROR & EXIT: Sample file missing or incorrect (was '$sample_file'). Specify sample file using --sample_file\n");
        open( SAMPLE, '<', $sample_file) or die("Cannot open sample file ($sample_file): $!");
        while (<SAMPLE>) {
            chomp $_;
            unless ( $_ =~ m/^$/ ) {
                if ( $_ =~ m/^\S+$/ ) {
                    $_ =~ s/ //g;
                    (-d $_)
                        or die("Sample $_ does not seem to be a directory.");
                    push( @samples, $_ );
                    my $pair_endiness = check_pairs($_);
                    if ($IS_PAIRED_END == -1) { $IS_PAIRED_END = $pair_endiness; }
                    elsif ($IS_PAIRED_END != $pair_endiness) {
                        die("All samples must be either single end or all samples must be paired-end.");
                    }
                }
            }
        }
    }
}


sub detect_format {
    # format = detect_format(sample)
    #
    #
    # Returns one of 's' (sanger), 'x' (solexa), or 'i' (solill)
    my $lane = shift;
    if ($lane =~ /\.gz$/) {
        open(LANE, "gunzip -c $lane|")
            or die("Cannot open gunzip pipe to read '$lane': $!");
    } else {
        open(LANE, '<', $lane)
            or die("Cannot open '$lane': $!");
    }
    my $NR = 0;
    my $is_solexa = 0;
    my $is_solill = 0;
    while (<LANE>) {
        ++$NR;
        chomp;
        if ( ($NR % 4) == 0 ) {
            if (/[!-:]/) { return 's' }
            elsif (/[;-?]/) { $is_solexa = 1; }
            elsif (/[@-h]/) { $is_solill = 1; }
        }
        if ($NR == 1000) { last }
    }
    close(LANE);
    if ($is_solexa) { return 'x'; }
    if ($is_solill) { return 'i'; }

    die "\nERROR & EXIT: Unknown format of files in sample $lane. Please use the --fastq-format argument to $0.\n";
}


# READ TRIM FILTER

sub read_trim_filter {
	foreach my $sample (@samples) {
        &mkdir_or_die("$temp_dir/$sample/temp");
        &mkdir_or_die("$basedir/$sample/stats");
        &mkdir_or_die("$basedir/$sample/reads.processed.solexaqa");

        my $file_formats_array = '';
        my @files = glob("$sample/*.fq $sample/*.fq.gz");
        foreach my $lane (@files) {
            if ($INPUT_FORMAT eq 'sanger') { $file_formats_array .= 's'; }
            elsif ($INPUT_FORMAT eq 'auto') {
                $file_formats_array .= &detect_format($lane);
            } else { $file_formats_array .= 'i'; }
        }
        my $trim5prime = 'yes';

        my $is_paired_end_str = ($IS_PAIRED_END ? 'yes' : 'no');
		my $cmd = "$src_dir/MOCATReadTrimFilter_aux.pl " .
                    "-sample $sample " .
                    "-trim_5prime_end $trim5prime " .
                    "--solexaqa_or_fastx solexaqa " .
                    "-paired_end_data $is_paired_end_str " .
                    " -length_cutoff $length_cutoff " .
                    "-qual_cutoff $quality_cutoff " .
                    "-src_dir $src_dir " .
                    "-bin_dir $bin_dir " .
                    "-cwd $basedir " .
                    "-temp_dir $temp_dir " .
                    "-zcat $zcat " .
                    "-use_5prime_file no " .
                    "-file_formats_array $file_formats_array";
        system_($cmd);
	}
}

sub check_index {
    my $ref = shift;
    if (-e "$data_dir/$ref.index.ann") { return; }
    my $bin_dir = abs_path($bin_dir);
    system_("$bin_dir/2bwt-builder $data_dir/$ref");
}

sub screen {
    my $database = shift;
    my $reads = shift;

    check_index($database);
    my @screen = ($database);

    my $mapping_mode;
	if ( $conf{MOCAT_mapping_mode} eq 'unique' ) {
		$mapping_mode = "-r 0";
	}
	elsif ( $conf{MOCAT_mapping_mode} eq 'random' ) {
		$mapping_mode = "-r 1";
	}
	elsif ( $conf{MOCAT_mapping_mode} eq 'allbest' ) {
		$mapping_mode = "-r 2";
	}
	else {
		die "ERROR & EXIT: Unknown MOCAT_mapping_mode";
	}
	foreach my $sample (@samples) {
        mkdir_or_die("$temp_dir/$sample/temp");
        mkdir_or_die("$temp_dir/$sample/stats");
        my $inputfile = "$temp_dir/$sample/temp/SOAP_INPUT.gz";
        if ( $systemType =~ m/Darwin/ ) {
            system_("$zcat $cwd/$sample/$reads/*pair*gz $cwd/$sample/$reads/*single*gz > $inputfile");
        } else {
            system_("cat $cwd/$sample/$reads/*pair*gz $cwd/$sample/$reads/*single*gz > $inputfile");
        }

		foreach my $screen (@screen) {
			# Define variables
            my $db_on_db    = $screen;
			my $sf          = "$cwd/$sample/reads.screened.$db_on_db.$conf{MOCAT_data_type}";
			my $ef          = "$cwd/$sample/reads.extracted.$db_on_db.$conf{MOCAT_data_type}";
			my $mf          = "$cwd/$sample/reads.mapped.$db_on_db.$conf{MOCAT_data_type}";
			my $file_output = "$mf/$sample.mapped.$reads.on.$screen.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}";
			my $screen_index    = "$data_dir/$screen.index";
			my $ids_file    = "$mf/$sample.aligned.$reads.on.$database.ids";
			my $stats_file  = "$cwd/$sample/stats/$sample.reads.screened.$db_on_db.$conf{MOCAT_data_type}.stats";
			my $estats_file = "$cwd/$sample/stats/$sample.reads.extracted.$db_on_db.$conf{MOCAT_data_type}.stats";

			# Get lane IDs
			my @lanes = glob("$cwd/$sample/$reads/*pair.1.fq.gz");
			foreach my $i ( 0 .. ( scalar @lanes - 1 ) ) {
				chomp( $lanes[$i] );
                $lanes[$i] = File::Basename::basename($lanes[$i]);
				$lanes[$i] =~ s/.pair.1.fq.gz//;
			}

			# Get lanes in the sample folder, for fasta file screen
			chomp( my @fqs = `ls -1 $cwd/$sample/*.fq $cwd/$sample/*.fq.gz 2>/dev/null | grep -v 'trimmed\.filtered' | grep -v '\.single\.fq' | grep -v '\.pair\.'| grep -v "\\.2\\.fq"` );
			foreach my $i ( 0 .. ( scalar @fqs - 1 ) ) {
				chomp( $fqs[$i] );
				$fqs[$i] =~ s/$cwd\/$sample\///;
			}

            mkdir_or_die($mf);

            my $cmd =
                "$bin_dir/soap2.21 " .
                "-a $inputfile " .
                "-o $file_output.soap.tmp " .
                "-D $screen_index $mapping_mode " .
                "$conf{screen_soap_cmd} " .
                "-l " . $conf{screen_soap_seed_length} .
                " -v " . $conf{screen_soap_max_mm} .
                " -p $processors";
            system_($cmd);
            (-e "$file_output.soap.tmp") or die("SOAP failed");
            system_("mv $file_output.soap.tmp $file_output.soap");
            mkdir_or_die($sf);
            mkdir_or_die($ef);

            open(INPUT, "cat $file_output.soap | perl $src_dir/MOCATFilter_remove_in_padded.pl -format SOAP -db $data_dir/$database -step screen -ids $ids_file |")
                or die("Cannot open soap file '$file_output.soap': $!");
            my %inserts;
            while (<INPUT>) {
                chomp;
                my @F = split "\t";
                my $len = $F[5];
                my $mm = $F[-1] =~ tr/[a-zA-Z]/[a-zA-Z]/;
                my $as = 100-($mm/$len)*100;
                if ($as >= $identity_cutoff && $len >= $length_cutoff) {
                    $F[0] =~ m#(.+)/[12]#;
                    $inserts{$1}++;
                }
            }
            close(INPUT);
            open(OUTPUT, '>', $ids_file)
                or die("Cannot open '$ids_file' for writing: $!");
            foreach my $i (keys %inserts) {
                print OUTPUT "$i/1\n$i/2\n";
            }
            close(OUTPUT);

            $cmd =
			    "$src_dir/MOCATScreen_filter.pl " .
                "-zip gzip " .
                "-ziplevel 6 " .
			    " -in ";
			foreach my $lane (@lanes) {
				$cmd .= " $cwd/$sample/$reads/$lane ";
			}
            $cmd .= " -out ";
            foreach my $lane (@lanes) {
                    $cmd .= " $sf/$lane.screened.$screen ";
                }
			$cmd .= " -ex ";
            foreach my $lane (@lanes) {
				$cmd .= " $ef/$lane.extracted.$screen ";
            }
            $cmd .=
			    " -toremove $ids_file " .
                "-stats $stats_file " .
                "-estats $estats_file " .
                "-identity $identity_cutoff " .
                "-length $length_cutoff " .
                "-soapmaxmm $conf{screen_soap_max_mm}";

            system_($cmd);
			# If saving in SAM format
            system_("$src_dir/MOCATExternal_soap2sam.pl < $file_output.soap | gzip > $file_output.sam.gz ");

			# Zip soap file
            system_("gzip -f $file_output.soap");
            unlink("$ids_file");

		}
        unlink($inputfile);
	}    # End, each sample
}

sub filter {
    my $database = shift;
    my $reads = shift;

	foreach my $sample (@samples) {
        &mkdir_or_die("$temp_dir/$sample/temp");

		my $tosort_file = "$temp_dir/$sample/temp/tosort.tmp";
        my $stats = "$cwd/$sample/stats/$sample.$reads.stats";
        if ($reads eq 'reads.processed.solexaqa') {
            $stats = "$cwd/$sample/stats/$sample.readtrimfilter.solexaqa.stats";
        }
        (-e $stats)
            or die("Missing stats file '$stats'.");

        my $len_file = "$temp_dir/$sample/temp/lengths.tmp";
		system_("cat $data_dir/$database.len | sort -u > $len_file");
        my $output_folder = "$cwd/$sample/reads.filtered.$database.$conf{MOCAT_data_type}";
        my $output_file   = "$output_folder/$sample.filtered.$reads.on.$database.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff";

        if (!(-e "$data_dir/$database.len") ) {
            system_("$src_dir/MOCATFilter_falen.pl -infile $data_dir/$database -outfile $data_dir/$database.len");
        }

        my $input_folder = "$cwd/$sample/reads.mapped.$database.solexaqa";
        my $file = "$sample.mapped.$reads.on.$database.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.soap.gz";
        ( -e "$input_folder/$file" )
            or die "\nERROR & EXIT: Missing $input_folder/$file";
        (-e $len_file)
            or die("Length file '$len_file' missing.\n");
        my $input_file = "$input_folder/$file";
        mkdir_or_die("$temp_dir/$sample/temp/psort");
        mkdir_or_die("$output_folder");
        system_(
            "$zcat $input_file | " .
            "perl $src_dir/MOCATFilter_remove_in_padded.pl -format SOAP -db $data_dir/$database -step filter | " .
            "$src_dir/MOCATFilter_soap2sam.awk " .
            "-v MIN_LEN=$length_cutoff " .
            "-v MIN_AS=$identity_cutoff " .
            "> $tosort_file ");
        my $cmd =
            "LC_ALL=C " .
                "sort " .
                #"--parallel=$processors " .
                "--buffer-size=$conf{filter_psort_buffer} " .
                "--temporary-directory=$temp_dir/$sample/temp/psort  " .
                $tosort_file;
        if ($IS_PAIRED_END) {
            $cmd .= "| $src_dir/MOCATFilter_filterPE.pl ";
        }
        $cmd .=
			"| $bin_dir/msamtools " .
                "-S -m filter " .
                "-l $length_cutoff " .
                "-p $identity_cutoff " .
                "-z 0 " .
                "--besthit " .
                "-t $len_file " .
                "- | $src_dir/MOCATFilter_stats.pl " .
                "-format SAM " .
                "-length $length_cutoff " .
                "-identity $identity_cutoff " .
                "-stats $stats " .
                "| $bin_dir/msamtools " .
                    "-Sb -m merge " .
                    "-t $len_file " .
                    "- > $output_file.bam.tmp";
        system_($cmd);
        unlink($tosort_file);
        (-e "$output_file.bam.tmp") or die("Failed to create $output_file.bam.tmp");
        system_("mv $output_file.bam.tmp $output_file.bam");
        unlink($len_file);
        unlink($input_file);
	}    # End loop samples

}

sub profile {
    my $taxo_profiling_mode = shift;
    my $reads = shift;
    my $database = shift;

    my $rownames_dir = $conf{MOCAT_rownames_dir};
    &mkdir_or_die($rownames_dir);
    $rownames_dir = abs_path($rownames_dir);
    my $sample_file_basename = chomp($sample_file);

    my $taxo_profiling_map            = "$data_dir/$database.refmg.map";
    my $taxo_profiling_map_tot_len    = "$data_dir/$database.refmg.map.total_length";
    my $taxo_profiling_motu_map       = "$data_dir/$database.motu.map";
    my $taxo_profiling_functional_map = "$data_dir/$database.functional.map";
    my $databases = $database;
	my $profile_type;

	if ( $taxo_profiling_mode eq 'RefMG' ) {
		$profile_type = "taxonomic";
	}
	elsif ( $taxo_profiling_mode eq 'mOTU' ) {
		$taxo_profiling_map = $taxo_profiling_motu_map;
		$profile_type       = "motu";
	} else {
        die("BUG\n");
	}

    ( -e "$taxo_profiling_map" )
        or die("\nERROR & EXIT: Missing map file $taxo_profiling_map");

    my $counter = -1;
    foreach my $sample (@samples) {
        ++$counter;
        mkdir_or_die("$temp_dir/$sample/temp");
        my $output              = "$sample.filtered.$reads.on.$databases.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff";
        my $outputTax           = "$sample.$profile_type.profile.$reads.on.$databases.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff";
        my $outputTaxRownames   = "$sample_file_basename.$profile_type.profile.$reads.on.$databases.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff";
        my $folder_filtered     = "$cwd/$sample/reads.filtered.$databases.$conf{MOCAT_data_type}";
        my $fullname = "NOFILE";

        my $folder_tax      = "$profile_type.profiles.$databases.$conf{MOCAT_data_type}";
        my $input;
        if ( $conf{MOCAT_mapping_mode} eq "allbest" ) {
            $input = "$folder_filtered/$output.bam";
        }
        elsif ($conf{MOCAT_mapping_mode} eq "unique"
            || $conf{MOCAT_mapping_mode} eq "random" )
        {
            $input = "$folder_filtered/$output.soap.gz";
        } else {
            die("Unknown MOCAT_mapping_mode: '$conf{MOCAT_mapping_mode}'.");
        }
        (-e $input)
            or die("\nERROR & EXIT: Missing filtered mapping results file: $input");
        my $inserts;
        my $padded_stats_file;
        if ($calculateTaxonomy_previous_calc_coverage_stats_file) {
            $inserts = "$cwd/$sample/stats/$sample.extracted.screened.reads.processed.solexaqa.on.mOTU.v1.padded.after.PE.filter.and.within.padded.region.solexaqa";
        }
        else {
            $inserts = "$cwd/$sample/stats/$sample.readtrimfilter.$conf{MOCAT_data_type}";
        }
        (-e "$inserts.stats")
            or die("ERROR & EXIT: Missing stats file: $inserts.stats");
        my $covfile = "$cwd/$sample/stats/$sample.coverage.$reads.on.$databases.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff.stats";
        
        $padded_stats_file = "$cwd/$sample/stats/$sample.extracted.screened.$reads.on.$databases.after.PE.filter.and.within.padded.region.$conf{MOCAT_data_type}.stats";
        if ( $reads eq 'reads.processed' ) {
        	$padded_stats_file = "$cwd/$sample/stats/$sample.extracted.$databases.after.PE.filter.and.within.padded.region.$conf{MOCAT_data_type}.stats";
        }
        
        
        my $mmfile = "$cwd/$sample/stats/$sample.coverage.$reads.on.$databases.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff.multiplemapper.stats";
		my $folder_coverage = "coverage.$databases.$conf{MOCAT_data_type}";

        mkdir_or_die("$cwd/$sample/base.$folder_coverage");
        mkdir_or_die("$cwd/$sample/insert.$folder_coverage");
        mkdir_or_die("$cwd/$sample/$folder_tax/$outputTax");
        my $is_paired_end_str = ($IS_PAIRED_END ? 'yes' : 'no');
        my $cmd =
                "$src_dir/MOCATCalculateTaxonomy.pl " .
                "-zcat $zcat " .
                "-bin $bin_dir " . 
                "-i $input " .
                "-s $inserts " .
                "-sample $sample " .
                "-cwd $cwd " .
                "-rcr $reads " .
                "-dt solexaqa " .
                "-taxrownames $rownames_dir/$outputTaxRownames " .
                "-rownames $rownames_dir/$databases.rownames " .
                "-out $folder_coverage/$output " .
                "-taxout $cwd/$sample/$folder_tax/$outputTax " .
                "-match $conf{MOCAT_mapping_mode} " .
                "-datadir $data_dir " .
                "-pos $databases " .
                "-file_list no " .
                "-file $fullname " .
                "-falen $src_dir/MOCATFilter_falen.pl " .
                "-covfile $covfile " .
                "-mmfile $mmfile " .
                "-map $taxo_profiling_map " .
                "-len $taxo_profiling_map_tot_len " .
                "-mode $taxo_profiling_mode " .
                "-PE_filter $is_paired_end_str " .
                "-counter $counter " .
                "-padded_stats_file $padded_stats_file";
       system_($cmd); 
    }
}



sub paste1 {
    my $taxo_profiling_mode = shift;
    my $reads = shift;
    my $database = shift;
    my $sample_file_basename = chomp($sample_file);
    

    chomp( my $username = `whoami` );
    chomp( my $hostname = `hostname` );

    my $PUBLIC = 0;

    my $databases = $database;
    my $assembly_type = 'assembly';
    my $end;
    my $profile_type;
    my @levels;
    if ( $taxo_profiling_mode eq 'mOTU' ) {
        @levels       = ('mOTU');
        $profile_type = "motu";
    }
    if ( $taxo_profiling_mode eq 'RefMG' ) {
        @levels = ( 'kingdom', 'phylum', 'class', 'order', 'family', 'genus', 'species', 'specI_clusters', 'taxaid' );
        $profile_type = "taxonomic";

    }
    if ( $taxo_profiling_mode eq 'identifier' ) {
        @levels       = ('identifier');
        $profile_type = "identifier";
    }
    if ( $taxo_profiling_mode eq 'functional' ) {
        @levels       = ('cog', 'ko', 'module', 'pathway');
        $profile_type = "functional";
    }

    &mkdir_or_die("$cwd/$profile_type.profiles/$databases/" .
            "$reads.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff");
    &mkdir_or_die("$cwd/$profile_type.profiles/$databases/" .
            "$reads.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff");

    my @BOI = ( 'insert' );
    my @NORM = ('mm.dist.among.unique.scaled') ;
    if ($PUBLIC == 0) {
        @BOI = ( 'base', 'insert' );
        @NORM = ( 'raw', 'norm', 'only.unique.raw', 'only.unique.norm', 'scaled', 'only.unique.scaled', 'mm.dist.among.unique.raw', 'mm.dist.among.unique.norm', 'mm.dist.among.unique.scaled' );
    }
    
    foreach my $boi ( @BOI ) {
        foreach my $norm ( @NORM ) {
            foreach my $i (@levels) {

            	my $to_paste = '';
            	my $rownames;
            	my $folder;
            	my $file;
            	my %hash;
            	my $counter1 = 0;
            	my $counter2 = 0;
            	foreach my $sample (@samples) {
            		$counter1++;
                    $folder = "$cwd/$sample/$profile_type.profiles.$databases.$conf{MOCAT_data_type}";

                    $file     = "$folder/" .
                            "$sample.$profile_type.profile.$reads.on.$databases.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff.$boi.$norm.$i";
                    $rownames = "$folder/$sample.$profile_type.profile.$reads.on.$databases.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff.$i.rownames";
            		( -e $file )
                        or die "\nERROR & EXIT: Missing $file";
            		unless ( $hash{$counter2} ) {
            			$hash{$counter2} = "";
            		}
            		$hash{$counter2} = $hash{$counter2} . " $file ";
            		if ( $counter1 == 100 ) {
            			$counter1 = 0;
            			$counter2++;
            		}
            		print ".";
            	}
            	my $norm2 = $norm;
            	if ( $norm eq 'count' ) {
            		$norm2 = 'raw';
            	}
            	( -e $rownames )
                    or die("\nERROR & EXIT: Missing file '$rownames'");

            	my $name     = "$cwd/$profile_type.profiles/$databases/" .
                                        "$reads.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff/" .
                                        "$sample_file_basename.$profile_type.profile.$reads.on.$databases.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff.$boi.$norm.$i";
            	my $COG_name = "$cwd/$profile_type.profiles/$databases/" .
                                        "$reads.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff/" .
                                        "COGs/$sample_file_basename.$profile_type.profile.$reads.on.$databases.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff.$boi.$norm.$i";
            	if ( $taxo_profiling_mode eq 'mOTU' ) {
            		system_("mkdir -p $cwd/$profile_type.profiles/$databases/$reads.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff/COGs");
            	}
            	system_("cp $rownames $name.tmp1");
            	foreach my $f ( sort { $a <=> $b } keys %hash ) {
            		system_("paste $name.tmp1 $hash{$f} > $name.tmp2 && mv $name.tmp2 $name.tmp1");
            		print ".";
            	}
            	chomp( my $currentDate = `date` );
            	open C, ">$name.comments" or die "ERROR & EXIT: Cannot write to $!";
                print C "# MOCAT $profile_type abundance table ($boi.$norm) summarized at $i level\n";
                print C "# Created $currentDate by $username @ $hostname\n";
                print C "# $cwd/motus.pl " . join( " ", @ARGV ) . "\n";
            	print C "# Original filename: $name\n";
            	close C;
            	system_("cat $name.comments $name.tmp1 > $name && rm -f $name.comments $name.tmp1");

            	my @mg = ( 'COG0012', 'COG0049', 'COG0052', 'COG0048', 'COG0016', 'COG0018', 'COG0080', 'COG0088', 'COG0081', 'COG0087', 'COG0090', 'COG0085', 'COG0091', 'COG0092', 'COG0093', 'COG0094', 'COG0096', 'COG0097', 'COG0098', 'COG0099', 'COG0100', 'COG0102', 'COG0103', 'COG0124', 'COG0172', 'COG0184', 'COG0185', 'COG0186', 'COG0197', 'COG0200', 'COG0201', 'COG0202', 'COG0215', 'COG0256', 'COG0522', 'COG0495', 'COG0533', 'COG0525', 'COG0552', 'COG0541' );
            	if ( $taxo_profiling_mode eq 'mOTU' ) {
            		print localtime() . ": Grouping $i by COGs...";
            		foreach my $i (@mg) {
            			open IN,  "$name"         or die "ERROR & EXIT: Missing $name";
            			open OUT, ">$COG_name.$i" or die "ERROR & EXIT: Cannot write to $COG_name.$i\n";
                        print OUT "# MOCAT $version $profile_type abundance table ($boi.$norm) summarized for $i\n";
                        print OUT "# Created $currentDate by $username @ $hostname\n";
                        print OUT "# $cwd/motus.pl " . join( " ", @ARGV ) . "\n";
            			print OUT "# Original filename: $COG_name.$i\n";

            			while (<IN>) {
            				if ( $. == 5 || $_ =~ /^$i\./ ) {    ### WE HAVE 5 HEADER ROWS ###
            					print OUT $_;
            				}
            			}
            			close IN;
            			close OUT;
            			system_("gzip -f $COG_name.$i");
            		}
            		print " OK!\n";
            	}
            	system_("$src_dir/MOCATFraction.pl -in $name -out $name.fraction");
            	system_("gzip -f $name $name.fraction");
            }
        }
    }

    &mkdir_or_die($OUTPUT_FOLDER);
    my @input_files = ();
    my @output_files = ();
    
    
    if ($taxo_profiling_mode eq 'mOTU'){
        my $name = "$cwd/$profile_type.profiles/$databases/" .
                        "$reads.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff/" .
                        "$sample_file_basename.$profile_type.profile.$reads.on.$databases.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff.insert.mm.dist.among.unique.scaled.mOTU.gz";
        my $pre = "$reads.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff";
        my $file = "$sample_file_basename.$profile_type.profile.$reads.on.$databases.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff";
        my $db = "$cwd/motu.profiles/$databases";
        &mkdir_or_die("$db/$pre");
        system_("perl $src_dir/MOCATPasteTaxonomyCoverageFiles_generate_mOTU_tables.pl " .
                "--wd=$cwd --map=$data_dir/$databases.motu.linkage.map --table='$name' --prefix='$db/$pre/$file' --prevalence=$prevalence_minimum");
        system_("gzip -f $db/$pre/$file*tab");
        # Make easy output
        @input_files = ("$db/$pre/$file.annotated.mOTU.clusters.fraction.tab.gz", "$db/$pre/$file.mOTU.clusters.fraction.tab.gz",
                        "$db/$pre/$file.annotated.mOTU.clusters.tab.gz",           "$db/$pre/$file.mOTU.clusters.tab.gz");
		@output_files = ('annotated.mOTU.abundances.gz', 'mOTU.abundances.gz', 'annotated.mOTU.counts.gz', 'mOTU.counts.gz')
    }
    
    if ($taxo_profiling_mode eq 'RefMG'){
        @input_files = ("$cwd/$profile_type.profiles/$databases/$reads.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff/$sample_file_basename.$profile_type.profile.$reads.on.$databases.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff.insert.mm.dist.among.unique.scaled.species.fraction.gz",
                        "$cwd/$profile_type.profiles/$databases/$reads.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff/$sample_file_basename.$profile_type.profile.$reads.on.$databases.$conf{MOCAT_data_type}.$conf{MOCAT_mapping_mode}.l$length_cutoff.p$identity_cutoff.insert.mm.dist.among.unique.scaled.species.gz");
		@output_files = ('NCBI.species.abundances.gz', 'NCBI.species.counts.gz')
    }
    
    if (scalar @input_files > 0) {
        for my $i (0 .. scalar @input_files - 1) {
            system_("ln -sf $input_files[$i] $OUTPUT_FOLDER/$output_files[$i]");
        }
    }
}

sub check_or_expand_data {
    if (-d "$cwd/motus_data") { return }

    my $motus_dir;
    my $arg0dir = dirname abs_path($0);
    if (-d "$arg0dir/motus_data") {
        $motus_dir = "$arg0dir/motus_data";
    } elsif (make_path("$arg0dir/motus_data")) {
        $motus_dir = "$arg0dir/motus_data";
        expand_data($motus_dir);
    } else {
        $motus_dir = "$cwd/motus_data";
        expand_data($motus_dir);
    }
    $data_dir = "$motus_dir/data";
    $src_dir  = "$motus_dir/src";
    $bin_dir  = "$motus_dir/bin";
}

sub check_executable {
    my $bin = shift;
    ( -x "$bin_dir/$bin" ) or die "$bin was not found in $bin_dir\n";
}

sub pre_check_files_bins {
    check_executable("fastx_quality_stats");
    check_executable("fastq_trim_filter_v5_EMBL");
    check_executable("soap2.21");
    check_executable("2bwt-builder");
    check_executable("msamtools");
    for my $sample (@samples) {
        my @files = glob("$sample/*.fq $sample/*.fq.gz");
        if ($IS_PAIRED_END) {
            my %lanes = ();
            for my $fname (@files) {
                $fname =~ s/\.[12]\.fq(\.gz)?//;
                ++$lanes{$fname};
            }
            for my $v (values %lanes) {
                if ($v != 2) {
                    die "Input files are not paired up in sample $sample.\n";
                }
            }
        } else {
            if (! (scalar @files)) {
                die "No input files in sample $sample";
            }
        }
    }
}

sub check_args {
    (grep {$_ eq $INPUT_FORMAT} ('auto', 'sanger', 'illumina'))
        or die("fastq-format must be one of 'auto' [the default], 'sanger', or 'illumina' [for older Illumina machines, newer models use sanger format]. Got $INPUT_FORMAT.\n");
}

sub usage {
    my $hard_exit = shift;
    print <<EOF;

For single-sample mode:

If your data is single-end:
$0 <INPUTFILE>

If your data is paired-end:

$0 <INPUTFILE1> <INPUTFILE2>

For processing multiple samples at once, you must set up the following:

1.  Create a directory for each sample with the read files inside (you can even
    have more than one lane per sample).
2.  Create a text file <SAMPLE_FILE> with the directory names (one per line)
3.  Pass the name of that text file to the ``$0`` script::

$0 --sample-file <SAMPLE_FILE>

This is the same organization is used by the MOCAT pipeline.

The most important results are saved in the RESULTS folder:

    RESULTS/annotated.mOTU.abundances.gz
    RESULTS/mOTU.abundances.gz
    RESULTS/NCBI.species.abundances.gz

See webpage for details: http://www.bork.embl.de/software/mOTU/

Citation: Sunagawa et al.. Nature Methods 10, 1196-1199 (2013)
		http://dx.doi.org/10.1038/nmeth.2693

Options

    --verbose           Be more verbose while running the analysis

    --processors=N      This should be an integer and defines the number of
                        processors that the script will use.

    --length-cutoff=L   The minimum size per read (after quality-based
                        trimming), default: 45.

    --identity-cutoff=I Minimum percentage identity in alignment (default: 97)

    --quality-cutoff=Q  Basepair quality cutoff (default: 20)

    --fastq-format      The format of the input files. Must be one of 'auto'
                        (the default), 'sanger', or 'illumina'. Note that new
                        Illumina machines actually use the 'sanger' format. The
                        auto-detection should generally work well.

    --output-directory  Where to place the final results file (by default it
                        uses a directory named ``RESULTS``).

Website: http://www.bork.embl.de/software/mOTU

EOF

    if ($hard_exit) {
        exit(1);
    }
    exit;
}

sub copy_map_tables {
    system_("cp $data_dir/mOTU.v1.map.txt $OUTPUT_FOLDER");
    system_("cp $data_dir/mOTU-LG.v1.annotations.txt $OUTPUT_FOLDER");
}

sub main {
    check_args();
    read_samples();
    check_or_expand_data();
    pre_check_files_bins();
    read_trim_filter();
    screen('mOTU.v1.padded', 'reads.processed.solexaqa');
    filter('mOTU.v1.padded', 'reads.processed.solexaqa');
    profile('mOTU', 'reads.processed.solexaqa', 'mOTU.v1.padded');
    paste1('mOTU', 'reads.processed.solexaqa', 'mOTU.v1.padded');

    $calculateTaxonomy_previous_calc_coverage_stats_file = 1;
    screen('RefMG.v1.padded', 'reads.extracted.mOTU.v1.padded.solexaqa');
    filter('RefMG.v1.padded', 'reads.extracted.mOTU.v1.padded.solexaqa');
    profile('RefMG', 'reads.extracted.mOTU.v1.padded.solexaqa', 'RefMG.v1.padded');
    paste1('RefMG', 'reads.extracted.mOTU.v1.padded.solexaqa', 'RefMG.v1.padded');

    copy_map_tables();
}

GetOptions(
    'sample-file=s'             => \$sample_file,
    'processors=i'              => \$processors,
    'length-cutoff=i'           => \$length_cutoff,
    'identity-cutoff=i'         => \$identity_cutoff,
    'quality-cutoff=i'          => \$quality_cutoff,
    'prevalence-min=i'          => \$prevalence_minimum,
    'fastq-format=s'            => \$INPUT_FORMAT,
    'output-directory=s'        => \$OUTPUT_FOLDER,
    'verbose'                   => \$verbose,
) or usage(1);


&main();
1;

__DATA__
