Commit f591ea0e authored by TomKellyGenetics's avatar TomKellyGenetics
Browse files

Merge branch 'master' of https://github.com/minoda-lab/universc

parents 902e7c89 df7a1c53

README.md

+1 −1
Original line number Diff line number Diff line
@@ -32,7 +32,7 @@ tags: 
![Docker Automated build](https://img.shields.io/docker/automated/tomkellygenetics/universc)

![Docker Build Status](https://img.shields.io/docker/build/tomkellygenetics/universc)

![Docker Image Size (tag)](https://img.shields.io/docker/image-size/tomkellygenetics/universc/latest)

[![GitHub Views](http://hits.dwyl.com/tomkellygenetics/universc.svg)](http://hits.dwyl.com/tomkellygenetics/universc)

[![GitHub Views](http://hits.dwyl.com/minoda-lab/universc.svg)](http://hits.dwyl.com/minoda-lab/universc)



![Docker Stars](https://img.shields.io/docker/stars/tomkellygenetics/universc)

![Docker Pulls](https://img.shields.io/docker/pulls/tomkellygenetics/universc)

docker-compose.test.yml

+8 −5
Original line number Diff line number Diff line 
sut:

  build: .

  command: echo "build complete" && bash test/test_call.sh

  command: echo "build complete"



test:

  build: .

  command: bash /universc/test/test_call.sh



test_10x:

  build: .

  command: bash test/run_tests_10x.sh

  command: bash /universc/test/run_tests_10x.sh



test_dropseq:

  build: .

  command: bash test/run_tests_dropseq.sh

  command: bash /universc/test/run_tests_dropseq.sh



test_icell8:

  build: .

  command: bash test/run_tests_icell8.sh

  command: bash /universc/test/run_tests_icell8.sh

docker-compose.yml

+1 −13
Original line number Diff line number Diff line 
sut:

  build: .

  command: echo "build complete" && bash test/test_call.sh



test_10x:

  build: .

  command: bash test/run_tests_10x.sh



test_dropseq:

  build: .

  command: bash test/run_tests_dropseq.sh



test_icell8:

  build: .

  command: bash test/run_tests_icell8.sh
 
  command: echo "build complete"

launch_universc.sh

+71 −28
Original line number Diff line number Diff line
@@ -1019,13 +1019,13 @@ for key in ${keys[@]}; do 
        fi

        

        if [[ -h $read ]]; then

            path=`readlink -f $read`

            fullpath=`readlink -f $read`

            if [[ $verbose ]]; then

                echo "***Warning: file $read not in current directory. Path to the file captured instead.***"

                echo " (file) $read"

                echo " (path) $path"

                echo " (path) $fullpath"

            fi

            read=${path}

            read=${fullpath}

        fi

        case $read in

            #check if contains lane before read
@@ -1040,7 +1040,7 @@ for key in ${keys[@]}; do 
                    echo "***Warning: file $read does not have lane value in its name. Lane 1 is assumed.***"

                echo "  renaming $read ..."

                fi

                rename -f "s/_$readkey/_L001_$readkey/" $read

                rename -f "s/_$readkey/_L001_$readkey/" ${read}*

                #update file variable

                read=`echo $read | sed -e "s/_${readkey}/_L001_${readkey}/g"`

                list[$j]=$read
@@ -1060,7 +1060,7 @@ for key in ${keys[@]}; do 
                    echo "  renaming $read ..."

                fi

                k=$((${j}+1))

                rename -f "s/_L0/_S${k}_L0/" $read

                rename -f "s/_L0/_S${k}_L0/" ${read}*

                #update file variable

                read=`echo $read | sed -e "s/_L0/_S${k}_L0/g"`

                list[$j]=$read
@@ -1073,15 +1073,52 @@ for key in ${keys[@]}; do 
                    echo "  $read compatible with suffix"

                fi

            ;;

            *)

            *.*)

                #rename file

                if [[ $verbose ]]; then

                    echo "***Warning: file $read does not have suffix in its name. Suffix 001 is given.***"

                    echo "  renaming $read ..."

                fi

                rename -f "s/_${readkey}.*\./_${readkey}_001\./" $read

                if [[  -f $(find $(dirname ${read}) -name $(basename ${read})'*.gz') ]]; then

                    rename -f "s/_${readkey}.(.*).gz/_${readkey}_001\.\$1.gz/g" ${read}*gz

                fi

                if [[ ${read} == *.gz ]]; then

                    rename -f "s/_${readkey}.(.*).gz/_${readkey}_001\.\$1.gz/g" ${read}

                fi

                if [[ -f $(find $(dirname ${read}) -name $(basename ${read})'*.fastq') ]]; then

                    rename -f "s/_${readkey}.(.*)/_${readkey}_001\.\$1/" ${read}*.fastq

                fi

                if [[ -f $(find $(dirname ${read}) -name $(basename ${read})'*.fq') ]]; then

                    rename -f "s/_${readkey}.(.*)/_${readkey}_001\.\$1/" ${read}*.fq

                fi

                if [[ ${read} == *.fastq ]]; then

                    rename -f "s/_${readkey}(.*).fastq/_${readkey}_001\.fastq/" ${read} ${read}.gz

               fi

              if [[ ${read} == *.fq ]]; then

                   rename -f "s/_${readkey}(.*).fq/_${readkey}_001\.fq/" ${read}

              fi

              #update file variable

              if [[ ${read} == *.gz ]] || [[ ${read} == *.fastq ]] || [[ ${read} == *.fq ]] || [[ -f ${read} ]]; then

                  #assumes read name already contains . in file extension

                  read=`echo $read | sed -e "s/_${readkey}.*\./_${readkey}_001\./g"`

              else

                  #replace everything after read key (R1, R2, I1, I2) with 001 suffix (detects file later)

                  rename -f "s/_${readkey}.*/_${readkey}_001/g" ${read}

                  read=`echo $read | sed -e "s/_${readkey}.*/_${readkey}_001/g"`

              fi

              #remove characters after read key (R1, R2, I1, I2) required as above

              if [[ ${read} != *_${readkey}_001.* ]] && [[ ${read} != *.* ]]; then

                  rename -f "s/_${readkey}_*\./_${readkey}_001\./" ${read}

                  read=`echo $read | sed -e "s/_${readkey}.*\./_${readkey}_001\./g"`

              elif [[ ${read} != *_${readkey}_001 ]] || [[ ${read} != *_${readkey}*00#1 ]]; then

                  rename -f "s/_${readkey}*_001/_${readkey}_001/" ${read#}

                  read=`echo $read | sed -e "s/_${readkey}*_001/_${readkey}_001/g"`

              fi

              if [[ ${read} == *_${readkey}_*_001.* ]]; then

                   rename -f "s/_${readkey}.*_001\./_${readkey}_001\./" ${read}

                   read=`echo $read | sed -e "s/_${readkey}*_001\./_${readkey}_001\./g"`

              fi



              list[$j]=$read

            ;;

        esac
@@ -1682,6 +1719,9 @@ if [[ $lock -eq 0 ]]; then 
    fi

    if [[ $technology == "10x" ]] && [[ $barcodefile == "default:10x" ]]; then

        #restore logo in HTML template

        if [[ $verbose  ]]; then

                echo "restore logo in summary HTML"

        fi

        if [[ -f ${cellrangerpath}-cs/${cellrangerversion}/lib/python/cellranger/webshim/template/navbar.backup.html ]];then

            cp ${cellrangerpath}-cs/${cellrangerversion}/lib/python/cellranger/webshim/template/navbar.backup.html ${cellrangerpath}-cs/${cellrangerversion}/lib/python/cellranger/webshim/template/navbar.html

        fi
@@ -1769,6 +1809,9 @@ if [[ $lock -eq 0 ]]; then 
    #convert whitelist to the appropriate barcode

    echo " converting whitelist"

    if [[ ${barcodefile} == "default:10x" ]]; then

        if [[ $verbose ]]; then

            echo "  ... restoring 10x barcodes"

        fi

        #for version 2

        cp ${v2}.backup ${v2}

        #for version 3
@@ -1784,9 +1827,9 @@ if [[ $lock -eq 0 ]]; then 
            echo As: $As

            sed -i "s/^/$As/" ${v2} #Trim the first n characters from the beginning of the quality

        fi

    fi

        #for version 3

        cat ${v2} > ${v3}

    fi

        if [[ -f translation/${v3}.gz ]]; then

            rm translation/${v3}.gz

            awk -F , -v OFS="\t" '{print $1, "\t", $1}' $v3 > translation/${v3}

test/run_tests_10x.sh

+56 −7
Original line number Diff line number Diff line
@@ -4,6 +4,20 @@ 
cd $(dirname ${BASH_SOURCE[0]})/..

pwd



# used to export to PATH for testing on SGE server

export PATH=${HOME}/local/bin/cellranger-2.1.0:$PATH



cellrangerversion=`cellranger count --version | head -n 2 | tail -n 1 | cut -f2 -d'(' | cut -f1 -d')'`

cellrangerpath=`which cellranger`



# set up cellranger reference

if [[ ! -f test/cellranger_reference/cellranger-tiny-ref/3.0.0/star/SA ]] && [[ -f $(dirname $cellrangerpath)/cellranger-tiny-ref/3.0.0/star/SA ]]; then

    rsync $(dirname $cellrangerpath)/cellranger-tiny-ref/3.0.0/star/SA test/cellranger_reference/cellranger-tiny-ref/3.0.0/star/SA

fi

if [[ ! -f test/cellranger_reference/cellranger-tiny-ref/1.2.0/star/SA ]] && [[ -f $(dirname $cellrangerpath)/cellranger-tiny-ref/1.2.0/star/SA ]]; then

    rsync $(dirname $cellrangerpath)/cellranger-tiny-ref/1.2.0/star/SA test/cellranger_reference/cellranger-tiny-ref/1.2.0/star/SA

fi



# used to export to PATH for testing on SGE server

export PATH=${HOME}/local/bin/cellranger-3.0.2:$PATH
@@ -11,11 +25,11 @@ cellrangerversion=`cellranger count --version | head -n 2 | tail -n 1 | cut -f2 
cellrangerpath=`which cellranger`



# set up cellranger reference

if [[ ! -f cellranger_reference/cellranger-tiny-ref/3.0.0/star/SA ]] && [[ -f $(dirname $cellrangerpath)/cellranger-tiny-ref/3.0.0/star/SA ]]; then

    rsync $(dirname $cellrangerpath)/cellranger-tiny-ref/3.0.0/star/SA cellranger_reference/cellranger-tiny-ref/3.0.0/star/SA

if [[ ! -f test/cellranger_reference/cellranger-tiny-ref/3.0.0/star/SA ]] && [[ -f $(dirname $cellrangerpath)/cellranger-tiny-ref/3.0.0/star/SA ]]; then

    rsync $(dirname $cellrangerpath)/cellranger-tiny-ref/3.0.0/star/SA test/cellranger_reference/cellranger-tiny-ref/3.0.0/star/SA

fi

if [[ ! -f cellranger_reference/cellranger-tiny-ref/1.2.0/star/SA ]] && [[ -f $(dirname $cellrangerpath)/cellranger-tiny-ref/1.2.0/star/SA ]]; then

    rsync $(dirname $cellrangerpath)/cellranger-tiny-ref/1.2.0/star/SA cellranger_reference/cellranger-tiny-ref/1.2.0/star/SA

if [[ ! -f test/cellranger_reference/cellranger-tiny-ref/1.2.0/star/SA ]] && [[ -f $(dirname $cellrangerpath)/cellranger-tiny-ref/1.2.0/star/SA ]]; then

    rsync $(dirname $cellrangerpath)/cellranger-tiny-ref/1.2.0/star/SA test/cellranger_reference/cellranger-tiny-ref/1.2.0/star/SA

fi



# reset barcodes for test
@@ -23,28 +37,63 @@ bash launch_universc.sh -t "10x" --setup 


## test 10x data

# unzip input data

ls ${cellrangerpath}-cs/${cellrangerversion}/lib/python/cellranger/barcodes/3M-february-2018.txt*

if [[ ! -f ${cellrangerpath}-cs/${cellrangerversion}/lib/python/cellranger/barcodes/3M-february-2018.txt.gz ]]; then

    gzip -f ${cellrangerpath}-cs/${cellrangerversion}/lib/python/cellranger/barcodes/3M-february-2018.txt

fi

rm -rf ${cellrangerpath}-cs/${cellrangerversion}/lib/python/cellranger/barcodes/3M-february-2018.txt

if [[ -d tiny-test ]]; then

    rm -rf tiny-test

fi

# test cellranger call

cellranger testrun --id="tiny-test"

# unzip input data

if [[ -f test/shared/cellranger-tiny-fastq/3.0.0/*fastq.gz ]]; then

    gunzip -f test/shared/cellranger-tiny-fastq/3.0.0/*fastq.gz

fi

if [[ -f ${cellrangerpath}-cs/${cellrangerversion}/lib/python/cellranger/barcodes/3M-february-2018.txt.gz ]]; then

    gunzip -f ${cellrangerpath}-cs/${cellrangerversion}/lib/python/cellranger/barcodes/3M-february-2018.txt.gz 

if [[ -f test/shared/cellranger-tiny-fastq/3.0.0/*fastq ]]; then

    rm -rf test/shared/cellranger-tiny-fastq/3.0.0/*fastq.gz

fi

# test cellranger call

if [[ -d tiny-count-v3 ]]; then

    rm -rf tiny-count-v3

fi

if [[ ! -f ${cellrangerpath}-cs/${cellrangerversion}/lib/python/cellranger/barcodes/3M-february-2018.txt.gz ]]; then

    gzip -f ${cellrangerpath}-cs/${cellrangerversion}/lib/python/cellranger/barcodes/3M-february-2018.txt

fi

if [[ -f ${cellrangerpath}-cs/${cellrangerversion}/lib/python/cellranger/barcodes/3M-february-2018.txt ]]; then

    rm -rf ${cellrangerpath}-cs/${cellrangerversion}/lib/python/cellranger/barcodes/3M-february-2018.txt

fi

cellranger count --id="tiny-count-v3" \

 --fastqs="test/shared/cellranger-tiny-fastq/3.0.0/" --sample="tinygex" \

 --transcriptome="test/cellranger_reference/cellranger-tiny-ref/3.0.0"



if [[ -d tiny-count-v2 ]]; then

    rm -rf tiny-count-v2

fi 

if [[ ! -f ${cellrangerpath}-cs/${cellrangerversion}/lib/python/cellranger/barcodes/3M-february-2018.txt.gz ]]; then

    gzip -f ${cellrangerpath}-cs/${cellrangerversion}/lib/python/cellranger/barcodes/3M-february-2018.txt

fi

if [[ -f ${cellrangerpath}-cs/${cellrangerversion}/lib/python/cellranger/barcodes/3M-february-2018.txt ]]; then

    rm -rf ${cellrangerpath}-cs/${cellrangerversion}/lib/python/cellranger/barcodes/3M-february-2018.txt

fi

cellranger count --id="tiny-count-v2" \

 --fastqs="test/shared/cellranger-tiny-fastq/1.2.0/" --sample="" --chemistry="threeprime" \

 --transcriptome="test/cellranger_reference/cellranger-tiny-ref/1.2.0"



# call convert on 10x with multiple lanes

if [[ -d test-10x-v3 ]]; then

    rm -rf test-10x-v3

fi

if [[ -d input4cellranger_test-10x-v3 ]]; then

    rm -rf input4cellranger_test-10x-v3

fi

# unzip input data

if [[ -f test/shared/cellranger-tiny-fastq/3.0.0/tinygex_S1_L001/*fastq.gz ]]; then

    unpigz -f test/shared/cellranger-tiny-fastq/3.0.0/tinygex_S1_L001/*fastq.gz

fi

if [[ -f test/shared/cellranger-tiny-fastq/3.0.0/tinygex_S1_L002/*fastq.gz ]]; then

    unpigz -f test/shared/cellranger-tiny-fastq/3.0.0/tinygex_S1_L002/*fastq.gz

fi

bash launch_universc.sh --id "test-10x-v3" --technology "10x" \

 --reference "test/cellranger_reference/cellranger-tiny-ref/3.0.0" \

 --file "test/shared/cellranger-tiny-fastq/3.0.0/tinygex_S1_L001" \

CRA Git | Maintained and supported by SUSTech CRA and CCSE