Unverified Commit 1f73de8d authored by Chris Cheshire's avatar Chris Cheshire Committed by GitHub
Browse files

Merge pull request #106 from luslab/dev 

Dev Update
parents 32219490 f169fea0

.github/workflows/awsfulltest.yml

+0 −3
Original line number Diff line number Diff line
@@ -15,9 +15,6 @@ jobs: 
    steps:

      - name: Launch workflow via tower

        uses: nf-core/tower-action@v2

        # TODO nf-core: You can customise AWS full pipeline tests as required

        # Add full size test data (but still relatively small datasets for few samples)

        # on the `test_full.config` test runs with only one set of parameters

        

        with:

          workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}

.github/workflows/ci.yml

+64 −18
Original line number Diff line number Diff line
@@ -49,9 +49,11 @@ jobs: 
          wget -q https://raw.githubusercontent.com/nf-core/test-datasets/cutandrun/reference/genomes/e_coli_U00096_3.fa.gz

          wget -q https://raw.githubusercontent.com/nf-core/test-datasets/cutandrun/reference/genomes/e_coli_U00096_3.tar.gz



      - name: Run pipeline

        run: |

          nextflow run ${GITHUB_WORKSPACE} -profile docker,test_local_zip

      - name: Run Pipeline

        uses: Wandalen/wretry.action@v1.0.11

        with:

          command: nextflow run ${GITHUB_WORKSPACE} -profile docker,test_local_zip

          attempt_limit: 2



  ##############################

  ### LARGE INTEGRATION TEST ###
@@ -89,9 +91,11 @@ jobs: 
          wget -q https://raw.githubusercontent.com/nf-core/test-datasets/cutandrun/reference/genomes/e_coli_U00096_3.fa.gz

          wget -q https://raw.githubusercontent.com/nf-core/test-datasets/cutandrun/reference/genomes/e_coli_U00096_3.tar.gz



      - name: Run pipeline with test data

        run: |

          nextflow run ${GITHUB_WORKSPACE} -profile docker,test_full_small_local_zip

      - name: Run Pipeline

        uses: Wandalen/wretry.action@v1.0.11

        with:

          command: nextflow run ${GITHUB_WORKSPACE} -profile docker,test_full_small_local_zip

          attempt_limit: 2



  #####################################

  ### EDGE VERSION INTEGRATION TEST ###
@@ -127,9 +131,11 @@ jobs: 
          wget -q https://raw.githubusercontent.com/nf-core/test-datasets/cutandrun/reference/genomes/e_coli_U00096_3.fa.gz

          wget -q https://raw.githubusercontent.com/nf-core/test-datasets/cutandrun/reference/genomes/e_coli_U00096_3.tar.gz



      - name: Run pipeline with test data

        run: |

          nextflow run ${GITHUB_WORKSPACE} -profile docker,test_local_zip

      - name: Run Pipeline

        uses: Wandalen/wretry.action@v1.0.11

        with:

          command: nextflow run ${GITHUB_WORKSPACE} -profile docker,test_local_zip

          attempt_limit: 2



  ##############################

  ###      UNIT TESTS        ###
@@ -145,35 +151,71 @@ jobs: 
      fail-fast: false

      matrix:

        nxf_version: ['21.10.3', '']

        tags: ["samplesheet", "verify_output_input", "verify_output_align_short", "verify_output_align_long", "verify_output_peak_calling", "verify_output_reporting", "verify_output_save", "verify_output_skip"]

        tags:

          - params

          - samplesheet

          - verify_output_save_ref

          - verify_output_only_input

          - verify_output_save_merged

          - verify_output_skip_fastqc

          - verify_output_save_trimmed

          - verify_output_skip_trimming

          - verify_output_align_intermed

          - verify_output_align_only_align

          - verify_output_align_save_spikein_align

          - verify_output_align_save_unaligned

          - verify_output_only_filtering

          - verify_output_align_duplicates_mark

          - verify_output_align_duplicates_remove

          - verify_output_align_duplicates_remove_target

          - verify_output_peak_calling_only_peak_calling

          - test_bam_scale_none

          - test_bam_scale_spikein

          - test_bam_scale_cpm

          - test_bam_scale_rpkm

          - test_bam_scale_bpm

          - test_bam_scale_cpm_iggscale

          - test_peak_callers_seacr

          - test_peak_callers_macs2

          - test_peak_callers_invalid

          - test_peak_callers_seacr_macs2

          - test_peak_callers_macs2_seacr

          - test_peak_callers_seacr_macs2_noigg

          - test_conseneus_peaks_group

          - test_conseneus_peaks_all

          - test_conseneus_peaks_invalid

          - verify_output_reporting_skip_reporting

          - verify_output_reporting_skip_igv

          - verify_output_reporting_skip_heatmaps

          - verify_output_reporting_skip_multiqc

    steps:

      - uses: actions/checkout@v2

      - name: Checkout Code

        uses: actions/checkout@v2



      - name: Set up Python

        uses: actions/setup-python@v2

        with:

          python-version: "3.x"



      - uses: actions/cache@v2

      - name: Setup Pip Cache

        uses: actions/cache@v2

        with:

          path: ~/.cache/pip

          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}

          restore-keys: |

            ${{ runner.os }}-pip-

      - name: Set up Python

        uses: actions/setup-python@v2

        with:

          python-version: "3.x"



      - name: Install Python dependencies

        run: python -m pip install --upgrade pip pytest-workflow



      - uses: actions/cache@v2

      - name: Restore Keys

        uses: actions/cache@v2

        with:

          path: /usr/local/bin/nextflow

          key: ${{ runner.os }}-nextflow-${{ matrix.nxf_version }}

          restore-keys: |

            ${{ runner.os }}-nextflow-



      - name: Install Nextflow

        env:

          NXF_VER: ${{ matrix.nxf_version }}
@@ -181,8 +223,12 @@ jobs: 
        run: |

          wget -qO- get.nextflow.io | bash

          sudo mv nextflow /usr/local/bin/



      - name: Run pytest-workflow

        run: pytest --tag ${{ matrix.tags }} --kwdof

        uses: Wandalen/wretry.action@v1.0.11

        with:

          command: pytest --tag ${{ matrix.tags }} --kwdof

          attempt_limit: 3



      - name: Upload logs on failure

        if: failure()

CHANGELOG.md

+3 −3
Original line number Diff line number Diff line
@@ -57,6 +57,6 @@ Note, since the pipeline is now using Nextflow DSL2, each process will be run wi 
|------------|-------------|-------------|

| `samtools` | 1.13        | 1.14        |



> **NB:** Dependency has been __updated__ if both old and new version information is present.

> **NB:** Dependency has been __added__ if just the new version information is present.

> **NB:** Dependency has been __removed__ if version information isn't present.
 
> **NB:** Dependency has been **updated** if both old and new version information is present.

> **NB:** Dependency has been **added** if just the new version information is present.

> **NB:** Dependency has been **removed** if version information isn't present.

assets/multiqc_config.yaml

+2 −2
Original line number Diff line number Diff line
@@ -37,11 +37,11 @@ module_order: 
    - bowtie2:

        name: 'Bowtie2 (target)'

        path_filters:

            - "./bowtie2/*target.bowtie2.log"

            - "./bowtie2/*.bowtie2.log"

    - bowtie2:

        name: 'Bowtie2 (spike-in)'

        path_filters:

            - "./bowtie2/*spikein.bowtie2.log"

            - "./bowtie2_spikein/*.bowtie2.log"

    - samtools

    - picard

    - custom_content

bin/lib/reports.py

+10 −4
Original line number Diff line number Diff line
@@ -99,10 +99,15 @@ class Reports: 


        self.metadata_noctrl_table = pd.read_csv(self.meta_path, sep=',')

        self.metadata_table = pd.read_csv(self.meta_ctrl_path, sep=',')



        self.duplicate_info = False

        if 'dedup_percent_duplication' in self.metadata_table.columns:

            self.duplicate_info = True



        self.scale_factor_info = False

        if 'scale_factor' in self.metadata_table.columns:

            self.scale_factor_info = True



        # Make new perctenage alignment columns

        self.metadata_table['target_alignment_rate'] = self.metadata_table.loc[:, ('bt2_total_aligned_target')] / self.metadata_table.loc[:, ('bt2_total_reads_target')] * 100

        self.metadata_table['spikein_alignment_rate'] = self.metadata_table.loc[:, ('bt2_total_aligned_spikein')] / self.metadata_table.loc[:, ('bt2_total_reads_spikein')] * 100
@@ -303,6 +308,7 @@ class Reports: 
        data["04_replicate_heatmap"] = data5



        # Plot section 6

        if self.scale_factor_info == True:

            multi_plot, data6 = self.scale_factor_summary()

            plots["05_01_scale_factor"] = multi_plot[0]

            plots["05_02_frag_count"] = multi_plot[1]

CRA Git | Maintained and supported by SUSTech CRA and CCSE