Commit d90fe421 authored by Chris Cheshire's avatar Chris Cheshire
Browse files

Merge remote-tracking branch 'origin/nf-core-template-merge-2.1' into dev

parents a46a2a34 f28ced86

.github/workflows/awsfulltest.yml

+22 −30
Original line number Diff line number Diff line 
name: nf-core AWS full size tests

# This workflow is triggered on published releases.

# It can be additionally triggered manually with GitHub actions workflow dispatch.

# It can be additionally triggered manually with GitHub actions workflow dispatch button.

# It runs the -profile 'test_full' on AWS batch



on:

  workflow_run:

    workflows: ["nf-core Docker push (release)"]

    types: [completed]

  release:

    types: [published]

  workflow_dispatch:





env:

  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

  TOWER_ACCESS_TOKEN: ${{ secrets.AWS_TOWER_TOKEN }}

  AWS_JOB_DEFINITION: ${{ secrets.AWS_JOB_DEFINITION }}

  AWS_JOB_QUEUE: ${{ secrets.AWS_JOB_QUEUE }}

  AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}





jobs:

  run-awstest:

  run-tower:

    name: Run AWS full tests

    if: github.repository == 'nf-core/cutandrun'

    runs-on: ubuntu-latest

    steps:

      - name: Setup Miniconda

        uses: conda-incubator/setup-miniconda@v2

      - name: Launch workflow via tower

        uses: nf-core/tower-action@master

        # TODO nf-core: You can customise AWS full pipeline tests as required

        # Add full size test data (but still relatively small datasets for few samples)

        # on the `test_full.config` test runs with only one set of parameters

        

        with:

          auto-update-conda: true

          python-version: 3.7

      - name: Install awscli

        run: conda install -c conda-forge awscli

      - name: Start AWS batch job

        run: |

          aws batch submit-job \

            --region eu-west-1 \

            --job-name nf-core-cutandrun \

            --job-queue $AWS_JOB_QUEUE \

            --job-definition $AWS_JOB_DEFINITION \

            --container-overrides '{"command": ["nf-core/cutandrun", "-r '"${GITHUB_SHA}"' -profile test_full --outdir s3://'"${AWS_S3_BUCKET}"'/cutandrun/results-'"${GITHUB_SHA}"' -w s3://'"${AWS_S3_BUCKET}"'/cutandrun/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}'
 
          workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}

          bearer_token: ${{ secrets.TOWER_BEARER_TOKEN }}

          compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}

          pipeline: ${{ github.repository }}

          revision: ${{ github.sha }}

          workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/cutandrun/work-${{ github.sha }}

          parameters: |

            {

              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/cutandrun/results-${{ github.sha }}"

            }

          profiles: '[ "test_full", "aws_tower" ]'

.github/workflows/awstest.yml

+18 −29
Original line number Diff line number Diff line 
name: nf-core AWS test

# This workflow is triggered on push to the master branch.

# It can be additionally triggered manually with GitHub actions workflow dispatch.

# It runs the -profile 'test' on AWS batch.

# This workflow can be triggered manually with the GitHub actions workflow dispatch button.

# It runs the -profile 'test' on AWS batch



on:

  workflow_dispatch:





env:

  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

  TOWER_ACCESS_TOKEN: ${{ secrets.AWS_TOWER_TOKEN }}

  AWS_JOB_DEFINITION: ${{ secrets.AWS_JOB_DEFINITION }}

  AWS_JOB_QUEUE: ${{ secrets.AWS_JOB_QUEUE }}

  AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}





jobs:

  run-awstest:

  run-tower:

    name: Run AWS tests

    if: github.repository == 'nf-core/cutandrun'

    runs-on: ubuntu-latest

    steps:

      - name: Setup Miniconda

        uses: conda-incubator/setup-miniconda@v2

      - name: Launch workflow via tower

        uses: nf-core/tower-action@master

        

        with:

          auto-update-conda: true

          python-version: 3.7

      - name: Install awscli

        run: conda install -c conda-forge awscli

      - name: Start AWS batch job

        run: |

          aws batch submit-job \

          --region eu-west-1 \

          --job-name nf-core-cutandrun \

          --job-queue $AWS_JOB_QUEUE \

          --job-definition $AWS_JOB_DEFINITION \

          --container-overrides '{"command": ["nf-core/cutandrun", "-r '"${GITHUB_SHA}"' -profile test --outdir s3://'"${AWS_S3_BUCKET}"'/cutandrun/results-'"${GITHUB_SHA}"' -w s3://'"${AWS_S3_BUCKET}"'/cutandrun/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}'
 
          workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}

          bearer_token: ${{ secrets.TOWER_BEARER_TOKEN }}

          compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}

          pipeline: ${{ github.repository }}

          revision: ${{ github.sha }}

          workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/cutandrun/work-${{ github.sha }}

          parameters: |

            {

              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/cutandrun/results-${{ github.sha }}"

            }

          profiles: '[ "test", "aws_tower" ]'

.github/workflows/ci.yml

+1 −51
Original line number Diff line number Diff line
@@ -20,57 +20,7 @@ jobs: 
    strategy:

      matrix:

        # Nextflow versions: check pipeline minimum and current latest

        nxf_ver: ["21.04.0", ""]

    steps:

      - name: Check out pipeline code

        uses: actions/checkout@v2



      - name: Install Nextflow

        run: |

          wget -qO- get.nextflow.io | bash

          sudo mv nextflow /usr/local/bin/



      - name: Run pipeline with test data

        run: |

          nextflow run ${GITHUB_WORKSPACE} -profile test,docker

# ---------------------------------------------------------------------

  test_no_igg:

    name: Run pipeline test with no IgG

    # Only run on push if this is the nf-core dev branch (merged PRs)

    if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/cutandrun') }}

    runs-on: ubuntu-latest

    env:

      NXF_VER: ${{ matrix.nxf_ver }}

      NXF_ANSI_LOG: false

    strategy:

      matrix:

        # Nextflow versions: check pipeline minimum and current latest

        nxf_ver: ["21.04.0", ""]

    steps:

      - name: Check out pipeline code

        uses: actions/checkout@v2



      - name: Install Nextflow

        run: |

          wget -qO- get.nextflow.io | bash

          sudo mv nextflow /usr/local/bin/



      - name: Run pipeline with test data

        run: |

          nextflow run ${GITHUB_WORKSPACE} -profile test_no_igg,docker

# ---------------------------------------------------------------------

  test_all_exp_one_igg_small:

    name: Run pipeline test with 2 groups of 2 replicate and one replicate of igg

    # Only run on push if this is the luslab dev branch (merged PRs)

    if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/cutandrun') }}

    runs-on: ubuntu-latest

    env:

      NXF_VER: ${{ matrix.nxf_ver }}

      NXF_ANSI_LOG: false

    strategy:

      matrix:

        # Nextflow versions: check pipeline minimum and current latest

        nxf_ver: ["21.04.0", ""]

        nxf_ver: ['21.04.0', '']

    steps:

      - name: Check out pipeline code

        uses: actions/checkout@v2

.github/workflows/linting.yml

+6 −6
Original line number Diff line number Diff line
@@ -18,7 +18,7 @@ jobs: 
      - name: Install markdownlint

        run: npm install -g markdownlint-cli

      - name: Run Markdownlint

        run: markdownlint ${GITHUB_WORKSPACE} -c ${GITHUB_WORKSPACE}/.markdownlint.yml

        run: markdownlint .



      # If the above check failed, post a comment on the PR explaining the failure

      - name: Post PR comment
@@ -35,8 +35,8 @@ jobs: 
                * On Mac: `brew install markdownlint-cli`

                * Everything else: [Install `npm`](https://www.npmjs.com/get-npm) then [install `markdownlint-cli`](https://www.npmjs.com/package/markdownlint-cli) (`npm install -g markdownlint-cli`)

            * Fix the markdown errors

                * Automatically: `markdownlint . --config markdownlint.yml --fix`

                * Manually resolve anything left from `markdownlint . --config .github/markdownlint.yml`

                * Automatically: `markdownlint . --fix`

                * Manually resolve anything left from `markdownlint .`



            Once you push these changes the test should pass, and you can hide this comment :+1:
@@ -53,13 +53,13 @@ jobs: 


      - uses: actions/setup-node@v1

        with:

          node-version: "10"

          node-version: '10'



      - name: Install editorconfig-checker

        run: npm install -g editorconfig-checker



      - name: Run ECLint check

        run: editorconfig-checker --config ${GITHUB_WORKSPACE}/.ecrc -exclude README.md $(git ls-files | grep -v test)

        run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test)



  YAML:

    runs-on: ubuntu-latest
@@ -126,7 +126,7 @@ jobs: 
          GITHUB_COMMENTS_URL: ${{ github.event.pull_request.comments_url }}

          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

          GITHUB_PR_COMMIT: ${{ github.event.pull_request.head.sha }}

        run: nf-core -l lint_log.txt lint ${GITHUB_WORKSPACE} --markdown lint_results.md

        run: nf-core -l lint_log.txt lint --dir ${GITHUB_WORKSPACE} --markdown lint_results.md



      - name: Save PR number

        if: ${{ always() }}

.markdownlint.yml

+0 −2
Original line number Diff line number Diff line
@@ -12,5 +12,3 @@ no-inline-html: 
        - kbd

        - details

        - summary
 
single-title:

    level: 2

CRA Git | Maintained and supported by SUSTech CRA and CCSE