Merge pull request #38 from SchlossLab/iss-35

Create demo of running the workflow on a larger github-actions runner

.github/workflows/run-snakemake.yml

@@ -0,0 +1,47 @@
+name: run-snakemake
+
+on: 
+  workflow_dispatch:
+    inputs:
+      configfile:
+        description: "Path to the config file"
+        required: true
+        default: 'config/test.yaml'
+        type: string
+
+jobs:
+  run-snakemake:
+    runs-on: ubuntu-latest-16-cores
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          python-version: 3.11
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          activate-environment: smk
+          environment-file: workflow/envs/smk.yml
+      - name: Run the snakemake workflow
+        uses: snakemake/[email protected]
+        with:
+          directory: .
+          snakefile: workflow/Snakefile
+          args: "--cores 16 --use-conda --conda-frontend mamba --conda-cleanup-pkgs cache  --show-failed-logs --configfile ${{ inputs.configfile }}"
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: log
+          path: log/
+      - name: Get current date
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+      - name: Upload archive
+        uses: actions/upload-artifact@v3
+        with:
+          name: workflow-results
+          path: workflow_*.zip
+          if-no-files-found: warn
+

.github/workflows/tests.yml

@@ -39,7 +39,7 @@ jobs:
         with:
           directory: .
           snakefile: workflow/Snakefile
-          args: "--use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache --all-temp --conda-frontend mamba --configfile config/test.yaml"
+          args: "--cores 2 --use-conda --conda-frontend mamba --conda-cleanup-pkgs cache  --show-failed-logs --all-temp --configfile config/test.yaml"
 #      - name: Test with pytest
 #        run: |
 #          pytest --workers 2 .tests/

.gitignore

@@ -14,3 +14,4 @@ __pycache__/
 figures/otu*
 results/otu*
 report_otu*
+*.zip

Dockerfile

@@ -1,6 +1,6 @@
 FROM condaforge/mambaforge:latest
 LABEL io.github.snakemake.containerized="true"
-LABEL io.github.snakemake.conda_env_hash="a3713ec99770d7dcc2bbdbe9b79e1a9f0c798f396d49f533486fb3193730b7b3"
+LABEL io.github.snakemake.conda_env_hash="d6e12a7cc311122c424028af8b7df4211faef9b3f3bc94298fd01affb1243862"
 
 # Step 1: Retrieve conda environments
 
@@ -41,20 +41,21 @@ COPY workflow/envs/mikropml.yml /conda-envs/67570867c99c9c3db185b41548ad6071/env
 
 # Conda environment:
 #   source: workflow/envs/smk.yml
-#   prefix: /conda-envs/9af38ac9af9135264be16dd13f279cd7
+#   prefix: /conda-envs/457b7b75191d44b96e5086432876e333
 #   name: smk
 #   channels:
 #     - conda-forge
 #     - bioconda
 #   dependencies:
 #     - snakemake=7
 #     - snakedeploy
-RUN mkdir -p /conda-envs/9af38ac9af9135264be16dd13f279cd7
-COPY workflow/envs/smk.yml /conda-envs/9af38ac9af9135264be16dd13f279cd7/environment.yaml
+#     - zip
+RUN mkdir -p /conda-envs/457b7b75191d44b96e5086432876e333
+COPY workflow/envs/smk.yml /conda-envs/457b7b75191d44b96e5086432876e333/environment.yaml
 
 # Step 2: Generate conda environments
 
 RUN mamba env create --prefix /conda-envs/b42323b0ffd5d034544511c9db1bdead --file /conda-envs/b42323b0ffd5d034544511c9db1bdead/environment.yaml && \
     mamba env create --prefix /conda-envs/67570867c99c9c3db185b41548ad6071 --file /conda-envs/67570867c99c9c3db185b41548ad6071/environment.yaml && \
-    mamba env create --prefix /conda-envs/9af38ac9af9135264be16dd13f279cd7 --file /conda-envs/9af38ac9af9135264be16dd13f279cd7/environment.yaml && \
+    mamba env create --prefix /conda-envs/457b7b75191d44b96e5086432876e333 --file /conda-envs/457b7b75191d44b96e5086432876e333/environment.yaml && \
     mamba clean --all -y

workflow/Snakefile

@@ -1,3 +1,4 @@
+import datetime
 import os
 from snakemake.utils import min_version
 import sys
@@ -50,7 +51,7 @@ report: "report/workflow.rst"
 
 rule targets:
     input:
-        f"report_{dataset}.md",
+        f"workflow_{dataset}.zip",
 
 
 rule render_report:
@@ -64,7 +65,7 @@ rule render_report:
         roc_plot=rules.plot_roc_curves.output.plot,
         rulegraph="figures/graphviz/rulegraph.png",
     output:
-        doc="report_{dataset}.md",
+        html="report_{dataset}.html",
     log:
         "log/{dataset}/render_report.txt",
     params:
@@ -79,3 +80,24 @@ rule render_report:
         "envs/mikropml.yml"
     script:
         "scripts/report.Rmd"
+
+
+rule archive:
+    input:
+        rules.render_report.input,
+        rules.render_report.output,
+        expand(
+            "results/{dataset}/{rtype}_results.csv",
+            dataset=dataset,
+            rtype=["performance", "feature-importance", "benchmarks", "sensspec"],
+        ),
+    output:
+        "workflow_{dataset}.zip",
+    log:
+        "log/archive_{dataset}.txt",
+    conda:
+        "envs/smk.yml"
+    shell:
+        """
+        zip -r {output} {input} 2> {log}
+        """

workflow/envs/smk.yml

@@ -5,3 +5,4 @@ channels:
 dependencies:
   - snakemake=7
   - snakedeploy
+  - zip

workflow/scripts/report.Rmd

@@ -2,8 +2,10 @@
 title: "ML Results"
 date: "`r Sys.Date()`"
 output:
-  github_document:
-    html_preview: false
+  html_document:
+    keep_md: true
+    self_contained: true
+    theme: spacelab
 ---
 
 ```{r setup, include=FALSE}
@@ -14,14 +16,16 @@ schtools::set_knitr_opts()
 library(knitr)
 ```
 
-Machine learning algorithms used were `r snakemake@params[['ml_methods']]`.
+Machine learning algorithm(s) used: `r snakemake@params[['ml_methods']]`.
 Models were trained with `r snakemake@params[['nseeds']]` different random
 partitions of the `r snakemake@params[['dataset']]` dataset into training and
 testing sets using `r snakemake@params[['kfold']]`-fold cross validation.
 See ``r snakemake@params[['config_path']]`` for the full configuration.
 
 ## Workflow
 
+`r snakemake@input[['rulegraph']]`
+
 ```{r rulegraph, out.width = '80%'}
 include_graphics(snakemake@input[['rulegraph']])
 ```
@@ -48,13 +52,13 @@ include_graphics(snakemake@input[['feat_plot']])
 
 ## Memory Usage & Runtime
 
+Each model training run was given `r snakemake@params[['ncores']]` cores
+for parallelization.
+
 ```{r runtime_plot, out.width='80%'}
 include_graphics(snakemake@input[['bench_plot']])
 ```
 
-Each model training run was given `r snakemake@params[['ncores']]` cores
-for parallelization.
-
 ## Hyperparameter Performance
 
 ```{r hp_plot, out.width='80%'}