feat: include muller plots from demonanalysis pkg (#52)

.dockerignore

@@ -0,0 +1,8 @@
+.git/
+*~
+.DS_Store
+._*
+.snakemake/
+tests/test1/output
+tests/test2/output
+.ipynb_checkpoints/

.github/workflows/changelog.yml

@@ -16,8 +16,8 @@ on:
 
 jobs:
   update-changelog:
-    name: ubuntu-20.04
-    runs-on: ubuntu-20.04
+    name: ubuntu-22.04
+    runs-on: ubuntu-22.04
 
     steps:
       - name: Checkout Code

.github/workflows/docker.yml

@@ -16,8 +16,8 @@ on:
 
 jobs:
   Docker:
-    name: ubuntu-20.04
-    runs-on: ubuntu-20.04
+    name: ubuntu-22.04
+    runs-on: ubuntu-22.04
 
     steps:
 

.github/workflows/pr.yml

@@ -11,12 +11,12 @@ on:
 jobs:
 
   assignee-reviewer:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - uses: kentaro-m/[email protected]
 
   conventional-title:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - uses: morrisoncole/[email protected]
         with:

.github/workflows/tag.yml

@@ -17,8 +17,8 @@ on:
 jobs:
 
   tag:
-    runs-on: ubuntu-20.04
-    name: ubuntu-20.04
+    runs-on: ubuntu-22.04
+    name: ubuntu-22.04
     if: github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'master'
 
     steps:

.github/workflows/test.yml

@@ -14,12 +14,10 @@ on: push
 jobs:
 
   pipeline-testruns:
-    runs-on: ${{ matrix.os }}
-    name: ${{ matrix.os }}
+    runs-on: ubuntu-22.04
+    name: ubuntu-22.04
     strategy:
       fail-fast: false
-      matrix:
-        os: [ubuntu-20.04, macos-11]
 
     steps:
 
@@ -46,140 +44,85 @@ jobs:
           conda list
           conda clean --all --yes
 
-      - name: Install md5sum (macOS)
-        shell: bash -l {0}
-        if: matrix.os == 'macos-11'
-        run: brew install md5sha1sum
-
       - name: Print compiler path (Ubuntu)
         shell: bash -l {0}
-        if: matrix.os == 'ubuntu-20.04'
         run: which g++
 
-      - name: Print compiler path (macOS)
-        shell: bash -l {0}
-        if: matrix.os == 'macos-11'
-        run: which clang++
-
       - name: Compile demon (Ubuntu)
         shell: bash -l {0}
-        if: matrix.os == 'ubuntu-20.04'
         run: >
             g++ resources/demon_model/src/demon.cpp
             -o resources/demon_model/bin/demon
             -I/usr/share/miniconda3/envs/warlock/include
             -lm
 
-      - name: Compile demon (macOS)
+      - name: Prepare internal conda envs (Ubuntu)
         shell: bash -l {0}
-        if: matrix.os == 'macos-11'
         run: >
-            clang++ resources/demon_model/src/demon.cpp
-            -o resources/demon_model/bin/demon
-            -I/Users/runner/miniconda3/envs/warlock/include
-            -lm
+            snakemake
+            --snakefile="workflow/Snakefile"
+            --configfile="tests/test1/config-Ubuntu.yml"
+            --use-conda
+            --conda-create-envs-only
+            --cores 1
+            all
+
+      - name: Install demonanalysis into the env (Ubuntu)
+        shell: bash -l {0}
+        run: |
+            for file in .snakemake/conda/*.yaml; do
+                if [ -f "$file" ]; then
+                    if grep -q "name: warlock-r" "$file"; then
+                        ENVPATH="${file%.yaml}"
+                        break
+                    fi
+                fi
+            done
+            conda deactivate
+            conda activate $ENVPATH
+            Rscript -e "devtools::install('resources/demonanalysis', upgrade=TRUE)" || true
+            Rscript -e "devtools::install('resources/demonanalysis', upgrade=TRUE)" || true
 
       # Run the tests sequentially;
       # This could be parallelized in the CI but setting up conda
       # is expensive; its faster to run the tests one-by-one.
 
       - name: Run the pipeline (test1) (Ubuntu)
         shell: bash -l {0}
-        if: matrix.os == 'ubuntu-20.04'
         run: >
             bash warlock.sh
             --configfile tests/test1/config-Ubuntu.yml
             --environment local
 
-      - name: Run the pipeline (test1) (macOS)
-        shell: bash -l {0}
-        if: matrix.os == 'macos-11'
-        run: >
-            bash warlock.sh
-            --configfile tests/test1/config-macOS.yml
-            --environment local
-
-      - name: Check md5 (test1) (Ubuntu)
-        shell: bash -l {0}
-        if: matrix.os == 'ubuntu-20.04'
-        working-directory: ./tests/test1
-        run: md5sum -c output-Ubuntu.md5
-
-      - name: Check md5 (test1) (macOS)
+      - name: Check outfiles (test1) (Ubuntu)
         shell: bash -l {0}
-        if: matrix.os == 'macos-11'
         working-directory: ./tests/test1
-        run: md5sum -c output-macOS.md5
+        run: >
+            md5sum -c output-Ubuntu.md5;
+            [ -e output/analyses/00000000/muller.txt ]
 
       - name: Cleanup (test1)
         shell: bash -l {0}
         run: rm -rf tests/test1/output
 
       - name: Run the pipeline (test2) (Ubuntu)
         shell: bash -l {0}
-        if: matrix.os == 'ubuntu-20.04'
         run: >
             bash warlock.sh
             --configfile tests/test2/config-Ubuntu.yml
             --environment local
             --cores 4
 
-      - name: Run the pipeline (test2) (macOS)
-        shell: bash -l {0}
-        if: matrix.os == 'macos-11'
-        run: >
-            bash warlock.sh
-            --configfile tests/test2/config-macOS.yml
-            --environment local
-            --cores 4
-
-      - name: Check md5 (test2) (Ubuntu)
-        shell: bash -l {0}
-        if: matrix.os == 'ubuntu-20.04'
-        working-directory: ./tests/test2
-        run: md5sum -c output-Ubuntu.md5
-
-      - name: Check md5 (test2) (macOS)
+      - name: Check outfiles (test2) (Ubuntu)
         shell: bash -l {0}
-        if: matrix.os == 'macos-11'
         working-directory: ./tests/test2
-        run: md5sum -c output-macOS.md5
+        run: |
+            md5sum -c output-Ubuntu.md5;
+            for dir in $(find output/analyses -mindepth 1 -type d); do
+                [ -e "$dir/muller.txt" ]
+            done
 
       - name: Cleanup (test2)
         shell: bash -l {0}
         run: rm -rf tests/test2/output
-
-      - name: Run the pipeline (test3) (Ubuntu)
-        shell: bash -l {0}
-        if: matrix.os == 'ubuntu-20.04'
-        run: >
-            bash warlock.sh
-            --configfile tests/test3/config-Ubuntu.yml
-            --environment local
-            --cores 2
-
-      - name: Run the pipeline (test3) (macOS)
-        shell: bash -l {0}
-        if: matrix.os == 'macos-11'
-        run: >
-            bash warlock.sh
-            --configfile tests/test3/config-macOS.yml
-            --environment local
-            --cores 2
-
-      - name: Check md5 (test3) (Ubuntu)
-        shell: bash -l {0}
-        if: matrix.os == 'ubuntu-20.04'
-        working-directory: ./tests/test3
-        run: md5sum -c output-Ubuntu.md5
-
-      - name: Check md5 (test3) (macOS)
-        shell: bash -l {0}
-        if: matrix.os == 'macos-11'
-        working-directory: ./tests/test3
-        run: md5sum -c output-macOS.md5
-
-      - name: Cleanup (test3)
-        shell: bash -l {0}
-        run: rm -rf tests/test3/output
 ...

.gitignore

@@ -11,8 +11,12 @@
 # notebook checkpoints
 .ipynb_checkpoints/
 
-# conda env
+# conda env at GitPod
 .warlock/
 
 # core demon model
 resources/demon_model
+
+# test outputs
+tests/test1/output
+tests/test2/output

Dockerfile

@@ -32,11 +32,13 @@ WORKDIR warlock
 COPY . .
 
 RUN apt-get update && apt-get upgrade -y \
+  && apt-get install -y locales \
   && apt-get autoremove -y \
   && apt-get clean -y \
-  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
+  && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
 
-##### INSTALL MAMBA + BUILD CONDA ENV #####
+##### INSTALL MAMBA + BUILD MAIN CONDA ENV #####
 RUN \
   conda install mamba -c conda-forge --yes \
   && mamba env create --file environment.yml \
@@ -50,3 +52,8 @@ RUN \
   -o resources/demon_model/bin/demon \
   -I /opt/conda/envs/warlock/include \
   -lm
+
+##### GENERATE INTERNAL ENVS & INSTALL DEPS #####
+RUN \
+  chmod +x prepare-environments.sh \
+  && bash -c 'CONDA_PREFIX="/opt/conda/envs/warlock" ./prepare-environments.sh'

README.md

@@ -1,7 +1,6 @@
 [![Contribute with Gitpod](https://img.shields.io/badge/Contribute%20with-Gitpod-908a85?logo=gitpod)](https://gitpod.io/#https://github.com/AngryMaciek/warlock)
 [![ci-pipeline](https://github.com/AngryMaciek/warlock/workflows/ci-pipeline/badge.svg)](https://github.com/AngryMaciek/warlock/actions?query=workflow%3Aci-pipeline)
 [![CodeFactor](https://www.codefactor.io/repository/github/angrymaciek/warlock/badge/master)](https://www.codefactor.io/repository/github/angrymaciek/warlock/overview/master)
-[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/AngryMaciek/warlock/master?labpath=notebook.ipynb)
 [![Docker](https://badgen.net/badge/icon/docker?icon=docker&label)](https://hub.docker.com/r/angrymaciek/warlock)
 [![GitHub issues](https://img.shields.io/github/issues/AngryMaciek/warlock)](https://github.com/AngryMaciek/warlock/issues)
 [![GitHub license](https://img.shields.io/github/license/AngryMaciek/warlock)](https://github.com/AngryMaciek/warlock/blob/master/LICENSE)
@@ -18,7 +17,7 @@ The following repository encapsulates _demon_ into an automated and reproducible
 
 ## Installation
 
-**The workflow is designed to run on Linux and macOS systems.**  
+**The workflow is designed to run on Linux systems.**  
 We have prepared a dedicated [conda environment](https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/environments.html) recipe which will contain all prerequisites required to execute the workflow. Thus Anaconda/Miniconda package manager is a natural dependency (see [Appendix A](#appendix-a-miniconda-installation) for installation instructions.)
 
 1. Clone the repository and navigate inside that directory
@@ -35,17 +34,17 @@ We have prepared a dedicated [conda environment](https://docs.conda.io/projects/
    conda activate warlock
    ```
 3. Compile _demon_
-   * Linux:
-     ```bash
-     g++ resources/demon_model/src/demon.cpp -o resources/demon_model/bin/demon -I$HOME/miniconda3/envs/warlock/include -lm
-     ```
-   * macOS:
-     ```bash
-     clang++ resources/demon_model/src/demon.cpp -o resources/demon_model/bin/demon -I$HOME/miniconda3/envs/warlock/include -lm
-     ```
+   ```bash
+   g++ resources/demon_model/src/demon.cpp -o resources/demon_model/bin/demon -I$HOME/miniconda3/envs/warlock/include -lm
+   ```
 
     > Note: remember to adjust `miniconda3` (and its path) in the command above, in case you have a different manager installed on your system. All in all, the point is to provide the _include_ directory of your `warlock` environment to the compiler.
 
+4. Create internal environments and install dependencies
+   ```bash
+   bash prepare-environments.sh
+   ```
+
 ## Configuration
 
 For a detailed description of all available simulation parameters please inspect [GitHub repository](https://github.com/robjohnnoble/demon_model) of the core _demon_ model.

VERSION

@@ -1 +1 @@
-1.1.4
+1.2.0

prepare-environments.sh

@@ -0,0 +1,39 @@
+###############################################################################
+#
+#   Create internal conda envs with all dependencies
+#
+#   AUTHOR: Maciej_Bak
+#   CONTACT: [email protected]
+#
+###############################################################################
+
+# exit at a first command that exits with a !=0 status
+set -eo pipefail
+
+$CONDA_PREFIX/bin/snakemake \
+    --snakefile="workflow/Snakefile" \
+    --configfile="tests/localtest/config-template.yml" \
+    --config workflow_repo_path="$PWD" workflow_analysis_outdir="$PWD/tests/localtest/output" \
+    --use-conda \
+    --conda-create-envs-only \
+    --conda-frontend=conda \
+    --cores 1 \
+    --nolock \
+    all
+
+for file in .snakemake/conda/*.yaml; do
+    if [ -f "$file" ]; then
+        if grep -q "name: warlock-r" "$file"; then
+            ENVPATH="${file%.yaml}"
+            break
+        fi
+    fi
+done
+
+eval "$(conda shell.bash hook)"
+conda deactivate
+conda activate $ENVPATH
+
+# strange error often appears, installation needs to be called twice
+Rscript -e "devtools::install('resources/demonanalysis', upgrade=TRUE)" || true
+Rscript -e "devtools::install('resources/demonanalysis', upgrade=TRUE)" || true