Merge branch 'master' into reimplement-HERA-datasets

.github/workflows/check_newcd.yml

@@ -0,0 +1,34 @@
+# A CI script with github workflow to test the new commondata
+name: Test new commondata
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  test-commondata:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-tags: true
+          fetch-depth: 0
+      - name: Install NNPDF data package 🐍
+        run: pip install ./nnpdf_data/'[filter]'
+      - name: Run the filters 📦
+        shell: bash -l {0}
+        run: |
+          here=$PWD
+          readarray -d '' array < <(find ./nnpdf_data/nnpdf_data/commondata -name "filter.py" -print0)
+          for datname in "${array[@]}"; do dirpath=${datname%/*}; cd $dirpath; python filter.py || exit $?; cd $here; done
+      - name: Check for modified files 🛎️
+        uses: tj-actions/verify-changed-files@v20
+        id: verify-changed-files
+      - name: List all changed files and fail if any 🛎️
+        if: steps.verify-changed-files.outputs.files_changed == 'true'
+        env:
+          CHANGED_FILES: ${{ steps.verify-changed-files.outputs.changed_files }}
+        run: |
+          echo "Changed files: $CHANGED_FILES"
+          exit 1

.github/workflows/fitbot.yml

@@ -10,57 +10,36 @@ on:
 env:
   N3FIT_MAXNREP: 20 # total number of replicas to fit
   POSTFIT_NREP: 16 # requested replicas for postfit
-  REFERENCE_SET: NNBOT-06f20cf6a-2024-07-26 # reference set for exact results
+  REFERENCE_SET: NNBOT-b1374cba6-2024-11-28 # reference set for exact results
   STABLE_REFERENCE_SET: NNBOT-c0f99b7b3-2024-02-28 # reference set for last tag
-  CONDA_PY: 312
   PYTHONHASHSEED: "0"
 
 jobs:
-  build:
+  fitbot:
     # perform build only if PR has run-fit-bot label
     if: contains(github.event.pull_request.labels.*.name, 'run-fit-bot')
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: ["3.12"]
-    runs-on: ${{ matrix.os }}
-    env:
-      NETRC_FILE: ${{ secrets.NETRC_FILE }}
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
       with:
         fetch-tags: true
         fetch-depth: 0
     - uses: conda-incubator/setup-miniconda@v3
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: "3.12"
         use-mamba: true
-        channels: https://packages.nnpdf.science/public,conda-forge
+        channels: conda-forge
         show-channel-urls: true
         auto-update-conda: true
-    - name: Install SSH key
-      uses: shimataro/ssh-key-action@v2
-      with:
-        key: ${{ secrets.NNPDF_SSA }}
-        name: id_rsa
-        known_hosts: ${{ secrets.NNPDF_HOSTS_KEY }}
-    - name: Setup conda and install conda-build
-      shell: bash -l {0}
-      run: |
-        echo "$NETRC_FILE" | base64 --decode > ~/.netrc
-        conda install conda-build --yes
-    - name: Build recipe
+        activate-environment: nnpdfenv
+    - name: Install lhapdf, pandoc and mongodb
       shell: bash -l {0}
       run: |
-        CONDA_PY=$CONDA_PY conda build --no-test -q conda-recipe
-    # install local build
-    - name: Installing NNPDF conda package
+        conda install lhapdf pandoc mongodb
+    - name: Install nnpdf with pip
       shell: bash -l {0}
       run: |
-        conda update -n base conda
-        conda install -n base conda-libmamba-solver
-        conda config --set solver libmamba
-        conda create -n nnpdfenv -c file:///usr/share/miniconda/envs/test/conda-bld/linux-64/ nnpdf
+        pip install .
     # create fancy runcards
     - name: Preparing fit runcard
       run: |
@@ -69,29 +48,32 @@ jobs:
         export TIMESTAMP=`date --iso-8601`
         echo "RUNCARD=NNBOT-$COMMIT-$TIMESTAMP" >> $GITHUB_ENV
         echo "RUNFOLDER=n3fit/runcards/examples" >> $GITHUB_ENV
-    # downloading theory and t0
+    # downloading theory, t0 and eko
     - name: Downloading requirements
       shell: bash -l {0}
       run: |
-        conda activate nnpdfenv
         cd $RUNFOLDER
         cp developing.yml $RUNCARD.yml
         vp-setupfit $RUNCARD.yml
-    # running n3fit replicas
+    # run n3fit replicas sequentially
     - name: Running n3fit
       shell: bash -l {0}
       run: |
-        conda activate nnpdfenv
         cd $RUNFOLDER
         for ((i=1; i<=$N3FIT_MAXNREP; i+=1)); do n3fit $RUNCARD.yml $i ; done
     # performing DGLAP
     - name: Running dglap
       shell: bash -l {0}
       run: |
-        conda activate nnpdfenv
         cd $RUNFOLDER
         evolven3fit evolve $RUNCARD
-    # running postfit
+    # running postfit and upload the fit to the server
+    - name: Install SSH key
+      uses: shimataro/ssh-key-action@v2
+      with:
+        key: ${{ secrets.NNPDF_SSA }}
+        name: id_rsa
+        known_hosts: ${{ secrets.NNPDF_HOSTS_KEY }}
     - name: Postfit and upload fit
       shell: bash -l {0}
       run: |

.github/workflows/python_installation.yml

@@ -43,4 +43,4 @@ jobs:
       shell: bash -l {0}
       run: |
         conda activate test
-        pytest --pyargs --mpl validphys n3fit
+        pytest --pyargs --mpl validphys n3fit --mpl-default-tolerance 18

.github/workflows/pytorch_test.yml

@@ -0,0 +1,25 @@
+name: Test pytorch
+
+on: [push]
+
+jobs:
+  run_pytorch:
+    runs-on: ubuntu-latest
+    env:
+      KERAS_BACKEND: torch
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+    - name: Install nnpdf without LHAPDF
+      shell: bash -l {0}
+      run: |
+        pip install .[nolha,torch]
+        # Since there is no LHAPDF in the system, initialize the folder and download pdfsets.index
+        lhapdf-management update --init
+    - name: Test we can run one runcard
+      shell: bash -l {0}
+      run: |
+        cd n3fit/runcards/examples
+        n3fit Basic_runcard.yml 4

.github/workflows/redo_regressions.yml

@@ -13,15 +13,7 @@ env:
 jobs:
   build:
     if: contains(github.event.pull_request.labels.*.name, 'redo-regressions')
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: ["3.11"]
-        include:
-          - os: ubuntu-latest
-            CONDA_OS: linux-64
-      fail-fast: false
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
     env:
       NETRC_FILE: ${{ secrets.NETRC_FILE }}
       NNPDF_SSH_KEY: ${{ secrets.NNPDF_SSH_KEY }}
@@ -30,10 +22,9 @@ jobs:
       with:
         fetch-depth: 0
         ref: ${{ github.event.pull_request.head.ref }}
-
     - uses: conda-incubator/setup-miniconda@v3
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: "3.12"
         use-mamba: true
         auto-update-conda: true
         activate-environment: test
@@ -58,6 +49,7 @@ jobs:
       shell: bash -l {0}
       run: |
         git config user.name "Redo regressions bot"
+        git config user.email "<>"
         git add extra_tests/regression_fits/*
         git status
         git commit -m "Automatically regenerated regressions from PR ${{ github.event.number }}, branch ${{ github.event.pull_request.head.ref }}."

.github/workflows/regenerate_data.yml

@@ -0,0 +1,37 @@
+# CI script with github workflow to regenerate the data
+name: Regenerate Commondata
+
+on:
+  pull_request:
+    types: [labeled]
+
+jobs:
+  build:
+    if: contains(github.event.pull_request.labels.*.name, 'regenerate-data')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.ref }}
+      - name: Install NNPDF data package 🐍
+        run: pip install ./nnpdf_data/'[filter]'
+      - name: Run the filters 📦
+        shell: bash -l {0}
+        run: |
+          here=$PWD
+          readarray -d '' array < <(find ./nnpdf_data/nnpdf_data/commondata -name "filter.py" -print0)
+          for datname in "${array[@]}"; do dirpath=${datname%/*}; cd $dirpath; python filter.py || exit $?; cd $here; done
+      - name: Check for modified files 🛎️
+        uses: tj-actions/verify-changed-files@v20
+        id: verify-changed-files
+      - name: Commit the changed files 🛎️
+        if: steps.verify-changed-files.outputs.files_changed == 'true'
+        shell: bash -l {0}
+        run: |
+          git config user.name "Regenerate Commondata"
+          git config user.email "<>"
+          git add nnpdf_data/nnpdf_data/commondata/*
+          git status
+          git commit -m "Automatically regenerated commondata from PR ${{ github.event.number }}, branch ${{ github.event.pull_request.head.ref }}."
+          git push origin "${{ github.event.pull_request.head.ref }}"

.github/workflows/regression_tests.yml

@@ -4,27 +4,20 @@ on: [push]
 
 env:
   # https://keras.io/getting_started/faq/#how-can-i-obtain-reproducible-results-using-keras-during-development
+  # However, results might change between different versions of keras.
   PYTHONHASHSEED: "0"
 
 jobs:
   build:
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: ["3.11"]
-        include:
-          - os: ubuntu-latest
-            CONDA_OS: linux-64
-      fail-fast: false
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
     env:
       NETRC_FILE: ${{ secrets.NETRC_FILE }}
       NNPDF_SSH_KEY: ${{ secrets.NNPDF_SSH_KEY }}
     steps:
     - uses: actions/checkout@v4
     - uses: conda-incubator/setup-miniconda@v3
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: "3.12"
         use-mamba: true
         auto-update-conda: true
         activate-environment: test

.github/workflows/tests.yml

@@ -12,10 +12,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-14]
-        python-version: ["3.12"] # Test only the latest python in all system
-        include:
-          - os: ubuntu-latest # Test also the oldest python supported in ubuntu
-            python-version: "3.9"
+        python-version: ["3.12"]
       fail-fast: false
     runs-on: ${{ matrix.os }}
     env:
@@ -45,7 +42,7 @@ jobs:
       run: |
         conda build -q conda-recipe
     - name: Upload noarch conda package to NNPDF server
-      if: ${{ (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && startsWith(matrix.os, 'ubuntu') && matrix.python-version == '3.9' }}
+      if: ${{ (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && startsWith(matrix.os, 'ubuntu') }}
       shell: bash -l {0}
       run: |
         KEY=$( mktemp )
@@ -54,7 +51,7 @@ jobs:
           $CONDA_PREFIX/conda-bld/noarch/*.tar.bz2 \
           [email protected]:~/packages/conda/noarch
     - name: Build and upload sphinx documentation to NNPDF server
-      if: startsWith(matrix.os, 'ubuntu') && github.ref == 'refs/heads/master' && matrix.python-version == '3.9'
+      if: startsWith(matrix.os, 'ubuntu') && github.ref == 'refs/heads/master'
       shell: bash -l {0}
       run: |
         KEY=$( mktemp )

.pre-commit-config.yaml

@@ -1,24 +1,24 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: 'v4.5.0'
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: 'v5.0.0'
     hooks:
-      - id: check-merge-conflict
-      - id: check-toml
-      - id: check-yaml
-      - id: debug-statements
-      - id: end-of-file-fixer
-      - id: trailing-whitespace
+    -   id: check-merge-conflict
+    -   id: check-toml
+    -   id: check-yaml
+    -   id: debug-statements
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
 
-  - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: '24.3.0'
+-   repo: https://github.com/psf/black-pre-commit-mirror
+    rev: '24.10.0'
     hooks:
-      - id: black
+    -   id: black
         args: ['--config=./pyproject.toml']
 
-  - repo: https://github.com/pycqa/isort
+-   repo: https://github.com/pycqa/isort
     rev: '5.13.2'
     hooks:
-      - id: isort
+    -   id: isort
         args: ['--settings-path=./pyproject.toml']

conda-recipe/conda_build_config.yaml

@@ -1,7 +1,7 @@
 #For some reason the resolver decides to use an old version of numpy
 #without this
 numpy:
-    - 1.24
+    - 1.26
 
 pin_run_as_build:
     lhapdf: x.x.x

conda-recipe/meta.yaml

@@ -19,17 +19,19 @@ requirements:
         - pip
     run:
         - python >=3.9,<3.13
-        - tensorflow >=2.10,<2.17 # 2.17 works ok but the conda-forge package for macos doesn't
+        - tensorflow >=2.17
+        - keras >=3.1
         - psutil # to ensure n3fit affinity is with the right processors
         - hyperopt
         - mongodb
         - pymongo <4
+        - pyopenssl >=23.2
         - seaborn
         - lhapdf
         - numpy
         - pkg-config
         - reportengine
-        - matplotlib >=3.3.0,<3.8  # see https://github.com/NNPDF/nnpdf/pull/1809
+        - matplotlib >=3.9
         - blessings >=1.7
         - scipy >=0.19.1
         - pandas
@@ -39,11 +41,11 @@ requirements:
         - pineappl >=0.8.2
         - eko >=0.14.2
         - fiatlux
-        - sphinx >=5.0.2,<6 # documentation. Needs pinning temporarily due to markdown
-        - recommonmark
+        - sphinx >=5.0.2
+        - joblib
         - sphinx_rtd_theme >0.5
         - sphinxcontrib-bibtex
-        - ruamel.yaml <0.18
+        - ruamel.yaml >=0.15
 
 test:
     requires: