Add support for build binary installers for each release via conda constructor

.github/workflows/ci.yml

@@ -522,7 +522,7 @@ jobs:
         cmake -A x64 -DRI_BUILD_FULL_INSTALLER:BOOL=ON ${ROBOTOLOGY_SUPERBUILD_SOURCE_DIR}/packaging/windows
         cmake --build . --config Release --target PACKAGE
         # Move installer in installer directory with a fixed name
-        mv *.exe /c/robotology-full-installer-win64.exe
+        mv *.exe /c/robotology-full-installer-vcpkg-win64.exe
         # Cleanup and build dependencies installer
         cd /c
         rm -rf /c/build-installer-full
@@ -531,7 +531,7 @@ jobs:
         cmake -A x64 -DRI_BUILD_FULL_INSTALLER:BOOL=OFF ${ROBOTOLOGY_SUPERBUILD_SOURCE_DIR}/packaging/windows
         cmake --build . --config Release --target PACKAGE
         # Move installer in installer directory with a fixed name
-        mv *.exe /c/robotology-dependencies-installer-win64.exe
+        mv *.exe /c/robotology-dependencies-installer-vcpkg-win64.exe
 
     - name: Upload Full Installer [Windows]
       if: github.event_name == 'release' && matrix.project_tags == 'Default' && matrix.os == 'windows-2019'
@@ -540,8 +540,8 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}
       with:
           upload_url: ${{ github.event.release.upload_url }}
-          asset_path: C:/robotology-full-installer-win64.exe
-          asset_name: robotology-full-installer-win64.exe
+          asset_path: C:/robotology-full-installer-vcpkg-win64.exe
+          asset_name: robotology-full-installer-vcpkg-win64.exe
           asset_content_type: application/octet-stream
 
     - name: Upload Dependencies installer  [Windows]
@@ -551,6 +551,6 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}
       with:
           upload_url: ${{ github.event.release.upload_url }}
-          asset_path: C:/robotology-dependencies-installer-win64.exe
-          asset_name: robotology-dependencies-installer-win64.exe
+          asset_path: C:/robotology-dependencies-installer-vcpkg-win64.exe
+          asset_name: robotology-dependencies-installer-vcpkg-win64.exe
           asset_content_type: application/octet-stream

.github/workflows/generate-conda-packages.yaml

@@ -99,29 +99,45 @@ jobs:
         - name: Dependencies for conda recipes generation and upload
           shell: bash -l {0}
           run: |
-            mamba install pyyaml jinja2 conda-build ninja anaconda-client conda-forge-pinning mamba boa multisheller
+            mamba install pyyaml jinja2 conda-build ninja anaconda-client conda-forge-pinning mamba boa multisheller constructor
 
         - name: Print used environment
           shell: bash -l {0}
           run: |
             mamba list
             env
 
+        - name: Fail if both upload_conda_binaries and test_metapackages_generation are passed
+          if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true' && github.event.inputs.upload_conda_binaries == 'true'
+          uses: actions/github-script@v3
+          with:
+            script: |
+              core.setFailed('Build failed. It is not possible to set to true both upload_conda_binaries and test_metapackages_generation.')
+
+        - name: If test_metapackages_generation is enabled set an arbitrary version number
+          if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true'
+          shell: bash -l {0}
+          run: |
+            sed -i -e 's/set(INSTALLER_VERSION "")/set(INSTALLER_VERSION 1234.0)/g' ./packaging/windows/CMakeLists.txt
+            sed -i -e 's/set(CONDA_ROBOTOLOGY_SUPERBUILD_VERSION "")/set(CONDA_ROBOTOLOGY_SUPERBUILD_VERSION 1234.0)/g' ./conda/cmake/CondaGenerationOptions.cmake
+
         - name: Generate recipes [Linux&macOS]
           if: contains(matrix.os, 'macos') || contains(matrix.os, 'ubuntu')
           shell: bash -l {0}
           run: |
             mkdir build
             cd build
-            cmake -GNinja -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON -DCONDA_BUILD_NUMBER=${{needs.get-conda-build-number.outputs.conda_build_number}} ..
+            # Removed -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake for faster tests
+            cmake -GNinja -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON -DCONDA_BUILD_NUMBER=${{needs.get-conda-build-number.outputs.conda_build_number}} ..
 
         - name: Generate recipes [Windows]
           if: contains(matrix.os, 'windows')
           shell: bash -l {0}
           run: |
             mkdir build
             cd build
-            cmake -G"Visual Studio 16 2019" -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake  -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON -DCONDA_BUILD_NUMBER=${{needs.get-conda-build-number.outputs.conda_build_number}}  ..
+            # Removed -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake for faster tests
+            cmake -G"Visual Studio 16 2019"  -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON -DCONDA_BUILD_NUMBER=${{needs.get-conda-build-number.outputs.conda_build_number}}  ..
 
         - name: Specify additional option if we are in a release and we need to generate robotology-distro metapackages
           if: github.event_name == 'release'
@@ -137,6 +153,7 @@ jobs:
             cd build
             cmake -DCONDA_GENERATE_ROBOTOLOGY_METAPACKAGES:BOOL=ON .
 
+
         - name: Fail if both upload_conda_binaries and test_metapackages_generation are passed
           if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true' && github.event.inputs.upload_conda_binaries == 'true'
           uses: actions/github-script@v3
@@ -169,30 +186,9 @@ jobs:
             # specifically because we can't use --skip-existing as these packages are part of the robotology channel
             conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml icub-contrib-common
             rm -rf icub-contrib-common
-            conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml icub-firmware-shared
-            rm -rf icub-firmware-shared
-            conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml robots-configuration
-            rm -rf robots-configuration
             # icub-main needs robotology channel as it also depends on esdcan, and local to avoid that robotology packages are used instead of local built one
             conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge -c robotology icub-main
             rm -rf icub-main
-            conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml wearables
-            rm -rf wearables
-            # Channels specified explicitly due to https://github.com/robotology/robotology-superbuild/issues/1140
-            conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge human-dynamics-estimation
-            rm -rf human-dynamics-estimation
-            conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml robot-log-visualizer
-            rm -rf robot-log-visualizer
-            conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml idyntree-matlab-bindings
-            rm -rf idyntree-matlab-bindings
-            conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml blockfactory
-            rm -rf blockfactory
-            conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml yarp-matlab-bindings
-            rm -rf yarp-matlab-bindings
-            conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml wb-toolbox
-            rm -rf wb-toolbox
-            conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml blocktest
-            rm -rf blocktest
             conda build -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml .
 
         - name: Build conda metapackages (if necessary)
@@ -205,16 +201,56 @@ jobs:
             conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge -c robotology robotology-distro
             conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge -c robotology robotology-distro-all
 
-        - name: Upload conda packages
+        # - name: Upload conda packages
+        #  shell: bash -l {0}
+        #  # Upload by default on schedule events, and on workflow dispatch only if input upload_conda_binaries is 'true'
+        #  if: github.event_name == 'schedule' || github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true')
+        #  env:
+        #    ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
+        #  run: |
+        #    cd ${CONDA_PREFIX}/conda-bld/${{ matrix.conda_platform}}/
+        #    ls *.tar.bz2
+        #    anaconda upload --skip-existing *.tar.bz2
+
+        - name: Generated installer with all packages
           shell: bash -l {0}
-          # Upload by default on schedule events, and on workflow dispatch only if input upload_conda_binaries is 'true'
-          if: github.event_name == 'schedule' || github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true')
-          env:
-            ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
+          if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true')
           run: |
-            cd ${CONDA_PREFIX}/conda-bld/${{ matrix.conda_platform}}/
-            ls *.tar.bz2
-            anaconda upload --skip-existing *.tar.bz2
+            cd build/conda/generated_constructor_recipe
+            constructor .
+
+        - name: Upload Full Installer [Windows]
+          if: github.event_name == 'release' && contains(matrix.os, 'windows')
+          uses: actions/[email protected]
+          env:
+            GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+          with:
+            upload_url: ${{ github.event.release.upload_url }}
+            asset_path: build/conda/generated_constructor_recipe/robotology-distro-windows-x86_64.exe
+            asset_name: robotology-distro-windows-x86_64.exe
+            asset_content_type: application/octet-stream
+
+        - name: Upload Full Installer [macOS]
+          if: github.event_name == 'release' && contains(matrix.os, 'macos')
+          uses: actions/[email protected]
+          env:
+            GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+          with:
+            upload_url: ${{ github.event.release.upload_url }}
+            asset_path: build/conda/generated_constructor_recipe/robotology-distro-macos-x86_64.sh
+            asset_name: robotology-distro-macos-x86_64.sh
+            asset_content_type: application/octet-stream
+
+        - name: Upload Full Installer [Linux]
+          if: github.event_name == 'release' && contains(matrix.os, 'ubuntu')
+          uses: actions/[email protected]
+          env:
+            GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+          with:
+            upload_url: ${{ github.event.release.upload_url }}
+            asset_path: build/conda/generated_constructor_recipe/robotology-distro-linux-x86_64.sh
+            asset_name: robotology-distro-linux-x86_64.sh
+            asset_content_type: application/octet-stream
 
     # If the  generate-conda-packages completed correctly and binaries are uploaded,
     # bump automatically the CONDA_BUILD_NUMBER in conda/cmake/CondaGenerationOptions.cmake

cmake/RobotologySuperbuildOptions.cmake

@@ -71,7 +71,7 @@ if(NOT CMAKE_CONFIGURATION_TYPES)
 endif()
 
 set(ROBOTOLOGY_PROJECT_TAGS "Stable" CACHE STRING "The tags to be used for the robotology projects: Stable, Unstable, LatestRelease or Custom. This can be changed only before the first configuration.")
-set(ROBOTOLOGY_PROJECT_TAGS_CUSTOM_FILE CACHE FILEPATH "If ROBOTOLOGY_PROJECT_TAGS is custom, this file will be loaded to specify the tags of the projects to use.")
+set(ROBOTOLOGY_PROJECT_TAGS_CUSTOM_FILE "${PROJECT_SOURCE_DIR}/releases/latest.releases.yaml" CACHE FILEPATH "If ROBOTOLOGY_PROJECT_TAGS is custom, this file will be loaded to specify the tags of the projects to use.")
 set_property(CACHE ROBOTOLOGY_PROJECT_TAGS PROPERTY STRINGS "Stable" "Unstable" "LatestRelease" "Custom")
 
 if(ROBOTOLOGY_PROJECT_TAGS STREQUAL "Stable")

conda/cmake/RobotologySuperbuildGenerateCondaRecipes.cmake

@@ -249,7 +249,16 @@ macro(generate_metametadata_file)
       string(APPEND metametadata_file_contents "    - name: ${${_cmake_pkg}_CONDA_PKG_NAME}\n")
       string(APPEND metametadata_file_contents "      version: \"${${_cmake_pkg}_CONDA_VERSION}\"\n")
     endforeach()
-
+
+    # If metapackages are enabled, we also generated the constructor file in ${CMAKE_CURRENT_BINARY_DIR}/conda/generated_constructor_recipe
+    set(generated_constructor_recipe_dir ${CMAKE_CURRENT_BINARY_DIR}/conda/generated_constructor_recipe)
+    file(MAKE_DIRECTORY ${generated_constructor_recipe_dir})
+    if(WIN32)
+      set(CONDA_BUILD_CHANNEL "file:///$ENV{CONDA_PREFIX}/conda-bld")
+    else()
+      set(CONDA_BUILD_CHANNEL "file://$ENV{CONDA_PREFIX}/conda-bld")
+    endif()
+    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conda/constructor_template/construct.yaml.in ${generated_constructor_recipe_dir}/construct.yaml)
   endif()
 
   file(WRITE ${metametadata_file} ${metametadata_file_contents})

conda/constructor_template/construct.yaml.in

@@ -0,0 +1,22 @@
+name: robotology-distro
+version: @CONDA_ROBOTOLOGY_SUPERBUILD_VERSION@
+
+installer_filename: robotology-distro-macos-x86_64.sh  # [osx]
+installer_filename: robotology-distro-linux-x86_64.sh  # [linux]
+installer_filename: robotology-distro-windows-x86_64.exe  # [win]
+
+channels_remap:
+  -
+      src: @CONDA_BUILD_CHANNEL@
+      dest: robotology
+
+channels:
+  - @CONDA_BUILD_CHANNEL@
+  - conda-forge
+  - robotology
+
+specs:
+  - conda
+  - mamba
+  - robotology-distro-all @CONDA_ROBOTOLOGY_SUPERBUILD_VERSION@
+

packaging/windows/CMakeLists.txt

@@ -11,11 +11,11 @@ option(RI_INCLUDE_GAZEBO "If ON include the Gazebo simulator binaries." ON)
 option(RI_BUILD_FULL_INSTALLER "If ON build the full installer, otherwise the dependencies one." ON)
 
 if(RI_BUILD_FULL_INSTALLER)
-  project(robotology-full-installer
+  project(robotology-full-installer-vcpkg
           VERSION ${INSTALLER_VERSION}
           DESCRIPTION "Installer for robotology software and its dependencies.")
 else()
-  project(robotology-dependencies-installer
+  project(robotology-dependencies-installer-vcpkg
           VERSION ${INSTALLER_VERSION}
           DESCRIPTION "Installer for robotology software dependencies.")
 endif()