Windows CI:

* build with VS2019 * add support for github actions * also produce packages as artifacts * move test blacklist to .ci/windows-blacklist.txt (added qgis_layoutmanualtabletest and PyQgsVirtualLayerDefinition) * separate step to output dash link * include action badge * drop unused caching (and clcache) * update INSTALL
jef-n · May 18, 2020 · 7401e79 · 7401e79
1 parent db50e77
commit 7401e79
Show file tree

Hide file tree

Showing 11 changed files with 421 additions and 161 deletions.
diff --git a/.ci/azure-pipelines/azure-pipelines.yml b/.ci/azure-pipelines/azure-pipelines.yml
@@ -1,7 +1,6 @@
 variables:
   LR: release-3_12
   LTR: release-3_10
-  CTEST_CUSTOM_TESTS_IGNORE: "ProcessingGdalAlgorithmsRasterTest;ProcessingGdalAlgorithmsVectorTest;ProcessingGrass7AlgorithmsImageryTest;ProcessingGrass7AlgorithmsRasterTest;ProcessingGrass7AlgorithmsVectorTest;ProcessingGuiTest;ProcessingOtbAlgorithmsTest;ProcessingQgisAlgorithmsTestPt1;ProcessingQgisAlgorithmsTestPt2;ProcessingQgisAlgorithmsTestPt3;ProcessingQgisAlgorithmsTestPt4;ProcessingScriptUtilsTest;PyQgsAnnotation;PyQgsAppStartup;PyQgsAuthManagerOAuth2OWSTest;PyQgsAuthManagerPasswordOWSTest;PyQgsAuthManagerPKIOWSTest;PyQgsAuthManagerProxy;PyQgsAuthSettingsWidget;PyQgsAuxiliaryStorage;PyQgsBlockingNetworkRequest;PyQgsExifTools;PyQgsFileDownloader;PyQgsFileUtils;PyQgsGeometryTest;PyQgsImageCache;PyQgsImportIntoPostGIS;PyQgsLayoutAtlas;PyQgsLayoutLegend;PyQgsLayoutMap;PyQgsLayoutMapGrid;PyQgsMapLayer;PyQgsOfflineEditingWFS;PyQgsOGRProvider;PyQgsOGRProviderGpkg;PyQgsOGRProviderSqlite;PyQgsPalLabelingCanvas;PyQgsPalLabelingLayout;PyQgsPalLabelingPlacement;PyQgsPointDisplacementRenderer;PyQgsProject;PyQgsProviderConnectionGpkg;PyQgsProviderConnectionPostgres;PyQgsPythonProvider;PyQgsRasterFileWriter;PyQgsRasterLayer;PyQgsSelectiveMasking;PyQgsServerAccessControlWMSGetlegendgraphic;PyQgsServerApi;PyQgsServerCacheManager;PyQgsServerLocaleOverride;PyQgsServerSecurity;PyQgsServerSettings;PyQgsServerWMS;PyQgsServerWMSDimension;PyQgsServerWMSGetFeatureInfo;PyQgsServerWMSGetLegendGraphic;PyQgsServerWMSGetMap;PyQgsServerWMSGetPrint;PyQgsServerWMTS;PyQgsSettings;PyQgsShapefileProvider;PyQgsSpatialiteProvider;PyQgsSvgCache;PyQgsSymbolLayer;PyQgsTaskManager;PyQgsTextRenderer;PyQgsVectorFileWriter;PyQgsVectorLayer;PyQgsVectorLayerUtils;PyQgsVirtualLayerProvider;PyQgsWFSProviderGUI;PyQgsZipUtils;qgis_3drenderingtest;qgis_alignrastertest;qgis_arcgisrestutilstest;qgis_banned_keywords;qgis_browsermodeltest;qgis_callouttest;qgis_compositionconvertertest;qgis_coordinatereferencesystemtest;qgis_datadefinedsizelegendtest;qgis_datumtransformdialog;qgis_diagramtest;qgis_doxygen_order;qgis_dxfexporttest;qgis_expressiontest;qgis_filedownloader;qgis_geometrycheckstest;qgis_geometrytest;qgis_geonodeconnectiontest;qgis_grassprovidertest7;qgis_imagecachetest;qgis_invertedpolygonrenderertest;qgis_labelingenginetest;qgis_layerdefinitiontest;qgis_layout3dmaptest;qgis_layouthtmltest;qgis_layoutlabeltest;qgis_layoutmapgridtest;qgis_layoutmaptest;qgis_layoutpicturetest;qgis_layoutscalebartest;qgis_layouttabletest;qgis_legendrenderertest;qgis_licenses;qgis_defwindowtitle;qgis_maprendererjobtest;qgis_maprotationtest;qgis_mapsettingsutilstest;qgis_maptooladdfeatureline;qgis_mimedatautilstest;qgis_networkaccessmanagertest;qgis_openclutilstest;qgis_painteffecttest;qgis_pallabelingtest;qgis_processingtest;qgis_projecttest;qgis_qgisappclipboard;qgis_rasterlayersaveasdialog;qgis_shellcheck;qgis_sipify;qgis_sip_include;qgis_sip_uptodate;qgis_spelling;qgis_styletest;qgis_svgcachetest;qgis_taskmanagertest;qgis_transformdialog;qgis_vectorfilewritertest;qgis_wcsprovidertest;qgis_ziplayertest;qgis_meshcalculator;qgis_pointlocatortest;PyQgsExpressionBuilderWidget;PyQgsDatumTransform;qgis_vertextool;PyQgsCoordinateOperationWidget;PyQgsProviderConnectionSpatialite;qgis_maptoolsplitpartstest;qgis_vectortilelayertest"
   Agent.Source.Git.ShallowFetchDepth: 120
 
 trigger:
@@ -20,26 +19,19 @@ pr:
 jobs:
 - job: OSGeo4W
   pool:
-    vmImage: vs2017-win2016
+    vmImage: windows-latest
   timeoutInMinutes: 360
+
   strategy:
     maxParallel: 4
     matrix:
       x86:
-         OSGEO4W_ROOT: C:\OSGeo4W
          OSGEO4W_ARCH: x86
-         CLCACHE_DIR: c:\clcache-x86
-         PLATFORM: x86
-         CC: C:\OSGeo4W\bin\clcache.bat
-         CXX: C:\OSGeo4W\bin\clcache.bat
+         OSGEO4W_ROOT: C:\OSGeo4W
 
       x86_64:
-         OSGEO4W_ROOT: C:\OSGeo4W64
          OSGEO4W_ARCH: x86_64
-         CLCACHE_DIR: c:\clcache-x86_64
-         PLATFORM: x64
-         CC: C:\OSGeo4W64\bin\clcache.bat
-         CXX: C:\OSGeo4W64\bin\clcache.bat
+         OSGEO4W_ROOT: C:\OSGeo4W64
 
   steps:
   - bash: |
@@ -51,7 +43,7 @@ jobs:
       fi
 
       echo "BRANCH: ${branch}"
-      echo "PR: ${pr}"
+      [ -n "$pr" ] && echo "PR: ${pr}"
       echo "LR: ${LR}"
       echo "LTR: ${LTR}"
 
@@ -74,23 +66,29 @@ jobs:
       major=$(sed -ne 's/^SET(CPACK_PACKAGE_VERSION_MAJOR "\([0-9]*\)")\s*$/\1/p' CMakeLists.txt)
       minor=$(sed -ne 's/^SET(CPACK_PACKAGE_VERSION_MINOR "\([0-9]*\)")\s*$/\1/p' CMakeLists.txt)
       patch=$(sed -ne 's/^SET(CPACK_PACKAGE_VERSION_PATCH "\([0-9]*\)")\s*$/\1/p' CMakeLists.txt)
-      binary=$(curl --location-trusted http://ftp.osuosl.org/pub/osgeo/download/osgeo4w/$OSGEO4W_ARCH/release/qgis/$OSGEO4W_PKG/LATEST.sha | sed -e "s/:.*$//")
-      (( binary++ )) || true
-
       version=$major.$minor.$patch
       sha="${BUILD_SOURCEVERSION:0:10}"
 
-      if [ "$BUILD_REASON" = "PullRequest" ]; then
-        buildname="PR $pr / $branch ($BUILD_BUILDID) ($sha) ($OSGEO4W_PKG $target $OSGEO4W_ARCH)"   # no colons allowed here
+      if [ -z "$pr" ]; then
+        binary=$(curl --location-trusted http://download.osgeo.org/osgeo4w/$OSGEO4W_ARCH/release/qgis/$OSGEO4W_PKG/LATEST.sha | sed -e "s/:.*$//")
+        (( binary++ )) || true
+        buildname="$OSGEO4W_PKG-$version-$sha-$target-VC16-$OSGEO4W_ARCH"
       else
-        buildname="$OSGEO4W_PKG-$version-$sha-$target-VC14-$OSGEO4W_ARCH"
+        OSGEO4W_PKG="${OSGEO4W_PKG}-${pr}"
+        buildname="PR $pr / $branch ($BUILD_BUILDID) ($sha) ($OSGEO4W_PKG $target $OSGEO4W_ARCH)"   # no colons allowed here
+        binary=${BUILD_BUILDID}
+      fi
+
+      if [ "${BUILD_REPOSITORY_URI#https://github.com/}" != "qgis/QGIS" ]; then
+        buildname="${BUILD_REPOSITORY_URI#https://github.com/}: $buildname"
       fi
 
       url=$buildname
       url=${url//(/%28}
       url=${url//)/%29}
       url=${url// /+}
       url="https://cdash.orfeo-toolbox.org/index.php?project=QGIS&filtercombine=and&filtercombine=and&filtercombine=and&filtercombine=and&filtercombine=and&filtercombine=and&filtercombine=and&filtercombine=and&filtercombine=and&filtercombine=and&filtercombine=and&filtercombine=and&filtercombine=and&filtercombine=and&filtercombine=and&filtercombine=and&filtercount=4&showfilters=0&filtercombine=and&field1=buildname&compare1=61&value1=$url&field2=site&compare2=65&value2=azure-pipelines&field3=buildstarttime&compare3=83&value3=$(date +%Y-%m-%d --date=yesterday)&field4=buildstarttime&compare4=84&value4=$(date +%Y-%m-%d --date=tomorrow)"
+      url=$(curl --data-urlencode "url=$url" -s http://tinyurl.com/api-create.php)
 
       echo "##vso[task.setvariable variable=TARGET]$target"
       echo "##vso[task.setvariable variable=OSGEO4W_PKG]$OSGEO4W_PKG"
@@ -101,7 +99,8 @@ jobs:
       echo "##vso[task.setvariable variable=BINARY]$binary"
       echo "##vso[task.setvariable variable=VERSION]$version"
       echo "##vso[task.setvariable variable=BUILDNAME]$buildname"
-      echo "##vso[task.setvariable variable=DASHURL]${url//&/^&}"
+      echo "##vso[task.setvariable variable=DASHURL]$url"
+      echo "##vso[task.setvariable variable=CTEST_CUSTOM_TESTS_IGNORE]$(sed -r '/^(#.*?)?$/d' .ci/windows-blacklist.txt | paste -sd ';' -)"
 
     displayName: 'Setup build variables'
 
@@ -111,95 +110,41 @@ jobs:
   - script: curl --output c:\osgeo4w-setup.exe http://ftp.osuosl.org/pub/osgeo/download/osgeo4w/osgeo4w-setup-%OSGEO4W_ARCH%.exe
     displayName: 'Download OSGeo4W Installer'
 
-  - script: curl --location-trusted --output c:\ninja.zip https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-win.zip
+  - script: curl --location-trusted --output c:\ninja.zip https://github.com/ninja-build/ninja/releases/download/v1.10.0/ninja-win.zip
     displayName: 'Download Ninja'
 
-#  - script: curl --location-trusted --output c:\depends.zip http://www.dependencywalker.com/depends22_%PLATFORM%.zip
-#    displayName: 'Download Dependency walker'
-
-# Too large…
-#  - task: Cache@2
-#    inputs:
-#      key: 'cygwin | $(Date:yyyyMMdd)'
-#      path: 'c:\cygwin'
-#      restoreKeys: |
-#        cygwin | $(Date:yyyyMM)
-#        cygwin | $(Date:yyyy)
-#        cygwin
-#    displayName: Cache cygwin
-
   - powershell: ms-windows/osgeo4w/runasadmin.ps1 c:\setup-x86.exe -qnNdO -R C:/cygwin -s http://cygwin.mirror.constant.com -l C:/temp/cygwin -P "bison,flex,git,poppler,doxygen,unzip"
     displayName: 'Installing cygwin'
 
-# Too large…
-#  - task: Cache@2
-#    inputs:
-#      key: 'osgeo4w | $(OSGEO4W_ARCH) | $(Date:yyyyMMdd)'
-#      path: '$(OSGEO4W_ROOT)'
-#      restoreKeys: |
-#        osgeo4w | $(OSGEO4W_ARCH) | $(Date:yyyyMMdd)
-#        osgeo4w | $(OSGEO4W_ARCH) | $(Date:yyyyMM)
-#        osgeo4w | $(OSGEO4W_ARCH) | $(Date:yyyy)
-#        osgeo4w | $(OSGEO4W_ARCH)
-#    displayName: Cache OSGeo4W
-
-  - powershell: ms-windows/osgeo4w/runasadmin.ps1 c:\osgeo4w-setup.exe --autoaccept --advanced --arch $env:OSGEO4W_ARCH --quiet-mode --upgrade-also --root $env:OSGEO4W_ROOT --only-site -s http://ftp.osuosl.org/pub/osgeo/download/osgeo4w -l c:\temp\osgeo4w -P $env:OSGEO4W_DEPS -P python3-clcache
+  - powershell: ms-windows/osgeo4w/runasadmin.ps1 c:\osgeo4w-setup.exe --autoaccept --advanced --arch $env:OSGEO4W_ARCH --quiet-mode --upgrade-also --root $env:OSGEO4W_ROOT --only-site -s http://ftp.osuosl.org/pub/osgeo/download/osgeo4w -l c:\temp\osgeo4w -P $env:OSGEO4W_DEPS
     displayName: 'Installing OSGeo4W'
 
+  - script: c:\cygwin\bin\unzip -o c:\ninja.zip -d %OSGEO4W_ROOT%\bin
+    displayName: 'Extracting Ninja'
+
   - script: |
       rmdir /s /q c:\temp\cygwin
       rmdir /s /q c:\temp\osgeo4w
     displayName: 'Clear package caches'
 
-  - script: c:\cygwin\bin\unzip -o c:\ninja.zip -d %OSGEO4W_ROOT%\bin
-    displayName: 'Extracting Ninja'
-
-#  - script: c:\cygwin\bin\unzip -o c:\depends.zip -d %OSGEO4W_ROOT%\bin
-#    displayName: 'Extracting Dependency Walker'
-
-  - script: |
-      PATH %OSGEO4W_ROOT%\bin;%ProgramFiles%\CMake\bin;%PATH%
-      cmake --version
-      ctest --version
-      ninja --version
-    displayName: 'Display tool versions'
-
-# Too large…
-#  - task: Cache@2
-#    inputs:
-#      key: 'clcache | $(OSGEO4W_ARCH) | $(OSGEO4W_PKG) | $(Date:yyyyMMdd) | $(Hours)'
-#      path: '$(CLCACHE_DIR)'
-#      restoreKeys: |
-#        clcache | $(OSGEO4W_ARCH) | $(OSGEO4W_PKG) | $(Date:yyyyMMdd) | $(Hours)
-#        clcache | $(OSGEO4W_ARCH) | $(OSGEO4W_PKG) | $(Date:yyyyMMdd)
-#        clcache | $(OSGEO4W_ARCH) | $(OSGEO4W_PKG) | $(Date:yyyyMM)
-#        clcache | $(OSGEO4W_ARCH) | $(OSGEO4W_PKG) | $(Date:yyyy)
-#        clcache | $(OSGEO4W_ARCH) | $(OSGEO4W_PKG)
-#    displayName: Cache clcache
+  - bash: |
+      echo "##[section]$BUILDNAME results available at $DASHURL"
+    displayName: 'Link to test results'
 
   - script: |
       echo on
       PATH c:\cygwin\bin;%OSGEO4W_ROOT%\bin;%PATH%
       cd ms-windows\osgeo4w
-      touch skippackage
       set OSGEO4W_CXXFLAGS=/MD /MP /Od /D NDEBUG
-      @echo ##[section]%OSGEO4W_ARCH% results available at %DASHURL%
       package-nightly.cmd %VERSION% %BINARY% %OSGEO4W_PKG% %OSGEO4W_ARCH% %BUILD_SOURCEVERSION:~0,10% azure-pipelines
+      if not errorlevel 1 if exist testfailure exit /b 1
     displayName: 'Building QGIS'
 
-#  - script: |
-#      echo on
-#      PATH %OSGEO4W_ROOT%\bin;%PATH%
-#      cd ms-windows\osgeo4w\build-%OSGEO4W_PKG%-%OSGEO4W_ARCH%
-#      set /P tag=<Testing\TAG
-#      dir /s /b Testing\*.log
-#      echo ##vso[task.uploadfile]%CD%\Testing\Temporary\LastBuild_%tag%.log
-#      call %OSGEO4W_ROOT%\bin\o4w_env.bat
-#      call %OSGEO4W_ROOT%\bin\qt5_env.bat
-#      call %OSGEO4W_ROOT%\bin\py3_env.bat
-#      depends /c /f:1 /ot:c:\crssync.log output\bin\crssync.exe
-#      echo ##vso[task.uploadfile]c:\crssync.log
-#    displayName: 'Upload build log'
+  - task: PublishPipelineArtifact@1
+    inputs:
+      targetPath: ms-windows/osgeo4w/$(OSGEO4W_ARCH)
+      artifactName: OSGeo4W package $(OSGEO4W_ARCH)
+    displayName: Upload osgeo4w package
 
   - task: PublishTestResults@2
     inputs:
@@ -208,14 +153,4 @@ jobs:
       failTaskOnFailedTests: true
     displayName: 'Publishing tests'
 
-#  - script: |
-#      PATH c:\cygwin\bin;%PATH%
-#      pwd
-#      du -sc c:\cygwin
-#      du -sc %OSGEO4W_ROOT%
-#      du -sc %CLCACHE_DIR%
-#      du -sc .
-#      rmdir /s /q ms-windows\osgeo4w\build-$(OSGEO4W_PKG)-$(OSGEO4W_ARCH)
-#    displayName: Clean before creating the caches
-
 # vim: set nowrap :
diff --git a/.ci/windows-blacklist.txt b/.ci/windows-blacklist.txt
@@ -0,0 +1,143 @@
+ProcessingGdalAlgorithmsRasterTest
+ProcessingGdalAlgorithmsVectorTest
+ProcessingGrass7AlgorithmsImageryTest
+ProcessingGrass7AlgorithmsRasterTest
+ProcessingGrass7AlgorithmsVectorTest
+ProcessingGuiTest
+ProcessingOtbAlgorithmsTest
+ProcessingQgisAlgorithmsTestPt1
+ProcessingQgisAlgorithmsTestPt2
+ProcessingQgisAlgorithmsTestPt3
+ProcessingQgisAlgorithmsTestPt4
+ProcessingScriptUtilsTest
+PyQgsAnnotation
+PyQgsAppStartup
+PyQgsAuthManagerOAuth2OWSTest
+PyQgsAuthManagerPKIOWSTest
+PyQgsAuthManagerPasswordOWSTest
+PyQgsAuthManagerProxy
+PyQgsAuthSettingsWidget
+PyQgsAuxiliaryStorage
+PyQgsBlockingNetworkRequest
+PyQgsCoordinateOperationWidget
+PyQgsDatumTransform
+PyQgsExifTools
+PyQgsExpressionBuilderWidget
+PyQgsFileDownloader
+PyQgsFileUtils
+PyQgsGeometryTest
+PyQgsImageCache
+PyQgsImportIntoPostGIS
+PyQgsLayoutAtlas
+PyQgsLayoutLegend
+PyQgsLayoutMap
+PyQgsLayoutMapGrid
+PyQgsMapLayer
+PyQgsOGRProvider
+PyQgsOGRProviderGpkg
+PyQgsOGRProviderSqlite
+PyQgsOfflineEditingWFS
+PyQgsPalLabelingCanvas
+PyQgsPalLabelingLayout
+PyQgsPalLabelingPlacement
+PyQgsPointDisplacementRenderer
+PyQgsProject
+PyQgsProviderConnectionGpkg
+PyQgsProviderConnectionPostgres
+PyQgsProviderConnectionSpatialite
+PyQgsPythonProvider
+PyQgsRasterFileWriter
+PyQgsRasterLayer
+PyQgsSelectiveMasking
+PyQgsServerAccessControlWMSGetlegendgraphic
+PyQgsServerApi
+PyQgsServerCacheManager
+PyQgsServerLocaleOverride
+PyQgsServerSecurity
+PyQgsServerSettings
+PyQgsServerWMS
+PyQgsServerWMSDimension
+PyQgsServerWMSGetFeatureInfo
+PyQgsServerWMSGetLegendGraphic
+PyQgsServerWMSGetMap
+PyQgsServerWMSGetPrint
+PyQgsServerWMTS
+PyQgsSettings
+PyQgsShapefileProvider
+PyQgsSpatialiteProvider
+PyQgsSvgCache
+PyQgsSymbolLayer
+PyQgsTaskManager
+PyQgsTextRenderer
+PyQgsVectorFileWriter
+PyQgsVectorLayer
+PyQgsVectorLayerUtils
+PyQgsVirtualLayerDefinition
+PyQgsVirtualLayerProvider
+PyQgsWFSProviderGUI
+PyQgsZipUtils
+qgis_3drenderingtest
+qgis_alignrastertest
+qgis_arcgisrestutilstest
+qgis_banned_keywords
+qgis_browsermodeltest
+qgis_callouttest
+qgis_compositionconvertertest
+qgis_coordinatereferencesystemtest
+qgis_datadefinedsizelegendtest
+qgis_datumtransformdialog
+qgis_defwindowtitle
+qgis_diagramtest
+qgis_doxygen_order
+qgis_dxfexporttest
+qgis_expressiontest
+qgis_filedownloader
+qgis_geometrycheckstest
+qgis_geometrytest
+qgis_geonodeconnectiontest
+qgis_grassprovidertest7
+qgis_imagecachetest
+qgis_invertedpolygonrenderertest
+qgis_labelingenginetest
+qgis_layerdefinitiontest
+qgis_layout3dmaptest
+qgis_layouthtmltest
+qgis_layoutlabeltest
+qgis_layoutmanualtabletest
+qgis_layoutmapgridtest
+qgis_layoutmaptest
+qgis_layoutpicturetest
+qgis_layoutscalebartest
+qgis_layouttabletest
+qgis_legendrenderertest
+qgis_licenses
+qgis_maprendererjobtest
+qgis_maprotationtest
+qgis_mapsettingsutilstest
+qgis_maptooladdfeatureline
+qgis_maptoolsplitpartstest
+qgis_meshcalculator
+qgis_mimedatautilstest
+qgis_networkaccessmanagertest
+qgis_openclutilstest
+qgis_painteffecttest
+qgis_pallabelingtest
+qgis_pointlocatortest
+qgis_processingtest
+qgis_projecttest
+qgis_qgisappclipboard
+qgis_rasterlayersaveasdialog
+qgis_shellcheck
+qgis_sip_include
+qgis_sip_uptodate
+qgis_sipify
+qgis_spelling
+qgis_styletest
+qgis_svgcachetest
+qgis_taskmanagertest
+qgis_transformdialog
+qgis_vectorfilewritertest
+qgis_vectortilelayertest
+qgis_vertextool
+qgis_wcsprovidertest
+qgis_ziplayertest