Merge branch 'master' into add-gcc-test

.appveyor.yml

@@ -1,16 +1,14 @@
-version: 4.3.0.99.{build}
+version: 4.4.0.99.{build}
 
 image: Visual Studio 2015
 platform: x64
-configuration:  # a trick to construct a build matrix with multiple Python versions
+configuration:
   - '3.8'
 
-# only build pull requests and
-# commits to 'master' or any branch starting with 'release'
+# only build on 'master' and pull requests targeting it
 branches:
   only:
     - master
-    - /^release/
 
 environment:
   matrix:
@@ -25,12 +23,13 @@ install:
   - git submodule update --init --recursive  # get `external_libs` folder
   - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%
   - set PYTHON_VERSION=%CONFIGURATION%
-  - set CONDA_ENV="test-env"
   - ps: |
+      $env:ALLOW_SKIP_ARROW_TESTS = "1"
+      $env:APPVEYOR = "true"
+      $env:CMAKE_BUILD_PARALLEL_LEVEL = 4
       $env:MINICONDA = "C:\Miniconda3-x64"
       $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH"
       $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER"
-      $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim()
 
 build: false
 

.ci/check_python_dists.sh

@@ -26,11 +26,12 @@ fi
 PY_MINOR_VER=$(python -c "import sys; print(sys.version_info.minor)")
 if [ $PY_MINOR_VER -gt 7 ]; then
     echo "pydistcheck..."
-    pip install pydistcheck
+    pip install 'pydistcheck>=0.7.0'
     if { test "${TASK}" = "cuda" || test "${METHOD}" = "wheel"; }; then
         pydistcheck \
             --inspect \
-            --ignore 'compiled-objects-have-debug-symbols,distro-too-large-compressed' \
+            --ignore 'compiled-objects-have-debug-symbols'\
+            --ignore 'distro-too-large-compressed' \
             --max-allowed-size-uncompressed '100M' \
             --max-allowed-files 800 \
             ${DIST_DIR}/* || exit 1

.ci/conda-envs/ci-core-py38.txt

@@ -0,0 +1,51 @@
+# [description]
+#
+#   Similar to ci-core.txt, but specific to Python 3.8.
+#
+#   Unlike ci-core.txt, this includes a Python version and uses
+#   `=` and `<=` pins to make solves faster and prevent against
+#   issues like https://github.com/microsoft/LightGBM/pull/6370.
+#
+# [usage]
+#
+#   conda create \
+#     --name test-env \
+#     --file ./.ci/conda-envs/ci-core-py38.txt
+#
+
+# python
+python=3.8.*
+
+# direct imports
+cffi=1.15.*
+dask=2023.5.*
+distributed=2023.5.*
+joblib=1.4.*
+matplotlib-base=3.7.*
+numpy=1.24.*
+pandas=1.5.*
+pyarrow-core=16.1.*
+python-graphviz=0.20.*
+scikit-learn=1.3.*
+scipy=1.10.*
+
+# testing-only dependencies
+cloudpickle=3.0.*
+pluggy=1.5.*
+psutil=5.9.8
+pytest=8.2.*
+
+# other recursive dependencies, just
+# pinned here to help speed up solves
+bokeh=3.1.*
+fsspec=2024.5.*
+msgpack-python=1.0.*
+pluggy=1.5.*
+pytz=2024.1
+setuptools=69.5.*
+snappy=1.2.*
+tomli=2.0.*
+tornado=6.4.*
+wheel=0.43.*
+zict=3.0.*
+zipp=3.17.*

.ci/get_workflow_status.py

@@ -6,15 +6,12 @@
 
 TRIGGER_PHRASE: Code phrase that triggers workflow.
 """
+
 import json
 from os import environ
 from sys import argv, exit
 from time import sleep
-
-try:
-    from urllib import request
-except ImportError:
-    import urllib2 as request
+from urllib import request
 
 
 def get_runs(trigger_phrase):

.ci/lint_r_code.R

@@ -52,6 +52,8 @@ LINTERS_TO_USE <- list(
     , "inner_combine"        = lintr::inner_combine_linter()
     , "is_numeric"           = lintr::is_numeric_linter()
     , "lengths"              = lintr::lengths_linter()
+    , "length_levels"        = lintr::length_levels_linter()
+    , "length_test"          = lintr::length_test_linter()
     , "line_length"          = lintr::line_length_linter(length = 120L)
     , "literal_coercion"     = lintr::literal_coercion_linter()
     , "matrix"               = lintr::matrix_apply_linter()
@@ -66,6 +68,7 @@ LINTERS_TO_USE <- list(
     , "redundant_equals"     = lintr::redundant_equals_linter()
     , "regex_subset"         = lintr::regex_subset_linter()
     , "routine_registration" = lintr::routine_registration_linter()
+    , "scalar_in"            = lintr::scalar_in_linter()
     , "semicolon"            = lintr::semicolon_linter()
     , "seq"                  = lintr::seq_linter()
     , "spaces_inside"        = lintr::spaces_inside_linter()

.ci/setup.sh

@@ -14,26 +14,20 @@ if [[ $OS_NAME == "macos" ]]; then
     if  [[ $COMPILER == "clang" ]]; then
         brew install libomp
         if [[ $AZURE == "true" ]]; then
-            sudo xcode-select -s /Applications/Xcode_11.7.app/Contents/Developer || exit 1
+            sudo xcode-select -s /Applications/Xcode_13.1.0.app/Contents/Developer || exit 1
         fi
     else  # gcc
         # Check https://github.com/actions/runner-images/tree/main/images/macos for available
         # versions of Xcode
         sudo xcode-select -s /Applications/Xcode_14.3.1.app/Contents/Developer || exit 1
-        if [[ $TASK != "mpi" ]]; then
-            brew install gcc
-        fi
+        brew install gcc
     fi
     if [[ $TASK == "mpi" ]]; then
         brew install open-mpi
     fi
     if [[ $TASK == "swig" ]]; then
         brew install swig
     fi
-    curl \
-        -sL \
-        -o miniforge.sh \
-        https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-${ARCH}.sh
 else  # Linux
     if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then
         # fixes error "unable to initialize frontend: Dialog"
@@ -45,35 +39,30 @@ else  # Linux
             software-properties-common
 
         sudo apt-get install --no-install-recommends -y \
-            apt-utils \
             build-essential \
             ca-certificates \
             cmake \
             curl \
             git \
-            iputils-ping \
-            jq \
             libcurl4 \
             libicu-dev \
             libssl-dev \
-            libunwind8 \
             locales \
-            locales-all \
-            netcat \
-            unzip \
-            zip || exit 1
+            locales-all || exit 1
         if [[ $COMPILER == "clang" ]]; then
             sudo apt-get install --no-install-recommends -y \
                 clang \
                 libomp-dev
         elif [[ $COMPILER == "clang-17" ]]; then
-            sudo apt-get install wget
+            sudo apt-get install --no-install-recommends -y \
+                wget
             wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
             sudo apt-add-repository deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main
             sudo apt-add-repository deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main
             sudo apt-get update
-            sudo apt-get install -y clang-17
-            sudo apt-get install --no-install-recommends -y libomp-17-dev
+            sudo apt-get install -y \
+                clang-17 \
+                libomp-17-dev
         fi
 
         export LANG="en_US.UTF-8"
@@ -144,16 +133,14 @@ else  # Linux
         apt-get install --no-install-recommends -y \
             cmake
     fi
-    if [[ $SETUP_CONDA != "false" ]]; then
-        curl \
-            -sL \
-            -o miniforge.sh \
-            https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-${ARCH}.sh
-    fi
 fi
 
 if [[ "${TASK}" != "r-package" ]] && [[ "${TASK}" != "r-rchk" ]]; then
     if [[ $SETUP_CONDA != "false" ]]; then
+        curl \
+            -sL \
+            -o miniforge.sh \
+            https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-${ARCH}.sh
         sh miniforge.sh -b -p $CONDA
     fi
     conda config --set always_yes yes --set changeps1 no

.ci/test-python-latest.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -e -E -u -o pipefail
+
+# latest versions of lightgbm's dependencies,
+# including pre-releases and nightlies
+#
+# ref: https://github.com/pydata/xarray/blob/31111b3afe44fd6f7dac363264e94186cc5168d2/.github/workflows/upstream-dev-ci.yaml
+echo "installing testing dependencies"
+python -m pip install \
+    cloudpickle \
+    psutil \
+    pytest
+echo "done installing testing dependencies"
+
+echo "installing lightgbm's dependencies"
+python -m pip install \
+    --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \
+    --prefer-binary \
+    --pre \
+    --upgrade \
+        'numpy>=2.0.0.dev0' \
+        'matplotlib>=3.10.0.dev0' \
+        'pandas>=3.0.0.dev0' \
+        'scikit-learn>=1.6.dev0' \
+        'scipy>=1.15.0.dev0'
+
+python -m pip install \
+    --extra-index-url https://pypi.fury.io/arrow-nightlies/ \
+    --prefer-binary \
+    --pre \
+    --upgrade \
+        'pyarrow>=17.0.0.dev0'
+
+python -m pip install \
+    'cffi>=1.15.1'
+
+echo "done installing lightgbm's dependencies"
+
+echo "installing lightgbm"
+pip install --no-deps dist/*.whl
+echo "done installing lightgbm"
+
+echo "installed package versions:"
+pip freeze
+
+echo ""
+echo "running tests"
+pytest tests/c_api_test/
+pytest tests/python_package_test/

.ci/test-python-oldest.sh

@@ -3,19 +3,20 @@
 set -e -E -u -o pipefail
 
 # oldest versions of dependencies published after
-# minimum supported Python version's first release
+# minimum supported Python version's first release,
+# for which there are wheels compatible with the
+# python:{version} image
 #
 # see https://devguide.python.org/versions/
 #
 echo "installing lightgbm's dependencies"
 pip install \
   'cffi==1.15.1' \
-  'dataclasses' \
-  'numpy==1.16.6' \
-  'pandas==0.24.0' \
+  'numpy==1.19.0' \
+  'pandas==1.1.3' \
   'pyarrow==6.0.1' \
-  'scikit-learn==0.18.2' \
-  'scipy==0.19.0' \
+  'scikit-learn==0.24.0' \
+  'scipy==1.6.0' \
 || exit 1
 echo "done installing lightgbm's dependencies"