Try custom backend

.github/workflows/build.yml

@@ -30,6 +30,7 @@ jobs:
         python3-numpy
         python3-pil
         python3-pytest
+        python3-pip
 
     - name: Install collada2gltf
       run: |
@@ -43,13 +44,13 @@ jobs:
 
     - name: Build
       run: |
-        python setup.py --testing install --prefix=install-prefix
+        pip install -v --config-settings testing=True .
       env:
         DEBUG: 1
 
     - name: Test
       run: |
-        ./install-prefix/bin/pymol -ckqy testing/testing.py --run all
+        pymol -ckqy testing/testing.py --run all
 
   build-Windows:
 
@@ -72,7 +73,7 @@ jobs:
       shell: cmd
       run: |-
         CALL %CONDA_ROOT%\\Scripts\\activate.bat
-        conda install -y -c conda-forge -c schrodinger python cmake libpng freetype pyqt glew libxml2 numpy catch2=2.13.3 glm libnetcdf collada2gltf biopython pillow msgpack-python pytest
+        conda install -y -c conda-forge -c schrodinger python cmake libpng freetype pyqt glew libxml2 numpy catch2=2.13.3 glm libnetcdf collada2gltf biopython pillow msgpack-python pytest pip python-build
 
     - name: Conda info
       shell: cmd
@@ -92,13 +93,13 @@ jobs:
       shell: cmd
       run: |
         CALL %CONDA_ROOT%\\Scripts\\activate.bat
-        python setup.py --testing install --prefix=%GITHUB_WORKSPACE%\\install-prefix
+        pip install -v --config-settings testing=True .
 
     - name: Test
       shell: cmd
       run: |
         CALL %CONDA_ROOT%\\Scripts\\activate.bat
-        %GITHUB_WORKSPACE%\\install-prefix\\Scripts\\pymol.bat -ckqy testing\\testing.py --run all
+        pymol.bat -ckqy testing\\testing.py --run all
 
   build-MacOS:
 
@@ -115,7 +116,7 @@ jobs:
         bash $CONDA_ROOT.sh -b -p $CONDA_ROOT
         export PATH="$CONDA_ROOT/bin:$PATH"
         conda config --set quiet yes
-        conda install -y -c conda-forge -c schrodinger python cmake libpng freetype pyqt glew libxml2 numpy catch2=2.13.3 glm libnetcdf collada2gltf biopython pillow msgpack-python pytest
+        conda install -y -c conda-forge -c schrodinger python cmake libpng freetype pyqt glew libxml2 numpy catch2=2.13.3 glm libnetcdf collada2gltf biopython pillow msgpack-python pytest pip python-build
         conda info
 
     - name: Get additional sources
@@ -129,9 +130,9 @@ jobs:
       run: |-
         export MACOSX_DEPLOYMENT_TARGET=12.0
         export PATH="$CONDA_ROOT/bin:$PATH"
-        python setup.py install --prefix=${GITHUB_WORKSPACE}/install-prefix
+        pip install -v --config-settings testing=True .
 
     - name: Test
       run: |-
         export PATH="$CONDA_ROOT/bin:$PATH"
-        ${GITHUB_WORKSPACE}/install-prefix/bin/pymol -ckqy testing/testing.py --run all
+        pymol -ckqy testing/testing.py --run all

_custom_build/backend.py

@@ -0,0 +1,38 @@
+# Adopted from pillow and pycapnp
+
+import sys
+
+from setuptools.build_meta import *  # noqa: F401, F403
+from setuptools.build_meta import build_wheel
+
+backend_class = build_wheel.__self__.__class__
+
+
+class _CustomBuildMetaBackend(backend_class):
+    def run_setup(self, setup_script="setup.py"):
+        if self.config_settings:
+            flags = []
+            for key, values in self.config_settings.items():
+                if values == "True":
+                    flags.append(f"--{key}")
+            if flags:
+              sys.argv = sys.argv[:1] + ["build_ext"] + flags + sys.argv[1:]
+        return super().run_setup(setup_script)
+
+    def build_wheel(
+        self, wheel_directory, config_settings=None, metadata_directory=None
+    ):
+        self.config_settings = config_settings
+        return super().build_wheel(wheel_directory, config_settings, metadata_directory)
+
+    def build_editable(
+        self, wheel_directory, config_settings=None, metadata_directory=None
+    ):
+        self.config_settings = config_settings
+        return super().build_editable(
+            wheel_directory, config_settings, metadata_directory
+        )
+
+_backend = _CustomBuildMetaBackend()
+build_wheel = _backend.build_wheel
+build_editable = _backend.build_editable

pyproject.toml

@@ -18,9 +18,9 @@ dependencies = [
 ]
 
 [build-system]
-build-backend = "setuptools.build_meta"
+build-backend = "backend"
+backend-path = ["_custom_build"]
 requires = [
-  "meson-python>=0.15.0",
   "numpy>=1.26.4",
   "setuptools>=69.2.0",
 ]

setup.py

@@ -229,8 +229,8 @@ class options:
     action="store_false",
     help="Disable VMD molfile plugins (libnetcdf dependency)",
 )
-options, sys.argv[1:] = parser.parse_known_args(namespace=options)
 
+options, sys.argv[1:] = parser.parse_known_args(namespace=options)
 
 def get_prefix_path() -> list[str]:
     """
@@ -240,27 +240,30 @@ def get_prefix_path() -> list[str]:
     paths = []
 
     if (prefix_path := os.environ.get("PREFIX_PATH")) is not None:
-        return prefix_path.split(os.pathsep)
+        paths += prefix_path.split(os.pathsep)
 
     if sys.platform.startswith("freebsd"):
-        return ["/usr/local"]
+        paths += ["/usr/local"]
 
     if not options.osx_frameworks:
         paths += ["/usr/X11"]
 
     if sys.platform == "darwin":
         for prefix in ["/sw", "/opt/local", "/usr/local"]:
             if sys.base_prefix.startswith(prefix):
-                return [prefix] + paths
+                paths += [prefix]
 
     if is_conda_env():
         if sys.platform.startswith("win"):
-            return [os.path.join(sys.prefix, "Library")]
+            if "CONDA_PREFIX" in os.environ:
+                paths += [os.path.join(os.environ["CONDA_PREFIX"], "Library")]
+            paths += [os.path.join(sys.prefix, "Library")]
 
-        return [sys.prefix] + paths
+        paths += [sys.prefix] + paths
 
-    return ["/usr"] + paths
+    paths += ["/usr"]
 
+    return paths
 
 def is_conda_env():
     return (
@@ -470,12 +473,15 @@ def make_launch_script(self):
         else:
             launch_script = "pymol"
 
-        self.mkpath(self.install_scripts)
-        launch_script = os.path.join(self.install_scripts, launch_script)
+        install_scripts = sysconfig.get_path("scripts")
+        install_lib = sysconfig.get_path("purelib")
+
+        self.mkpath(install_scripts)
+        launch_script = os.path.join(install_scripts, launch_script)
 
         python_exe = os.path.abspath(sys.executable)
         pymol_file = self.unchroot(
-            os.path.join(self.install_libbase, "pymol", "__init__.py")
+            os.path.join(install_lib, "pymol", "__init__.py")
         )
         pymol_path = self.unchroot(self.pymol_path)
 
@@ -484,13 +490,13 @@ def make_launch_script(self):
                 # paths relative to launcher, if possible
                 try:
                     python_exe = "%~dp0\\" + os.path.relpath(
-                        python_exe, self.install_scripts
+                        python_exe, install_scripts
                     )
                 except ValueError:
                     pass
                 try:
                     pymol_file = "%~dp0\\" + os.path.relpath(
-                        pymol_file, self.install_scripts
+                        pymol_file, install_scripts
                     )
                 except ValueError:
                     pymol_file = os.path.abspath(pymol_file)