diff --git a/.github/workflows/pypiReleaseUpload.yml b/.github/workflows/pypiReleaseUpload.yml deleted file mode 100644 index 1d3b3e546..000000000 --- a/.github/workflows/pypiReleaseUpload.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Upload to PyPI - -on: - release: - types: [published] - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade build - pip install setuptools wheel twine - - - name: Build package - run: python setup.py sdist bdist_wheel - - - name: Upload to PyPI - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - run: | - python -m twine upload --repository pypi dist/* - - diff --git a/.github/workflows/pypi_build_publish_template.yaml b/.github/workflows/pypi_build_publish_template.yaml new file mode 100644 index 000000000..73de704fc --- /dev/null +++ b/.github/workflows/pypi_build_publish_template.yaml @@ -0,0 +1,41 @@ +# Reusable workflow to setup a specific T0 component for pip + +on: + release: + types: [published] + +jobs: + build_and_publish_from_template: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Setup python 3.8 + uses: actions/setup-python@v3 + with: + python-version: "3.8" + - name: Upgrade pip3 + run: | + python3 -m pip install --upgrade pip + - name: Update the setup script template with package name + run: | + sed "s/PACKAGE_TO_BUILD/${{ inputs.t0_component }}/" setup_template.py > setup.py + - name: Create requirements file + run: | + cp requirements.txt requirements.t0.txt + awk "/(${{ inputs.t0_component }}$)|(${{ inputs.t0_component }},)/ {print \$1}" requirements.t0.txt > requirements.txt + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade build + pip install setuptools wheel twine + + - name: Build sdist + run: python setup.py clean sdist + + - name: Upload to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: | + python -m twine upload --repository pypi dist/* + diff --git a/MANIFEST.in b/MANIFEST.in index 3d96bb2c8..4e61f821b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,7 @@ -graft bin -graft etc +include setup_build.py +include setup_dependencies.py +recursive-include bin * +recursive-include etc * include requirements.txt include LICENSE +include NOTICE diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index f4c5e0bcc..000000000 --- a/pyproject.toml +++ /dev/null @@ -1,45 +0,0 @@ -[build-system] -requires = ["setuptools>=61.0"] -build-backend = "setuptools.build_meta" - -[project] -name = "T0" -version = "0.0.0" -description = "This package contains the code that is involved in the Tier 0 agent when its deployed" -readme = "README.md" -requires-python = ">=3.8" -license = {file = "LICENSE"} -keywords = ["T0"] -authors = [ - {email = "Dirk.Hufnagel@cern.ch"}, - {name = "Dirk Hufnagel"} -] -maintainers = [ - {name = "WMCore", email = "ms.unmerged@cern.ch"} -] - -classifiers = [ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "License :: OSI Approved", - "Operating System :: MacOS :: MacOS X", - "Operating System :: Microsoft :: Windows", - "Operating System :: POSIX", - "Programming Language :: Python", - "Topic :: Database" -] - -[tool.setuptools.dynamic] -dependencies = {file = ["requirements.txt"]} - -[tool.setuptools] -include-package-data = true - -[tool.setuptools.packages.find] -namespaces = true -where = ["src/python"] - -[project.urls] -homepage = "https://github.com/dmwm/T0" - - diff --git a/requirements.txt b/requirements.txt index 3e6e777b6..d8f2ec0bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ -MySQL-python==1.2.4b4 +pyOpenSSL~=18.0.0 +setuptools>=69.5.1 wmagent>=2.3.0.2 diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..3dfcdc812 --- /dev/null +++ b/setup.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +""" +Build, clean and test the t0 package. +""" +from __future__ import print_function + +import imp +import os +import os.path +from distutils.core import Command, setup +from os.path import join as pjoin + +from setup_build import BuildCommand, InstallCommand, get_path_to_t0_root, list_packages, list_static_files + + +class CleanCommand(Command): + description = "Clean up (delete) compiled files" + user_options = [] + + def initialize_options(self): + self.cleanMes = [] + for root, dummyDirs, files in os.walk('.'): + for f in files: + if f.endswith('.pyc'): + self.cleanMes.append(pjoin(root, f)) + + def finalize_options(self): + pass + + def run(self): + for cleanMe in self.cleanMes: + try: + os.unlink(cleanMe) + except Exception: + pass + + +class EnvCommand(Command): + description = "Configure the PYTHONPATH, DATABASE and PATH variables to" + \ + "some sensible defaults, if not already set. Call with -q when eval-ing," + \ + """ e.g.: + eval `python setup.py -q env` + """ + + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + if not os.getenv('COUCHURL', False): + # Use the default localhost URL if none is configured. + print('export COUCHURL=http://localhost:5984') + here = get_path_to_t0_root() + + tests = here + '/test/python' + source = here + '/src/python' + # Stuff we want on the path + exepth = [here + '/etc', + here + '/bin'] + + pypath = os.getenv('PYTHONPATH', '').strip(':').split(':') + + for pth in [tests, source]: + if pth not in pypath: + pypath.append(pth) + + # We might want to add other executables to PATH + expath = os.getenv('PATH', '').split(':') + for pth in exepth: + if pth not in expath: + expath.append(pth) + + print('export PYTHONPATH=%s' % ':'.join(pypath)) + print('export PATH=%s' % ':'.join(expath)) + + # We want the t0 root set, too + print('export T0_ROOT=%s' % get_path_to_t0_root()) + print('export T0BASE=$T0_ROOT') + +# The actual setup command, and the classes associated to the various options + +# Need all the packages we want to build by default, this will be overridden in sub-system builds. +# Since it's a lot of code determine it by magic. +DEFAULT_PACKAGES = list_packages(['src/python/T0', + 'src/python/T0Component' + ]) + +# Divine out the version of t0 from t0.__init__, which is bumped by +# "bin/buildrelease.sh" + +# Obnoxiously, there's a dependency cycle when building packages. We'd like +# to simply get the current t0 version by using +# from t0 import __version__ +# But PYTHONPATH isn't set until after the package is built, so we can't +# depend on the python module resolution behavior to load the version. +# Instead, we use the imp module to load the source file directly by +# filename. +t0_root = get_path_to_t0_root() +t0_package = imp.load_source('temp_module', os.path.join(t0_root, + 'src', + 'python', + 'T0', + '__init__.py')) +t0_version = t0_package.__version__ + +setup(name='T0', + version=t0_version, + maintainer='CMS DMWM Group', + maintainer_email='cms-tier0-operations@cern.ch', + cmdclass={'deep_clean': CleanCommand, + 'coverage': CoverageCommand, + 'test': TestCommand, + 'env': EnvCommand, + 'build_system': BuildCommand, + 'install_system': InstallCommand}, + # base directory for all our packages + package_dir={'': 'src/python/'}, # % get_path_to_t0_root()}, + packages=DEFAULT_PACKAGES, + data_files=list_static_files(), + url="https://github.com/dmwm/T0", + license="Apache License, Version 2.0" + ) diff --git a/setup_build.py b/setup_build.py new file mode 100644 index 000000000..183ad92a0 --- /dev/null +++ b/setup_build.py @@ -0,0 +1,353 @@ +from __future__ import print_function +from distutils.core import Command +from distutils.command.build import build +from distutils.command.install import install +from distutils.spawn import spawn +import re, os, sys, os.path, shutil +from glob import glob +from setup_dependencies import dependencies + +def get_path_to_t0_root(): + """ + Work out the path to the T0 root from where the script is being run. Allows for + calling setup.py env from sub directories and directories outside the T0 tree. + """ + return os.path.dirname(os.path.abspath(os.path.join(os.getcwd(), sys.argv[0]))) + +def walk_dep_tree(system): + """ + Walk the dependancy tree/dictionary for the given system, and return a list of packages and + statics that the system depends on. + + system is a dict containing: + {'bin': ['bin_script_A'], + 'packages': ['package_name_A'], + 'modules': ['module_name_A'], + 'systems':['system_name_A'], + 'statics': ['non_python_statics_A']} + """ + packages = set() + statics = set() + modules = set() + bindir = set() + if 'bin' in system: + bindir = set(system.get('bin', set())) + if 'modules' in system: + modules = set(system.get('modules', set())) + if 'packages' in system: + packages = set(system.get('packages', set())) + if 'statics' in system: + statics = set(system.get('statics', set())) + if 'systems' in system: + for system in system['systems']: + dependants = walk_dep_tree(dependencies[system]) + bindir = bindir | dependants.get('bin', set()) + packages = packages | dependants.get('packages', set()) + statics = statics | dependants.get('statics', set()) + modules = modules | dependants.get('modules', set()) + return {'bin': bindir, 'packages': packages, 'statics': statics, 'modules': modules} + +def list_packages(package_dirs = [], recurse=True): + """ + Take a list of directories and return a list of all packages under those directories, skipping VCS files. + """ + packages = [] + # Skip the following files + ignore_these = set(['CVS', '.svn', 'svn', '.git', 'DefaultConfig.py']) + for a_dir in package_dirs: + if recurse: + # Recurse the sub-directories + for dirpath, dummy_dirnames, dummy_filenames in os.walk('%s' % a_dir, topdown=True): + pathelements = dirpath.split('/') + # If any part of pathelements is in the ignore_these set skip the path + if len(set(pathelements) & ignore_these) == 0: + rel_path = os.path.relpath(dirpath, get_path_to_t0_root()) + rel_path = rel_path.split('/')[2:] + packages.append('.'.join(rel_path)) + else: + print('Ignoring %s' % dirpath) + else: + rel_path = os.path.relpath(a_dir, get_path_to_t0_root()) + rel_path = rel_path.split('/')[2:] + packages.append('.'.join(rel_path)) + return packages + +def data_files_for(dir): + """ + Return list of data files in provided dir, do not walk through + """ + files = [] + add_static = files.append + # Skip the following files + ignore_these = set(['CVS', '.svn', 'svn', '.git', 'DefaultConfig.py']) + if dir.endswith('+'): + dir = dir.rstrip('+') + for dirpath, dummy_dirnames, filenames in os.walk('%s' % dir, topdown=True): + pathelements = dirpath.split('/') + # If any part of pathelements is in the ignore_these set skip the path + if len(set(pathelements) & ignore_these) == 0: + rel_path = os.path.relpath(dirpath, get_path_to_t0_root()) + localfiles = [os.path.join(rel_path, f) for f in filenames] + add_static((rel_path.replace('src/', 'data/'), localfiles)) + else: + print('Ignoring %s' % dirpath) + else: + localfiles = [] + for ifile in os.listdir(dir): + filename = os.path.join(dir, ifile) + if os.path.isfile(filename): + if filename[-1] == '~': + continue + localfiles.append(filename) + rel_path = os.path.relpath(dir, get_path_to_t0_root()) + add_static((rel_path.replace('src/', 'data/'), localfiles)) + + return files + +def list_static_files(system = None): + """ + Get a list of all the files that are classed as static (e.g. javascript, css, templates) + """ + # Skip the following files + ignore_these = set(['CVS', '.svn', 'svn', '.git', 'DefaultConfig.py']) + static_files = [] + if system: + expanded = walk_dep_tree(system) + static_files.append(('bin', sum((glob("bin/%s" % x) for x in expanded['bin']), []))) + for static_dir in expanded['statics']: + static_files.extend(data_files_for(static_dir)) + else: + for language in ['couchapps+', 'css+', 'html+', 'javascript+', 'templates+']: + static_files.extend(data_files_for('%s/src/%s' % (get_path_to_t0_root(), language))) + for toplevel in ['bin+', 'etc+']: + static_files.extend(data_files_for('%s/%s' % (get_path_to_t0_root(), toplevel))) + # The contents of static_files will be copied at install time + return static_files + +def check_system(command): + """ + Check that the system being built is known, print an error message and exit if it's not. + """ + if command.system in dependencies: + return + elif command.system == None: + msg = "System not specified: -s option for %s must be specified and provide one of:\n" % command.get_command_name() + msg += ", ".join(dependencies) + print(msg) + sys.exit(1) + else: + msg = "Specified system [%s] is unknown:" % command.system + msg += " -s option for %s must be specified and provide one of:\n" % command.get_command_name() + msg += ", ".join(dependencies) + print(msg) + sys.exit(1) + +def things_to_build(command, pypi=False): + """ + Take a build/install command and determine all the packages and modules it needs to build/install. Modules are + explicitly listed in the dependancies but packages needs to be generated (to pick up sub directories). + """ + # work out all the dependent packages + if pypi: + dependency_tree = walk_dep_tree(dependencies[command]) + else: + dependency_tree = walk_dep_tree(dependencies[command.system]) + + # and the corresponding source directories and files + package_src_dirs = [] + for package in dependency_tree['packages']: + # Need to recurse packages + recurse = package.endswith('+') + print(package, recurse) + package = package.rstrip('+') + src_path = '%s/src/python/%s' % (get_path_to_t0_root(), package.replace('.', '/')) + package_src_dirs.extend(list_packages([src_path], recurse)) + return package_src_dirs, dependency_tree['modules'] + +def print_build_info(command): + """ + print some helpful information about what needs to be built + """ + if len(command.distribution.packages): + command.announce('Installing %s requires the following packages:' % (command.system), 1) + for package in command.distribution.packages: + if os.path.exists('%s/build/lib/%s' % (get_path_to_t0_root(), package.replace('.','/'))) and not command.force: + command.announce('\t %s \t - already built!' % package, 1) + else: + command.announce('\t %s' % package, 1) + if len(command.distribution.py_modules): + command.announce('Installing %s requires the following modules:' % (command.system), 1) + for modules in command.distribution.py_modules: + if os.path.exists('%s/build/lib/%s' % (get_path_to_t0_root(), modules.replace('.','/'))) and not command.force: + command.announce('\t %s \t - already built!' % modules) + else: + command.announce('\t %s' % modules) + +def force_rebuild(): + """ + When building sub-systems its a good idea to always start from a fresh build area, otherwise + sub-systems can merge up, worst case is doing a sub-system install after a full build/install - + you'll get the contents of all packages. + + This method forcibly removes the build area, so that all sub-system builds/installs start from + a clean sheet. + """ + shutil.rmtree('%s/build' % get_path_to_t0_root(), True) + shutil.rmtree('%s/doc/build' % get_path_to_t0_root(), True) + +class BuildCommand(Command): + """ + Build a specific system, including it's dependencies. Run with --force to trigger a rebuild + """ + description = "Build a specific sub-system, including it's dependencies. Should be used with --force" + description = "to ensure a clean build of only the specified sub-system.\n\n" + description += "\tAvailable sub-systems: \n" + description += "\t[" + description += ", ".join(dependencies) + description += "]\n" + + user_options = build.user_options + user_options.append(('system=', 's', 'build the specified system')) + user_options.append(('skip-docs', None, 'skip documentation')) + user_options.append(('compress', None, 'compress assets')) + + def initialize_options(self): + # and add our additional option + self.system = None + self.skip_docs = False + self.compress = False + + def finalize_options (self): + # Check that the sub-system is valid + check_system(self) + # Set what to build + self.distribution.packages, self.distribution.py_modules = things_to_build(self) + print_build_info(self) + # Always do a rebuild + force_rebuild() + + def generate_docs (self): + if not self.skip_docs: + os.environ["PYTHONPATH"] = "%s/build/lib:%s" % (get_path_to_t0_root(), os.environ.get("PYTHONPATH", '')) + spawn(['make', '-C', 'doc', 'html', 'PROJECT=%s' % self.system.lower()]) + + def compress_assets(self): + if not self.compress: + for dir, files in self.distribution.data_files: + for f in files: + if f.find("-min.") >= 0: + print("removing", f) + os.remove(f) + else: + rxfileref = re.compile(r"(/[-A-Za-z0-9_]+?)(?:-min)?(\.(html|js|css))") + for dir, files in self.distribution.data_files: + files = [f for f in files if f.find("-min.") < 0] + if not files: + continue + elif dir == 'data/javascript': + spawn(['java', '-jar', os.environ["YUICOMPRESSOR"], '--type', 'js', + '-o', (len(files) > 1 and '.js$:-min.js') + or files[0].replace(".js", "-min.js")] + + files) + elif dir == 'data/css': + spawn(['java', '-jar', os.environ["YUICOMPRESSOR"], '--type', 'css', + '-o', (len(files) > 1 and '.css$:-min.css') + or files[0].replace(".css", "-min.css")] + + files) + elif dir == 'data/templates': + for f in files: + if f.endswith(".html"): + print("minifying", f) + minified = open(f).read() + minified = re.sub(re.compile(r"\n\s*([<>])", re.S), r"\1", minified) + minified = re.sub(re.compile(r"\n\s*", re.S), " ", minified) + minified = re.sub(r"", "", minified) + minified = re.sub(rxfileref, r"\1-min\2", minified) + open(f.replace(".html", "-min.html"), "w").write(minified) + + def run (self): + # Have to get the build command here and set force, as the build plugins only refer to the + # build command, not what calls them. The following is taken from the Distribution class, + # with the additional explicit setting of force + command = 'build' + if self.distribution.have_run.get(command): + return + cmd = self.distribution.get_command_obj(command) + # Forcibly set force + cmd.force = self.force + cmd.ensure_finalized() + cmd.run() + self.generate_docs() + self.compress_assets() + self.distribution.have_run[command] = 1 + +class InstallCommand(install): + """ + Install a specific system, including it's dependencies. + """ + description = "Install a specific system, including it's dependencies. Should be used with --force" + description = "to ensure a clean build of only the specified sub-system.\n\n" + description += "\tAvailable sub-systems: \n" + description += "\t[" + description += ", ".join(dependencies) + description += "]\n" + + user_options = install.user_options + user_options.append(('system=', 's', 'install the specified system')) + user_options.append(('patch', None, 'patch an existing installation (default: no patch)')) + user_options.append(('skip-docs', None, 'skip documentation')) + user_options.append(('compress', None, 'compress assets')) + + def initialize_options(self): + # Call the base class + install.initialize_options(self) + # and add our additionl options + self.system = None + self.patch = None + self.skip_docs = False + self.compress = False + + def finalize_options(self): + # Check that the sub-system is valid + check_system(self) + # Check install destination looks valid if patching. + if self.patch and not os.path.isdir("%s/xbin" % self.prefix): + print("Patch destination %s does not look like a valid location." % self.prefix) + sys.exit(1) + # Set what actually gets installed + self.distribution.packages, self.distribution.py_modules = things_to_build(self) + self.distribution.data_files = list_static_files(dependencies[self.system]) + docroot = "%s/doc/build/html" % get_path_to_t0_root() + for dirpath, dirs, files in os.walk(docroot): + self.distribution.data_files.append(("doc%s" % dirpath[len(docroot):], + ["%s/%s" % (dirpath, fname) for fname in files if + fname != '.buildinfo'])) + # Mangle data paths if patching. + if self.patch: + self.distribution.data_files = [('x' + dir, files) for dir, files in self.distribution.data_files] + + print_build_info(self) + + self.distribution.metadata.name = self.system + assert self.distribution.get_name() == self.system + + install.finalize_options(self) + + # Convert to patch install if so requested + if self.patch: + self.install_lib = re.sub(r'(.*)/lib/python(.*)', r'\1/xlib/python\2', self.install_lib) + self.install_scripts = re.sub(r'(.*)/bin$', r'\1/xbin', self.install_scripts) + + def run (self): + # Have to override the distribution to build and install only what we specify. + for cmd_name in self.get_sub_commands(): + cmd = self.distribution.get_command_obj(cmd_name) + cmd.distribution = self.distribution + # We don't want data files mixed in with the python + if cmd_name == 'install_data': + cmd.install_dir = self.prefix + else: + cmd.install_dir = self.install_lib + cmd.ensure_finalized() + self.run_command(cmd_name) + self.distribution.have_run[cmd_name] = 1 diff --git a/setup_dependencies.py b/setup_dependencies.py new file mode 100644 index 000000000..e45122e55 --- /dev/null +++ b/setup_dependencies.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python + +dependencies = { + 't0': { + 'packages': ['T0+', + 'T0Component+'], + 'modules': [], + 'systems': [], + 'statics': ['bin+', + 'etc+'] + } +} diff --git a/setup_t0.py b/setup_t0.py new file mode 100644 index 000000000..473cd9b7f --- /dev/null +++ b/setup_t0.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +# This setup script is used to build the T0 pypi package. +# The version number comes from T0/__init__.py and needs to +# follow PEP 440 conventions + +from __future__ import print_function, division +import os +import sys +import imp +from setuptools import setup +from setup_build import list_packages, list_static_files, get_path_to_t0_root + +# Obnoxiously, there's a dependency cycle when building packages. We'd like +# to simply get the current T0 version by using +# from T0 import __version__ +# But PYTHONPATH isn't set until after the package is built, so we can't +# depend on the python module resolution behavior to load the version. +# Instead, we use the imp module to load the source file directly by +# filename. +t0_root = get_path_to_t0_root() +t0_package = imp.load_source('temp_module', os.path.join(t0_root, + 'src', + 'python', + 'T0', + '__init__.py')) +t0_version = t0_package.__version__ + +# Requirements file for pip dependencies +requirements = "requirements.txt" + + +def parse_requirements(requirements_file): + """ + Create a list for the 'install_requires' component of the setup function + by parsing a requirements file + """ + + if os.path.exists(requirements_file): + # return a list that contains each line of the requirements file + return open(requirements_file, 'r').read().splitlines() + else: + print("ERROR: requirements file " + requirements_file + " not found.") + sys.exit(1) + + +setup(name='t0', + version=t0_version, + maintainer='CMS DMWM Group', + maintainer_email='cms-tier0-operations@cern.ch', + package_dir={'': 'src/python/'}, + packages=list_packages(['src/python/T0', + 'src/python/T0Component' + ]), + data_files=list_static_files(), + install_requires=parse_requirements(requirements), + url="https://github.com/dmwm/T0", + license="Apache License, Version 2.0", + ) diff --git a/setup_template.py b/setup_template.py new file mode 100644 index 000000000..d6bc94abb --- /dev/null +++ b/setup_template.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +# This template for the setup script is used to build several pypi packages +# from the T0 codebase. The variable package_name controls which package +# is built. PACKAGE_TO_BUILD is manipulated via tools/build_pypi_packages.sh +# at build time. +# +# The version number comes from T0/__init__.py and needs to +# follow PEP 440 conventions + +from __future__ import print_function, division +import os +import sys +from setuptools import setup, Command +from setup_build import list_static_files, things_to_build +from setup_dependencies import dependencies + +# get the t0 version (thanks rucio devs) +sys.path.insert(0, os.path.abspath('src/python')) +from T0 import __version__ +t0_version = __version__ + +# the contents of package_name are modified via tools/build_pypi_packages.sh +package_name = "t0" +packages, py_modules = things_to_build(package_name, pypi=True) +data_files = list_static_files(dependencies[package_name]) + +# we need to override 'clean' command to remove specific files +class CleanCommand(Command): + user_options = [] + def initialize_options(self): + pass + def finalize_options(self): + pass + def run(self): + os.system ('rm -rfv ./dist ./src/python/*.egg-info') + +def parse_requirements(requirements_file): + """ + Create a list for the 'install_requires' component of the setup function + by parsing a requirements file + """ + + if os.path.exists(requirements_file): + # return a list that contains each line of the requirements file + return open(requirements_file, 'r').read().splitlines() + else: + print("ERROR: requirements file " + requirements_file + " not found.") + sys.exit(1) + +setup(name=package_name, + version=t0_version, + package_dir={'': 'src/python/'}, + packages=packages, + py_modules=py_modules, + data_files=data_files, + install_requires=parse_requirements("requirements.txt"), + maintainer='CMS DMWM Group', + maintainer_email='hn-cms-wmdevelopment@cern.ch', + cmdclass={ + 'clean': CleanCommand, + }, + url="https://github.com/dmwm/T0", + license="Apache License, Version 2.0", + ) diff --git a/src/python/T0/__init__.py b/src/python/T0/__init__.py index b84c60d1f..dcd443af6 100644 --- a/src/python/T0/__init__.py +++ b/src/python/T0/__init__.py @@ -4,5 +4,5 @@ Core libraries for Workload Management Packages """ - -version = '1.9.18' +__version__ = '4.0.10.pre' +__all__ = [] diff --git a/tools/build_pypi_packages.sh b/tools/build_pypi_packages.sh new file mode 100644 index 000000000..4e6fcb64a --- /dev/null +++ b/tools/build_pypi_packages.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Script used to build each application from the T0 repo and upload to pypi. +# +# Usage +# Build a single package: +# sh tools/build_pypi_packages.sh +# Build all T0 packages: +# sh tools/build_pypi_packages.sh all +# + +set -x + +# package passed as parameter, can be one of PACKAGES or "all" +TOBUILD=$1 +# list of packages that can be built and uploaded to pypi +PACKAGES="T0" + +PACKAGE_REGEX="^($(echo $PACKAGES | sed 's/\ /|/g')|all)$" + +if [[ -z $TOBUILD ]]; then + echo "Usage: sh tools/build_pypi_packages.sh " + echo "Usage: sh tools/build_pypi_packages.sh all" + exit 1 +fi + + +# check to make sure a valid package name was passed +if [[ ! $TOBUILD =~ $PACKAGE_REGEX ]]; then + echo "$TOBUILD is not a valid package name" + echo "Supported packages are $PACKAGES" + exit 1 +fi + +# update package list when building all packages +if [[ $TOBUILD == "all" ]]; then + TOBUILD=$PACKAGES +fi + +# loop through packages to build +for package in $TOBUILD; do + # make a copy of requirements.txt to reference for each build + cp requirements.txt requirements.t0.txt + + # update the setup script template with package name + sed "s/PACKAGE_TO_BUILD/$package/" setup_template.py > setup.py + + # build requirements.txt file + awk "/($package$)|($package,)/ {print \$1}" requirements.t0.txt > requirements.txt + + # build the package + python setup.py clean sdist + if [[ $? -ne 0 ]]; then + echo "Error building package $package" + exit 1 + fi + + # upload the package to pypi + echo "Uploading package $package to PyPI" + twine upload dist/$package-* + + # replace requirements.txt contents + cp requirements.t0.txt requirements.txt +done