Merge pull request #79 from CDAT/build_tool_update

Build tool update

README.md

@@ -1,6 +1,82 @@
 # conda-recipes
 
-# Building metapackages
+This repository contains tools for:
+- CDAT developers to build their conda packages, create a test environment, and upload the built package to conda channel. The tools are under *build_tools*, the main script is *build_tools/conda_build.py*. *conda_build.py* can be called from project's *Makefile* which can then be called from project's *.circleci/config.yml*.
+
+# Notes for CDAT developers
+
+In order to use *build_tools/conda_build.py*, create a conda environment and activate the *base* environment. 
+
+## Clone conda-recipes repository.
+
+Clone conda-recipes repository to a work directory (referred as $WORKDIR in this documentation).
+
+```
+export WORKDIR=<some_work_directory>
+git clone https://github.com/CDAT/conda-recipes $WORKDIR/conda-recipes
+export BUILD_SCRIPT=$WORKDIR/conda-recipes/build_tools/conda_build.py
+```
+Run *build_tools/conda_build.py --help* to get information about this script.
+
+Clone the CDAT project you want to build. For example, to clone *cdms* project:
+
+```
+git clone https://github.com/CDAT/cdms
+cd cdms
+```
+
+Set the following environment variables.
+```bash  
+  export PKG_NAME=<package_name>
+  export REPO_NAME=<repo_name>
+  export LAST_STABLE=<last_stable_version>
+  export BRANCH=<project_branch>
+  export CONDA_ACTIVATE=<conda_path>/bin/activate
+  export CONDA_ENV=<test_environment_name>
+```
+
+For example:
+```
+export PKG_NAME=cdms2
+export REPO_NAME=cdms
+export LAST_STABLE=3.1.4
+export BRANCH=fix_flake8
+export CONDA_ACTIVATE=/home/username/miniconda3/bin/activate
+export PYTHON_VERSION=3.7
+export CONDA_ENV=test_cdms
+```
+
+## Rerender
+
+
+First step in building a conda package is to do a rerendering which will pick up latest conda-forge update so that we get latest pinned dependencies. You will need *recipe/meta.yaml.in* in the project repo
+
+```bash
+$ python $BUILD_SCRIPT --workdir $WORKDIR --last_stable $LAST_STABLE \
+		       --build 0 --package_name $PKG_NAME --repo_name $REPO_NAME \
+		       --branch $BRANCH --do_rerender 
+```
+
+## Build
+
+```bash
+$ export EXTRA_CHANNELS=cdat/label/nightly
+$ python $BUILD_SCRIPT --workdir $WORKDIR --package_name $PKG_NAME \
+  	 --repo_name $REPO_NAME --build_version $PYTHON_VERSION \
+	 --extra_channels $EXTRA_CHANNELS \
+	 --conda_activate $CONDA_ACTIVATE --do_build
+```
+
+## Setup an environment with the built package
+```bash
+conda create -y -n $CONDA_ENV --use-local -c $EXTRA_CHANNEL $PKG_NAME 
+```
+
+For example, to create a test environment for running cdms test cases:
+```bash
+conda create -y -n $CONDA_ENV --use-local -c $EXTRA_CHANNELS $PKG_NAME testsrunner pytest
+```
+
+
 
-Make sure you have a version of `conda-build` that is at least `3.0.28`. Please run `conda update conda-build` before.
 

build_tools/Utils.py

@@ -7,7 +7,7 @@
 SUCCESS = 0
 FAILURE = 1
 
-def run_command(cmd, join_stderr=True, shell_cmd=False, verbose=True, cwd=None):
+def run_command(cmd, join_stderr=True, shell_cmd=False, verbose=True, cwd=None, env=None):
     print("CMD: {c}".format(c=cmd))
     if isinstance(cmd, str):
         cmd = shlex.split(cmd)
@@ -22,8 +22,14 @@ def run_command(cmd, join_stderr=True, shell_cmd=False, verbose=True, cwd=None):
     else:
         current_wd = cwd
 
+    new_env = os.environ.copy()
+
+    if env is not None:
+        new_env.update(env)
+
     P = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr_setting,
-        bufsize=0, cwd=current_wd, shell=shell_cmd)
+        bufsize=0, cwd=current_wd, shell=shell_cmd, env=new_env)
+
     out = []
     while P.poll() is None:
         read = P.stdout.readline().rstrip()
@@ -34,21 +40,21 @@ def run_command(cmd, join_stderr=True, shell_cmd=False, verbose=True, cwd=None):
 
     ret_code = P.returncode
     return(ret_code, out)
-
-def run_cmd(cmd, join_stderr=True, shell_cmd=False, verbose=True, cwd=None):
 
-    ret_code, output = run_command(cmd, join_stderr, shell_cmd, verbose, cwd)
+def run_cmd(cmd, join_stderr=True, shell_cmd=False, verbose=True, cwd=None, env=None):
+
+    ret_code, output = run_command(cmd, join_stderr, shell_cmd, verbose, cwd, env)
     return(ret_code)
 
-def run_cmds(cmds, join_stderr=True, shell_cmd=False, verbose=True, cwd=None):
+def run_cmds(cmds, join_stderr=True, shell_cmd=False, verbose=True, cwd=None, env=None):
     for cmd in cmds:
-        ret_code, output = run_command(cmd, join_stderr, shell_cmd, verbose, cwd)
+        ret_code, output = run_command(cmd, join_stderr, shell_cmd, verbose, cwd, env)
         if ret_code != SUCCESS:
             return ret_code
     return ret_code
 
-def run_cmd_capture_output(cmd, join_stderr=True, shell_cmd=False, verbose=True, cwd=None):
+def run_cmd_capture_output(cmd, join_stderr=True, shell_cmd=False, verbose=True, cwd=None, env=None):
 
-    ret_code, output = run_command(cmd, join_stderr, shell_cmd, verbose, cwd)
+    ret_code, output = run_command(cmd, join_stderr, shell_cmd, verbose, cwd, env)
     return(ret_code, output)
 

build_tools/conda_build.py

@@ -1,19 +1,14 @@
-import glob
 import argparse
 import os
 import sys
-import subprocess
-import shlex
-import shutil
-import requests
 import time
-import re
 
 from Utils import run_cmd, run_cmds, run_cmd_capture_output
 from Utils import SUCCESS, FAILURE
+from release_tools import find_conda_activate, create_fake_feedstock
 from release_tools import prep_conda_env, check_if_conda_forge_pkg, clone_feedstock
 from release_tools import clone_repo, prepare_recipe_in_local_feedstock_repo
-from release_tools import copy_file_from_repo_recipe
+from release_tools import copy_files_from_repo
 from release_tools import prepare_recipe_in_local_repo, rerender, do_build
 from release_tools import rerender_in_local_feedstock, build_in_local_feedstock
 from release_tools import rerender_in_local_repo, build_in_local_repo, get_git_rev
@@ -48,13 +43,15 @@
 # this script in CircleCI.
 #
 
+conda_rc = os.path.join(os.getcwd(), "condarc")
+
 parser = argparse.ArgumentParser(
     description='conda build upload',
     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
 parser.add_argument("-p", "--package_name",
                     help="Package name to build")
-parser.add_argument("-o", "--github_organization_name",
+parser.add_argument("-o", "--organization",
                     help="github organization name", default="CDAT")
 parser.add_argument("-r", "--repo_name",
                     help="repo name to build")
@@ -69,22 +66,30 @@
 parser.add_argument("--do_rerender", action='store_true', help="do 'conda smithy rerender'")
 parser.add_argument("--do_build", action='store_true', help="do 'conda build -m <variant file> ...'")
 parser.add_argument("--build_version", default="3.7", help="specify python version to build 2.7, 3.7, 3.8")
+parser.add_argument("--conda_env", default="base", help="Conda environment to use, will be created if it doesn't exist")
+parser.add_argument("--extra_channels", nargs="+", type=str, default=[])
+parser.add_argument("--ignore_conda_missmatch", action="store_true", help="Will skip checking if packages are uptodate when rerendering recipe.")
+parser.add_argument("--conda_rc", default=conda_rc, help="File to use for condarc")
+parser.add_argument("--conda_activate", help="Path to conda activate script.")
+parser.add_argument("--copy_conda_package", help="Copies output conda package to directory")
+parser.add_argument("--local_repo", help="Path to local project repository")
 
 args = parser.parse_args(sys.argv[1:])
 
+print(args)
+
 pkg_name = args.package_name
 branch = args.branch
 workdir = args.workdir
 build = args.build
 do_conda_clean = args.conda_clean
+local_repo = args.local_repo
 
-if args.repo_name:
-    repo_name = args.repo_name
-else:
-    repo_name = pkg_name
+if local_repo is not None and not os.path.exists(local_repo):
+    print("Local repository {} does not exist".format(local_repo))
+    sys.exit(FAILURE)
 
-# github organization of projects
-organization = args.github_organization_name
+status = FAILURE
 
 # for calling run_cmds
 join_stderr = True
@@ -108,63 +113,86 @@ def construct_pkg_ver(repo_dir, arg_version, arg_last_stable):
 # main
 #
 
+kwargs = vars(args)
+kwargs["conda_activate"] = args.conda_activate or find_conda_activate()
+if kwargs["repo_name"] is None:
+    kwargs["repo_name"] = pkg_name
+
+repo_name = kwargs["repo_name"]
+
+if kwargs["conda_activate"] is None or not os.path.exists(kwargs["conda_activate"]):
+    print("Could not find conda activate script, try passing with --conda_activate argument and check file exists")
+    sys.exit(FAILURE)
+
 is_conda_forge_pkg = check_if_conda_forge_pkg(pkg_name)
 
+status = prep_conda_env(**kwargs)
+if status != SUCCESS:
+    sys.exit(status)
+
 if args.do_rerender:
-    status = prep_conda_env()
-    if status != SUCCESS:
-        sys.exit(status)
-
-    ret, repo_dir = clone_repo(organization, repo_name, branch, workdir)
-    if ret != SUCCESS:
-        sys.exit(ret)
-    version = construct_pkg_ver(repo_dir, args.version, args.last_stable)
+    if local_repo is None:
+        ret, repo_dir = clone_repo(**kwargs)
+        if ret != SUCCESS:
+            sys.exit(ret)
+    else:
+        repo_dir = local_repo
+
+    kwargs["version"] = version = construct_pkg_ver(repo_dir, args.version, args.last_stable)
 else:
-    repo_dir = "{w}/{p}".format(w=workdir, p=repo_name)
+    if local_repo is None:
+        repo_dir = os.path.join(workdir, repo_name)
+    else:
+        repo_dir = local_repo
 
+print("repo_dir: {d}".format(d=repo_dir))
+files = ["recipe/conda_build_config.yaml",
+         "recipe/build.sh",
+         ".ci_support/migrations/python38.yaml"]
 
 if is_conda_forge_pkg:
     if args.do_rerender:
-        status = clone_feedstock(pkg_name, workdir)
+        status = clone_feedstock(**kwargs)
         if status != SUCCESS:
             sys.exit(status)
 
-        status = prepare_recipe_in_local_feedstock_repo(pkg_name, organization, repo_name, branch, version, build, repo_dir, workdir)
+        status = prepare_recipe_in_local_feedstock_repo(pkg_version=version, repo_dir=repo_dir, **kwargs)
         if status != SUCCESS:
             sys.exit(status)
 
-        status = copy_file_from_repo_recipe(pkg_name, repo_dir, workdir,
-                                            "conda_build_config.yaml")
+        status = copy_files_from_repo(repo_dir=repo_dir, filenames=files, **kwargs)
         if status != SUCCESS:
             sys.exit(status)
 
-        status = copy_file_from_repo_recipe(pkg_name, repo_dir, workdir,
-                                            "build.sh")
-        if status != SUCCESS:
-            sys.exit(status)
-
-        status = rerender_in_local_feedstock(pkg_name, workdir)
+        status = rerender_in_local_feedstock(**kwargs)
 
     if args.do_build:
-        status = build_in_local_feedstock(pkg_name, workdir, args.build_version)
+        status = build_in_local_feedstock(**kwargs)
 
 else:
-    # non conda-forge package (does not have feedstock)
-    repo_dir = os.path.join(workdir, repo_name)
     print("Building non conda-forge package")
     print("...branch: {b}".format(b=branch))
     print("...build: {b}".format(b=build))
     print("...repo_dir: {d}".format(d=repo_dir))
 
     if args.do_rerender:
-        status = prepare_recipe_in_local_repo(branch, build, version, repo_dir)
+        status = prepare_recipe_in_local_repo(repo_dir=repo_dir, **kwargs)
         if status != SUCCESS:
             sys.exit(status)
 
-        status = rerender_in_local_repo(repo_dir)
+        # Create a fake feedstock in the workdir to run conda smithy in
+        feedstock_dir = create_fake_feedstock(repo_dir=repo_dir, **kwargs)
+
+        status = copy_files_from_repo(repo_dir=repo_dir, filenames=files, **kwargs)
+        if status != SUCCESS:
+            sys.exit(status)
+
+        status = rerender_in_local_repo(repo_dir=feedstock_dir, **kwargs)
+    else:
+        feedstock_dir = os.path.join(workdir, "{}-feedstock".format(pkg_name))
 
     if args.do_build:
-        status = build_in_local_repo(repo_dir, args.build_version) 
+        status = build_in_local_repo(repo_dir=feedstock_dir, **kwargs)
 
 sys.exit(status)