Use updated conda recipes (#447)

* clean up Makefile and .circleci/config.yml, use updated conda-recipes which support build for a release
CDAT · Jul 16, 2020 · 431b234 · 431b234
1 parent b0e917b
commit 431b234
Show file tree

Hide file tree

Showing 2 changed files with 51 additions and 62 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,26 +1,5 @@
 version: 2.1
 
-parameters:
-   pkg_name:
-      type: string
-      default: "vcs"
-   repo_name:
-      type: string
-      default: "vcs"
-   last_stable:
-      type: string
-      default: "8.2"
-   user:
-      type: string
-      default: "cdat"
-   label:
-      type: string
-      default: "nightly"
-   env_name:
-      type: string
-      default: "test_vcs"
-
-
 aliases:
   - &setup_env
     name: setup_env
@@ -37,7 +16,6 @@ aliases:
     name: setup_miniconda
     command: |
        source $BASH_ENV
-       mkdir -p $WORKDIR
        git clone https://github.com/CDAT/cdat.git $WORKDIR/cdat
        # install_miniconda.py installs miniconda3 under $WORKDIR/miniconda
        python $WORKDIR/cdat/scripts/install_miniconda.py -w $WORKDIR -p 'py3'
@@ -48,7 +26,7 @@ aliases:
        source $BASH_ENV
        source $WORKDIR/miniconda/etc/profile.d/conda.sh
        conda activate base
-       make conda-rerender conda=$WORKDIR/miniconda/bin/conda workdir=$WORKDIR last_stable=$LAST_STABLE branch=$CIRCLE_BRANCH
+       make conda-rerender workdir=$WORKDIR branch=$CIRCLE_BRANCH
 
   - &conda_build
     name: conda_build
@@ -58,7 +36,7 @@ aliases:
        conda activate base
        os=`uname`
        artifacts_dir="artifacts/artifacts.${os}.noarch"
-       make conda-build conda=$WORKDIR/miniconda/bin/conda workdir=$WORKDIR artifact_dir=$PWD/$artifacts_dir
+       make conda-build workdir=$WORKDIR artifact_dir=$PWD/$artifacts_dir
 
   - &setup_run_tests
     name: setup_run_tests
@@ -68,8 +46,8 @@ aliases:
        conda activate base
        export CONDA_PY_VER="python=$PY_VER"
        export LIBNETCDF_VER="libnetcdf=*=${LIBNETCDF}_*"
-       make setup-tests conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME extra_pkgs="$CONDA_PY_VER $LIBNETCDF_VER $COVERAGE_PKGS"
-       make conda-dump-env conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME artifact_dir=$PWD/spec_artifacts conda_env_filename=$CIRCLE_JOB
+       make setup-tests workdir=$WORKDIR extra_pkgs="$CONDA_PY_VER $LIBNETCDF_VER $COVERAGE_PKGS"
+       make conda-dump-env workdir=$WORKDIR artifact_dir=$PWD/spec_artifacts conda_env_filename=$CIRCLE_JOB
 
   - &get_testdata
     name: get_testdata
@@ -81,34 +59,32 @@ aliases:
     command: |
        source $BASH_ENV
        source $WORKDIR/miniconda/etc/profile.d/conda.sh
-       conda activate $ENV_NAME
-       make run-tests conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME
+       make run-tests workdir=$WORKDIR
 
   - &conda_upload
     name: conda_upload
     command: |
        source $BASH_ENV
        source $WORKDIR/miniconda/etc/profile.d/conda.sh
        conda activate base
-       UPLOAD_OPTIONS="conda_upload_token=$CONDA_UPLOAD_TOKEN user=$USER label=$LABEL"
-       make conda-upload $UPLOAD_OPTIONS conda=$WORKDIR/miniconda/bin/conda artifact_dir="$PWD/artifacts/*Linux*/"
+       UPLOAD_OPTIONS="conda_upload_token=$CONDA_UPLOAD_TOKEN"
+       make conda-upload workdir=$WORKDIR $UPLOAD_OPTIONS artifact_dir="$PWD/artifacts/*Linux*/"
 
   - &run_coveralls
     name: run_coveralls
     command: |
        source $BASH_ENV
        source $WORKDIR/miniconda/etc/profile.d/conda.sh
-       conda activate $ENV_NAME
-       coveralls
-       conda deactivate
+       conda activate base
+       make run-coveralls workdir=$WORKDIR
 
   - &make_doc_test
     name: make_doc_test
     command: |
        source $WORKDIR/miniconda/etc/profile.d/conda.sh
        conda activate base
        if [ `uname` == 'Linux' ] && [ $PY_VER == '3.7' ] && [ $LIBNETCDF == 'nompi' ]; then
-          make run-doc-test conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME
+          make run-doc-test workdir=$WORKDIR
        fi
     no_output_timeout: 20m
 
@@ -126,10 +102,6 @@ jobs:
          os:
             type: executor
       executor: << parameters.os >>
-      environment:
-         PKG_NAME: << pipeline.parameters.pkg_name >>
-         REPO_NAME: << pipeline.parameters.repo_name >>
-         LAST_STABLE: << pipeline.parameters.last_stable >>
       steps:
          - checkout
          - run: *setup_env
@@ -152,9 +124,6 @@ jobs:
             type: string
       executor: << parameters.os >>
       environment:
-         PKG_NAME: << pipeline.parameters.pkg_name >>
-         REPO_NAME: << pipeline.parameters.repo_name >>
-         ENV_NAME: << pipeline.parameters.env_name >>
          PY_VER: << parameters.py_ver >>
          LIBNETCDF: << parameters.libnetcdf >>
       steps:
@@ -180,11 +149,6 @@ jobs:
    upload:
       machine:
          image: circleci/classic:latest
-      environment:
-         PKG_NAME: << pipeline.parameters.pkg_name >>
-         VERSION: << pipeline.parameters.last_stable >>
-         USER: << pipeline.parameters.user >>
-         LABEL: << pipeline.parameters.label >>
       steps:
          - checkout
          - attach_workspace:
@@ -216,3 +180,4 @@ workflows:
               filters:
                  branches:
                     only: master
+
diff --git a/Makefile b/Makefile
@@ -1,11 +1,15 @@
 .PHONY: conda-info conda-list setup-build setup-tests conda-rerender \
-	conda-build conda-upload conda-dump-env get_testdata \
-	run-tests run-doc-tests run-coveralls
+	conda-build conda-upload conda-dump-env get-testdata \
+	run-tests run-doc-test run-coveralls
 
 SHELL = /bin/bash
 
 os = $(shell uname)
 pkg_name = vcs
+
+user ?= cdat
+label ?= nightly
+
 build_script = conda-recipes/build_tools/conda_build.py
 
 #
@@ -22,14 +26,30 @@ endif
 
 last_stable ?= 8.2
 
-conda_env ?= base
-workdir ?= $(PWD)/workspace
+conda_test_env = test-$(pkg_name)
+conda_build_env = build-$(pkg_name)
+
 branch ?= $(shell git rev-parse --abbrev-ref HEAD)
 extra_channels ?= cdat/label/nightly conda-forge
 conda ?= $(or $(CONDA_EXE),$(shell find /opt/*conda*/bin $(HOME)/*conda* -type f -iname conda))
-artifact_dir ?= $(PWD)/artifacts
 conda_env_filename ?= spec-file
 
+# Only populate if workdir is not defined
+ifeq ($(origin workdir),undefined)
+# Create .tempdir if it doesn't exist
+ifeq ($(wildcard $(PWD)/.tempdir),)
+workdir := $(shell mktemp -d -t "build_$(pkg_name).XXXXXXXX")
+$(shell echo $(workdir) > $(PWD)/.tempdir)
+endif
+
+# Read tempdir
+workdir := $(shell cat $(PWD)/.tempdir)
+endif
+
+$(info $(workdir))
+
+artifact_dir ?= $(PWD)/artifacts
+
 conda_recipes_branch ?= master
 
 conda_base = $(patsubst %/bin/conda,%,$(conda))
@@ -41,11 +61,15 @@ ifndef $(local_repo)
 local_repo = $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
 endif
 
+help: ## Prints help
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
+		sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
 conda-info:
-	source $(conda_activate) $(conda_env); conda info
+	source $(conda_activate) $(conda_test_env); conda info
 
 conda-list:
-	source $(conda_activate) $(conda_env); conda list
+	source $(conda_activate) $(conda_test_env); conda list
 
 setup-build:
 ifeq ($(wildcard $(workdir)/conda-recipes),)
@@ -55,30 +79,30 @@ else
 endif
 
 setup-tests:
-	source $(conda_activate) base; conda create -y -n $(conda_env) --use-local \
+	source $(conda_activate) base; conda create -y -n $(conda_test_env) --use-local \
 		$(foreach x,$(extra_channels),-c $(x)) $(pkg_name) $(foreach x,$(test_pkgs),"$(x)") \
 		$(foreach x,$(docs_pkgs),"$(x)") $(foreach x,$(pkgs),"$(x)") $(foreach x,$(extra_pkgs),"$(x)")
 
 conda-rerender: setup-build 
 	python $(workdir)/$(build_script) -w $(workdir) -l $(last_stable) -B 0 -p $(pkg_name) \
-		-b $(branch) --do_rerender --conda_env $(conda_env) --ignore_conda_missmatch \
+		-b $(branch) --do_rerender --conda_env $(conda_build_env) --ignore_conda_missmatch \
 		--conda_activate $(conda_activate)
 
 conda-build:
 	mkdir -p $(artifact_dir)
 
 	python $(workdir)/$(build_script) -w $(workdir) -p $(pkg_name) --build_version noarch \
-		--do_build --conda_env $(conda_env) --extra_channels $(extra_channels) \
-		--conda_activate $(conda_activate) $(conda_build_extra)
+		--do_build --conda_env $(conda_build_env) --extra_channels $(extra_channels) \
+		--conda_activate $(conda_activate) $(conda_build_extra) 
 
 conda-upload:
-	source $(conda_activate) $(conda_env); \
+	source $(conda_activate) $(conda_build_env); \
 		anaconda -t $(conda_upload_token) upload -u $(user) -l $(label) --force $(artifact_dir)/*.tar.bz2
 
 conda-dump-env:
 	mkdir -p $(artifact_dir)
 
-	source $(conda_activate) $(conda_env); conda list --explicit > $(artifact_dir)/$(conda_env_filename).txt
+	source $(conda_activate) $(conda_test_env); conda list --explicit > $(artifact_dir)/$(conda_env_filename).txt
 
 get-testdata:
 ifeq ($(wildcard uvcdat-testdata),)
@@ -88,11 +112,11 @@ else
 endif
 
 run-tests:
-	source $(conda_activate) $(conda_env); python run_tests.py -n 4 -H -v2 --timeout=100000 \
+	source $(conda_activate) $(conda_test_env); python run_tests.py -n 4 -H -v2 --timeout=100000 \
 		--checkout-baseline --no-vtk-ui
 
 run-doc-test:
-	source $(conda_activate) $(conda_env); cd docs; make doctest;
+	source $(conda_activate) $(conda_test_env); cd docs; make doctest;
 
 run-coveralls:
-	source $(conda_activate) $(conda_env); coveralls;
+	source $(conda_activate) $(conda_test_env); coveralls;