Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Release process #75

Merged
merged 9 commits into from
Feb 4, 2020
Merged

Release process #75

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
e57317d
adding release_tools.py
LinaMuryanto Jan 30, 2020
6521911
adding build_tools/get_release_info.py and release_info/release_info.…
LinaMuryanto Jan 30, 2020
285dfdd
added organization argument to prepare_recipe_in_local_feedstock_repo()
LinaMuryanto Jan 30, 2020
dce29f3
added build argument to prepare_recipe_in_local_feedstock_repo()
LinaMuryanto Jan 30, 2020
4be16e7
adding vcsaddons to release_info/release_info.json
LinaMuryanto Jan 30, 2020
ad650e7
adding some debug
LinaMuryanto Jan 31, 2020
a51d388
adding conda -y ...
LinaMuryanto Jan 31, 2020
78d01b8
conda config --set always_yes yes
LinaMuryanto Jan 31, 2020
bb7ada3
removed debug messages
LinaMuryanto Jan 31, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
249 changes: 8 additions & 241 deletions build_tools/conda_build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,16 @@

from Utils import run_cmd, run_cmds, run_cmd_capture_output
from Utils import SUCCESS, FAILURE
from release_tools import prep_conda_env, check_if_conda_forge_pkg, clone_feedstock
from release_tools import clone_repo, prepare_recipe_in_local_feedstock_repo
from release_tools import prepare_recipe_in_local_repo, rerender, do_build
from release_tools import rerender_in_local_feedstock, build_in_local_feedstock
from release_tools import rerender_in_local_repo, build_in_local_repo

p = subprocess.Popen(["git", "rev-parse", "--short", "HEAD"], stdout=subprocess.PIPE)
git_rev_parse = p.stdout.read().decode('utf-8')
git_rev = "g{0}".format(git_rev_parse).strip()
print("XXX XXX XXX git_rev: {g}".format(g=git_rev))
print("git_rev: {g}".format(g=git_rev))
if "VERSION" in os.environ.keys():
last_stable=os.environ['VERSION']
else:
Expand Down Expand Up @@ -106,244 +111,6 @@
shell_cmd = False
verbose = True

def prep_conda_env():
if do_conda_clean:
cmd = "conda clean --all"
ret = run_cmd(cmd, join_stderr, shell_cmd, verbose, workdir)

pkgs = "conda-build anaconda-client conda-smithy conda-verify conda-forge-pinning conda-forge-build-setup conda-forge-ci-setup"
cmds = [
#"conda update -y -q conda",
"conda config --add channels conda-forge --force",
"conda config --set channel_priority strict",
"conda install -n base -c conda-forge {p}".format(p=pkgs),
"conda config --set anaconda_upload no"
]
ret = run_cmds(cmds)
return ret

def check_if_conda_forge_pkg(pkg_name):
url = "https://www.github.com/conda-forge/{pkg}-feedstock".format(pkg=pkg_name)
try:
request = requests.get(url)
if request.status_code == 200:
print("{p} is a conda-forge package".format(p=pkg_name))
return True
else:
print("{p} is not a conda-forge package".format(p=pkg_name))
return False
except requests.ConnectionError:
print("Web site does not exist")
print("{p} is not a conda-forge package".format(p=pkg_name))
return False

def clone_feedstock(pkg_name, workdir):
pkg_feedstock = "{p}-feedstock".format(p=pkg_name)
conda_forge_pkg_feedstock = "conda-forge/{p}".format(p=pkg_feedstock)

feedstock_repo_dir = os.path.join(workdir, pkg_feedstock)
if os.path.exists(feedstock_repo_dir):
print("REMOVING existing {d}".format(d=feedstock_repo_dir))
shutil.rmtree(feedstock_repo_dir)

cmd = "git clone [email protected]:{c}.git".format(c=conda_forge_pkg_feedstock)
ret = run_cmd(cmd, join_stderr, shell_cmd, verbose, workdir)

return ret

def clone_repo(repo_name, branch, workdir):
repo_dir = os.path.join(workdir, repo_name)
if os.path.exists(repo_dir):
shutil.rmtree(repo_dir)

repo_url = "https://github.com/{o}/{r}.git\n\n".format(o=organization,
r=repo_name)
if branch == "master":
cmd = "git clone {u}".format(u=repo_url)
else:
cmd = "git clone -b {b} {u}".format(b=branch, u=repo_url)
ret = run_cmd(cmd, join_stderr, shell_cmd, verbose, workdir)

return ret, repo_dir

def prepare_recipe_in_local_feedstock_repo(pkg_name, repo_name, branch, pkg_version, repo_dir, workdir):
repo_url = "https://github.com/{o}/{r}.git\n\n".format(o=organization,r=repo_name)

pkg_feedstock = "{p}-feedstock".format(p=pkg_name)
feedstock_dir = os.path.join(workdir, pkg_feedstock)
recipe_file = os.path.join(feedstock_dir, 'recipe', 'meta.yaml')

#
# if repo has a recipe/meta.yaml.in, this means the branch is updating
# the recipe, use this recipe to build.
# NOTE: when we build the package for conda-forge, we will need to
# merge this recipe to feedstock and delete the recipe from the repo.
#
repo_recipe = os.path.join(repo_dir, "recipe", "meta.yaml.in")
if os.path.isfile(repo_recipe):
print("\nNOTE: {r} exists, we will build using this recipe.\n".format(r=repo_recipe))
recipe_file_source = repo_recipe
else:
print("\nNOTE: building with feedstock recipe with modified package source\n")
recipe_file_source = os.path.join(feedstock_dir, 'recipe', 'meta.yaml.SRC')

cmd = "mv {src} {dest}".format(src=recipe_file, dest=recipe_file_source)
ret = run_cmd(cmd, join_stderr, shell_cmd, verbose)
if ret != SUCCESS:
return ret

orig_fh = open(recipe_file_source, "r")
output_fh = open(recipe_file, "w")

output_fh.write("package:\n")
output_fh.write(" name: {n}\n".format(n=pkg_name))
output_fh.write(" version: {v}\n\n".format(v=pkg_version))

output_fh.write("source:\n")
output_fh.write(" git_rev: {b}\n".format(b=branch))
output_fh.write(" git_url: {r}\n".format(r=repo_url))

start_copy = False
lines = orig_fh.readlines()
for l in lines:
match_obj = re.match("build:", l)
if match_obj:
start_copy = True

match_build_number = re.match("\s+number:", l)
if match_build_number:
output_fh.write(" number: {b}\n".format(b=build))
continue
if start_copy:
output_fh.write(l)
else:
continue
output_fh.close()
orig_fh.close()

cmd = "cat {f}".format(f=recipe_file)
#ret = run_cmd(cmd, join_stderr, shell_cmd, verbose)
print("CMD: {c}".format(c=cmd))
os.system(cmd)

return SUCCESS

def prepare_recipe_in_local_repo(branch, build, version, repo_dir):

recipe_in_file = os.path.join(repo_dir, "recipe", "meta.yaml.in")
recipe_file = os.path.join(repo_dir, "recipe", "meta.yaml")
if not os.path.isfile(recipe_in_file):
print("Cannot find {r} file".format(r=recipe_in_file))
return FAILURE

with open(recipe_in_file, "r") as recipe_in_fh:
s = recipe_in_fh.read()
s = s.replace("@UVCDAT_BRANCH@", branch)
s = s.replace("@BUILD_NUMBER@", build)
s = s.replace("@VERSION@", version)

# write it out to recipe/meta.yaml file
with open(recipe_file, "w") as f:
f.write(s)

cmd = "cat {f}".format(f=recipe_file)
print("CMD: {c}".format(c=cmd))
os.system(cmd)

return SUCCESS

def rerender(dir):
# pkg_feedstock = "{p}-feedstock".format(p=pkg_name)
# repo_dir = "{w}/{p}".format(w=workdir, p=pkg_feedstock)

cmd = "conda smithy rerender"
ret = run_cmd(cmd, join_stderr, shell_cmd, verbose, dir)
if ret != SUCCESS:
return ret
return ret

def do_build(dir, py_version):
ret = SUCCESS
variant_files_dir = os.path.join(dir, ".ci_support")
if py_version == "noarch":
variant_file = os.path.join(variant_files_dir, "linux_.yaml")
cmd = "conda build -m {v} recipe/".format(v=variant_file)
ret = run_cmd(cmd, join_stderr, shell_cmd, verbose, dir)
else:
if sys.platform == 'darwin':
variant_files = glob.glob("{d}/.ci_support/osx*{v}*.yaml".format(d=dir, v=py_version))
else:
variant_files = glob.glob("{d}/.ci_support/linux*{v}*.yaml".format(d=dir, v=py_version))

for variant_file in variant_files:
cmd = "conda build -m {v} recipe/".format(v=variant_file)
ret = run_cmd(cmd, join_stderr, shell_cmd, verbose, dir)
if ret != SUCCESS:
print("FAIL: {c}".format(c=cmd))
break

return ret

def rerender_in_local_feedstock(pkg_name, workdir):
pkg_feedstock = "{p}-feedstock".format(p=pkg_name)
repo_dir = os.path.join(workdir, pkg_feedstock)

ret = rerender(repo_dir)
if ret != SUCCESS:
print("FAIL...rerender in {d}".format(d=repo_dir))
return ret

def build_in_local_feedstock(pkg_name, workdir, py_version):
pkg_feedstock = "{p}-feedstock".format(p=pkg_name)
repo_dir = os.path.join(workdir, pkg_feedstock)

ret = do_build(repo_dir, py_version)
return ret

def update_variant_files(dir):
if sys.platform == 'darwin':
variant_config_files = glob.glob("{d}/.ci_support/osx*.yaml".format(d=dir))
else:
variant_config_files = glob.glob("{d}/.ci_support/linux*.yaml".format(d=dir))
for f in variant_config_files:
tmp_f = "{fname}.tmp".format(fname=f)
tmp_fh = open(tmp_f, "w")
orig_fh = open(f, "r")
channel_source = False
for l in orig_fh:
match_obj = re.match("channel_sources:", l)
if match_obj:
channel_source = True
tmp_fh.write(l)
continue
if channel_source:
tmp_fh.write("- cdat/label/nightly,conda-forge,defaults\n")
channel_source = False
continue
tmp_fh.write(l)
orig_fh.close()
tmp_fh.close()
shutil.move(tmp_f, f)
return SUCCESS

def rerender_in_local_repo(repo_dir):

conda_forge_yml = os.path.join(repo_dir, "conda-forge.yml")
fh = open(conda_forge_yml, "w")
fh.write("recipe_dir: recipe\n")
fh.close()

ret = rerender(repo_dir)
return ret

ret = update_variant_files(repo_dir)
return ret

def build_in_local_repo(repo_dir, py_version):

ret = do_build(repo_dir, py_version)
return ret

#
# main
#
Expand All @@ -355,7 +122,7 @@ def build_in_local_repo(repo_dir, py_version):
if status != SUCCESS:
sys.exit(status)

ret, repo_dir = clone_repo(repo_name, branch, workdir)
ret, repo_dir = clone_repo(organization, repo_name, branch, workdir)
if ret != SUCCESS:
sys.exit(ret)

Expand All @@ -365,7 +132,7 @@ def build_in_local_repo(repo_dir, py_version):
if status != SUCCESS:
sys.exit(status)

status = prepare_recipe_in_local_feedstock_repo(pkg_name, repo_name, branch, version, repo_dir, workdir)
status = prepare_recipe_in_local_feedstock_repo(pkg_name, organization, repo_name, branch, version, build, repo_dir, workdir)
if status != SUCCESS:
sys.exit(status)

Expand Down
34 changes: 34 additions & 0 deletions build_tools/get_release_info.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
import sys
import json
import argparse

parser = argparse.ArgumentParser(
description='conda build upload',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)

parser.add_argument("-r", "--cdat_release",
help="cdat_release_version, ex: 8.2.1")
parser.add_argument("-t", "--package_type",
help="'cdat' or 'conda-forge' package")
parser.add_argument("-p", "--package_name",
help="Package name to build")
parser.add_argument("-f", "--release_info_json_file_name",
help="release_info.json full path name")

args = parser.parse_args(sys.argv[1:])

cdat_release = args.cdat_release
package_type = args.package_type
package_name = args.package_name
release_info_file = args.release_info_json_file_name


with open(release_info_file) as json_file:
release_info = json.load(json_file)
pkg_info = release_info[cdat_release][package_type][package_name]
version = pkg_info['version']
build = pkg_info['build']
type = pkg_info['type']
ret_info = "{v}:{b}:{t}".format(v=version, b=build, t=type)
print(ret_info)

Loading