diff --git a/git_differ.py b/git_differ.py index b34f0a1..1c89303 100644 --- a/git_differ.py +++ b/git_differ.py @@ -2,11 +2,14 @@ from git import Repo from os.path import isfile import os +from bioconda_utils import recipe -from .utils import get_brg_ci_homedir_path +from utils import get_brg_ci_homedir_path +from packagedb import PackageDBResource CI_HOMEDIR = get_brg_ci_homedir_path() -COMMIT_FILE = CI_HOMEDIR + "latest_commit.data" +COMMIT_FILE = CI_HOMEDIR + "/latest_commit.data" +PACKAGE_CHANGED_DB_PATH = CI_HOMEDIR + "/packages_changed.yaml" def get_latest_commit(): @@ -17,31 +20,42 @@ def get_latest_commit(): return sha +def add_packages_to_packages_changed_DB(packages): + with PackageDBResource(PACKAGE_CHANGED_DB_PATH) as packageDB: + packageDB.add_new_packages(packages) + + +def set_latest_commit_as_HEAD(recipes_path): + with open(COMMIT_FILE, "w") as fp: + fp.write(Repo(recipes_path + "/..").head.commit.hexsha) + + def retrive_packages_changed(recipes_path): """ Traverse bioconda-recipes and extract all packages which has changed. Assumptions: We can safely ignore Bioconductor and R packages """ - print("Fetching packages Name and Source...") - packages = [] + print("Fetching packages changed...") + packages = dict() num_of_packages = 0 num_of_skipped = 0 + num_of_errors = 0 if isfile(COMMIT_FILE): latest_commit = get_latest_commit() - head_commit = git.Repo(recipes_path + "/..").head.commit - diff = hcommit.diff(latest_commit) + head_commit = Repo(recipes_path + "/..").head.commit + diff = head_commit.diff(latest_commit) dirs = [item.a_path.split("/")[1] for item in diff] else: _, dirs, _ = next(os.walk(recipes_path)) + num_of_packages = len(dirs) for dir in dirs: - num_of_packages += 1 # Skip Bioconductor or R packages if "bioconductor" in dir or dir.startswith("r-"): num_of_skipped += 1 continue meta_yaml_path = "%s/%s/meta.yaml" % (recipes_path, dir) try: - if not os.path.isfile(meta_yaml_path): + if not isfile(meta_yaml_path): continue current_recipe = recipe.Recipe.from_file(recipes_path, meta_yaml_path) name = current_recipe.name @@ -51,11 +65,17 @@ def retrive_packages_changed(recipes_path): url = url[0] except: print("%s raised an Error" % name) + num_of_errors += 1 continue - packages.append(Package(name, url)) + packages.update({name: url}) + print(packages) except: print("%s raised an Error" % dir) + num_of_errors += 1 + add_packages_to_packages_changed_DB(packages) + set_latest_commit_as_HEAD(recipes_path) print("%d out of %d packages where skipped" % (num_of_skipped, num_of_packages)) + print("%d errors occured" % num_of_errors) def main(): @@ -65,8 +85,9 @@ def main(): parser.add_argument( "recipes_path", help="The path to the bioconda-recipe/recipes directory" ) - get_all_source_urls(args.recipes_path) + args = parser.parse_args() + retrive_packages_changed(args.recipes_path) -if __name == "__main__": +if __name__ == "__main__": main() diff --git a/packagedb.py b/packagedb.py index ac1b387..e16e9b0 100644 --- a/packagedb.py +++ b/packagedb.py @@ -1,6 +1,7 @@ import fcntl from ruamel_yaml import YAML import hashlib +from os.path import isfile class PackageDBResource: @@ -41,9 +42,15 @@ class PackageDB: """ def __init__(self, DB_path): - self._fp = open(DB_path, "r+") - fcntl.lockf(self._fp, fcntl.LOCK_EX) - self.__load_packages() + if isfile(DB_path): + self._fp = open(DB_path, "r+") + fcntl.lockf(self._fp, fcntl.LOCK_EX) + self.__load_packages() + else: + self._fp = open(DB_path, "w") + fcntl.lockf(self._fp, fcntl.LOCK_EX) + self._sha = "" + self._packages = dict() def __load_packages(self): yaml_dict = YAML().load(self._fp) diff --git a/utils.py b/utils.py index b5ac0d5..94f3874 100644 --- a/utils.py +++ b/utils.py @@ -7,5 +7,5 @@ def get_brg_ci_homedir_path(): if isdir(CI_HOMEDIR): return CI_HOMEDIR else: - os.mkdir(CI_HOMEDIR) + mkdir(CI_HOMEDIR) return CI_HOMEDIR