Avoid failing or printing warning when working without Gitlab

.gitlab-ci.yml

@@ -69,6 +69,9 @@ deploy:production:
 
 deploy:staging:
   <<: *deploy
+  variables:
+    DOCKER_HOST: qa
+    CI_ENVIRONMENT_SLUG: ""
   environment:
     name: staging
     url: http://qa:9000

backend/README.md

@@ -33,10 +33,10 @@ Consult also:
 
 Flask helps us create an HTTP server. It exposes API endpoints defined in the [api/](api/) folder.
 - `api.py`: read/list data about projects/commits/outputs
-- `webhooks.py`: listens for (i) push notification from gitlab (ii) new results sent by `qatools`.
+- `webhooks.py`: listens for (i) push notification from gitlab (ii) new results sent by `qa`.
 - `tuning.py`: ask for new tuning runs, 
 
-`database.py` manages how we access our database, and connect to the git repository via `gitpython`.
+`database.py` manages how we access our database, and connects to the git repository via `gitpython`.
 
 
 ## Changing the database schemas

backend/backend/__init__.py

@@ -16,12 +16,6 @@
 repos = Repos(git_server, qaboard_data_dir / 'git')
 
 
-# We could fetch the latest commits at startup
-# TODO: find which projects to pull using the latest commits in the database
-# from .git_utils import git_pull
-# default_repo = repos['dvs/psp_swip']
-# git_pull(default_repo)
-
 # Some magic to use sqlalchemy safely with Flask
 # http://flask.pocoo.org/docs/0.12/patterns/sqlalchemy/
 from backend.database import db_session, engine, Base

backend/backend/api/integrations.py

@@ -70,6 +70,9 @@ def gitlab_job():
   """
   Get information about a GitlabCI manual job.
   """
+  if "GITLAB_ACCESS_TOKEN" not in os.environ:
+    return jsonify({"error": f'Error: Missing GITLAB_ACCESS_TOKEN in environment variables'}), 500
+
   data = request.get_json()
   gitlab_api = f"{data['gitlab_host']}/api/v4"
   gitlab_headers = {
@@ -115,6 +118,8 @@ def gitlab_play_manual_job():
   """
   Trigger a GitlabCI manual job.
   """
+  if "GITLAB_ACCESS_TOKEN" not in os.environ:
+    return jsonify({"error": f'Error: Missing GITLAB_ACCESS_TOKEN in environment variables'}), 500
   data = request.get_json()
 
   gitlab_api = f"{data['gitlab_host']}/api/v4"

backend/backend/api/tuning.py

@@ -74,7 +74,7 @@ def get_commit_batches_paths(project, commit_id):
     ci_commit = CiCommit.query.filter(
         CiCommit.project_id == project.id, CiCommit.hexsha.startswith(commit_id)
     ).one()
-    commit_config_inputs = ci_commit.data['qatools_config'].get('inputs', {})
+    commit_config_inputs = ci_commit.data.get('qatools_config', {}).get('inputs', {})
     commit_group_files = commit_config_inputs.get('batches', commit_config_inputs.get('groups', []))
     print(commit_group_files, file=sys.stderr)
     if not (isinstance(commit_group_files, list) or isinstance(commit_group_files, tuple)):
@@ -112,9 +112,9 @@ def get_group():
           ).one()
       except NoResultFound:
           return jsonify("Sorry, the commit id was not found"), 404
-      qatools_config = ci_commit.data["qatools_config"]
+      qatools_config = ci_commit.data.get("qatools_config", {})
     else:
-      qatools_config = project.data["qatools_config"]
+      qatools_config = project.data.get("qatools_config", {})
 
 
     has_custom_iter_inputs = False

backend/backend/api/webhooks.py

@@ -45,10 +45,10 @@ def update_batch():
   # And each batch can have changes vs its commit's config and metrics.
   # The use case is usually working locally with `qa --share` and
   # seeing updated visualizations and metrics.
-  if "config" in data and data["config"] != ci_commit["qatools_config"]:
-    batch.data["config"] = data["config"]
-  if "metrics" in data and data["metrics"] != ci_commit["qatools_metrics"]:
-    batch.data["config"] = data["config"]
+  if "qaboard_config" in data and data["qaboard_config"] != ci_commit.data["qatools_config"]:
+    batch.data["config"] = data["qaboard_config"]
+  if "qaboard_metrics" in data and data["qaboard_metrics"] != ci_commit.data["qatools_metrics"]:
+    batch.data["qatools_config"] = data["qaboard_metrics"]
   batch.data = {**batch.data, **batch_data}
 
   # Save info on each "qa batch" command in the batch, mainly to list them in logs
@@ -227,7 +227,7 @@ def new_output_webhook():
 
 @app.route('/webhook/gitlab', methods=['GET', 'POST'])
 def gitlab_webhook():
-  """Gitlab calls this endpoint every push, it garantees we stay synced."""
+  """If Gitlab calls this endpoint every push, we get avatars and update our local copy of the repo."""
   # https://docs.gitlab.com/ce/user/project/integrations/webhooks.html
   data = json.loads(request.data)
   print(data, file=sys.stderr)

backend/backend/config.py

@@ -2,11 +2,12 @@
 from pathlib import Path
 
 # we clone our repositories locally here to access commit metadata
-git_server = os.getenv('QABOARD_GIT_SERVER', 'gitlab.com')
+git_server = os.getenv('GITLAB_HOST', 'https://gitlab.com')
 
 # Where we save "non-metadata" qaboard data
 qaboard_data_dir = Path(os.getenv('QABOARD_DATA_DIR', '/var/qaboard')).resolve()
 # Where we save custom per-project groups
 shared_data_directory = qaboard_data_dir / 'app_data'
-# Default location for project data
-default_ci_directory = qaboard_data_dir / 'data'
+
+# Default location for project outputs and artifacts
+default_ci_directory = '/mnt/qaboard'

backend/backend/git_utils.py

@@ -1,3 +1,5 @@
+import os
+
 from git import Repo
 from git import RemoteProgress
 from git.exc import NoSuchPathError
@@ -9,25 +11,35 @@ class Repos():
   def __init__(self, git_server, clone_directory):
     self._repos = {}
     self.git_server = git_server
+    if not self.git_server.endswith('/'):
+        self.git_server = self.git_server + '/'
     self.clone_directory = clone_directory
 
   def __getitem__(self, project_path):
     """
     Return a git-python Repo object representing a clone
     of $QABOARD_GIT_SERVER/project_path at $QABOARD_DATA
 
-    project_path: the full git repository namespace, eg dvs/psp_swip
+    project_path: the full git repository namespace, eg group/repo
     """
+    if "GITLAB_ACCESS_TOKEN" not in os.environ:
+      raise ValueError(f'[ERROR] Please provide $GITLAB_ACCESS_TOKEN as environment variable')
+    if "GITLAB_HOST" not in os.environ:
+      raise ValueError(f'[ERROR] Please provide $GITLAB_HOST as environment variable')
+
     clone_location = str(self.clone_directory / project_path)
     try:
       repo = Repo(clone_location)
     except NoSuchPathError:
       try:
+        # TODO: use access token :)
+        # git clone http://oauth2:xxxxxxxxxxxxxxxxx@gitlab-srv/cde/cde-python
+        gitlab_uri = self.git_server.replace('://', f"://oauth2:{os.environ['GITLAB_ACCESS_TOKEN']}@")
         print(f'Cloning <{project_path}> to {self.clone_directory}')
         repo = Repo.clone_from(
           # for now we expect everything to be on gitlab-srv via http
-          f'git@{self.git_server}:{project_path}',
-          str(clone_location)
+          f"{gitlab_uri}{project_path}",
+          str(clone_location),
         )
       except Exception as e:
         print(f'[ERROR] Could not clone: {e}. Please set $QABOARD_DATA to a writable location and verify your network settings')

backend/backend/models/CiCommit.py

@@ -9,7 +9,7 @@
 
 from requests.utils import quote
 from sqlalchemy import Column, Boolean, Integer, String, DateTime, JSON, ForeignKey
-from sqlalchemy import or_, UniqueConstraint
+from sqlalchemy import or_, UniqueConstraint, orm
 from sqlalchemy.orm import relationship, reconstructor, joinedload
 from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
 from sqlalchemy.orm.attributes import flag_modified
@@ -64,6 +64,10 @@ class CiCommit(Base):
   deleted = Column(Boolean(), default=False)
 
 
+  @orm.reconstructor
+  def init_on_load(self):
+    if not self.data:
+      self.data = {}
 
   def get_or_create_batch(self, label):
     matching_batches = [b for b in self.batches if b.label == label]
@@ -133,6 +137,7 @@ def __init__(self, hexsha, *, project, branch=None, message=None, parents=None,
     self.hexsha = hexsha
     self.project = project
     self.branch = branch
+    self.message = message
     self.parents = parents
     self.authored_datetime = authored_datetime
     self.committer_name = committer_name
@@ -192,22 +197,22 @@ def get_or_create(session, hexsha, project_id, data=None):
       try:
         from backend.models import Project
         project = Project.get_or_create(session=session, id=project_id)
-        if data.get("config"):
-          is_initialization = 'qatools_config' not in project.data
-          reference_branch = data["config"]['project'].get('reference_branch', 'master')
+        if data and data.get('qaboard_config'):
+          is_initialization = not project.data or 'qatools_config' not in data 
+          reference_branch = data["qaboard_config"]['project'].get('reference_branch', 'master')
           is_reference = data.get("commit_branch") == reference_branch
           if is_initialization or is_reference:
             # FIXME: We put in Project.data.git the content of
             #       https://docs.gitlab.com/ee/user/project/integrations/webhooks.html#push-events
             # FIXME: We should really have Project.data.gitlab/github/...
             if "git" not in project.data:
               project.data["git"] = {}
-            if "path_with_namespace" not in project.data["git"] and "name" in data["config"].get("project", {}): # FIXME: it really should be Project.root
+            if "path_with_namespace" not in project.data["git"] and "name" in data["qaboard_config"].get("project", {}): # FIXME: it really should be Project.root
               # FIXME: Doesn't support updates for now... again should have .id: int, name: str, root: str...
-              project.data["git"]["path_with_namespace"] = data["config"]["project"]["name"]
-            project.data.update({'qatools_config': data['config']})
-            if "metrics" in data:
-              project.data.update({'qatools_metrics': data["metrics"]})
+              project.data["git"]["path_with_namespace"] = data["qaboard_config"]["project"]["name"]
+            project.data.update({'qatools_config': data['qaboard_config']})
+            if "qaboard_metrics" in data:
+              project.data.update({'qatools_metrics': data["qaboard_metrics"]})
             flag_modified(project, "data")
         else:
           # For backward-compatibility we fallback to reading the data from the commit itself
@@ -219,17 +224,16 @@ def get_or_create(session, hexsha, project_id, data=None):
             print(error)
             raise ValueError(error)
 
-
         ci_commit = CiCommit(
           hexsha,
           project=project,
           commit_type='git', # we don't use anything else
-          parents=data["commit_parents"] if "commit_parents" in data else [c.hexsha for c in git_commit.parents],
-          message=data["commit_message"] if "commit_message" in data else git_commit.message,
-          committer_name=data["commit_committer_name"] if "commit_committer_name" in data else git_commit.committer_name,
-          authored_datetime=data["commit_authored_datetime"] if "commit_authored_datetime" in data else git_commit.authored_datetime,
+          parents=data["commit_parents"] if (data and "commit_parents" in data) else [c.hexsha for c in git_commit.parents],
+          message=data["commit_message"] if (data and "commit_message" in data) else git_commit.message,
+          committer_name=data["commit_committer_name"] if (data and "commit_committer_name" in data) else git_commit.committer.name,
+          authored_datetime=data["commit_authored_datetime"] if (data and "commit_authored_datetime" in data) else git_commit.authored_datetime,
           # commits belong to many branches, so this is a guess
-          branch=data["commit_branch"] if "commit_branch" in data else find_branch(hexsha, project.repo),
+          branch=data["commit_branch"] if (data and "commit_branch" in data) else find_branch(hexsha, project.repo),
         )
         session.add(ci_commit)
         session.commit()

backend/backend/models/Project.py

@@ -131,8 +131,12 @@ def update_project(data, db_session):
   root_project = Project.get_or_create(session=db_session, id=root_project_id)
   update_project_data(root_project, data, db_session)
 
-  repo = repos[root_project_id]
-  git_pull(repo)
+  try:
+    repo = repos[root_project_id]
+    git_pull(repo)
+  except:
+    print(f"Could not fetch the git info for {root_project_id}")
+    return
 
   @lru_cache(maxsize=128)
   def parsed_content(commit_id, path):

backend/backend/utils.py

@@ -33,6 +33,9 @@ def func_wrapper(*args, **kwargs):
 @cache(minutes=60)
 def get_users_per_name(search_filter):
   """Retrievies users from Gitlab"""
+  if 'GITLAB_ACCESS_TOKEN' not in os.environ:
+    return {}
+
   headers = {'Private-Token': os.environ['GITLAB_ACCESS_TOKEN']}
   gitlab_api = "http://gitlab-srv/api/v4"
   users_db = {} # tries to matche a name/fullname/firstname/id to a gitlab user

qaboard/api.py

@@ -131,8 +131,8 @@ def notify_qa_database(object_type='output', **kwargs):
   'commit_authored_datetime': commit_authored_datetime,
   'commit_parents': commit_parents,
   'commit_message': commit_message,
-  "config": serialize_paths(deepcopy(config)),
-  "metrics": _metrics,
+  "qaboard_config": serialize_paths(deepcopy(config)),
+  "qaboard_metrics": _metrics,
   })
   if 'QA_VERBOSE' in os.environ:
     click.secho(url, fg='cyan', err=True)

qaboard/gitlab.py

@@ -13,24 +13,28 @@
 # TODO: don't put credentials here...
 gitlab_host = os.getenv('GITLAB_HOST', secrets.get('GITLAB_HOST', 'https://gitlab.com'))
 gitlab_token = os.environ.get('GITLAB_ACCESS_TOKEN', secrets.get('GITLAB_ACCESS_TOKEN'))
-if not gitlab_token:
-  click.secho("WARNING: GITLAB_ACCESS_TOKEN is not defined.", fg='yellow', bold=True, err=True)
-  click.secho("         Please provide it as an environment variable: https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html", fg='yellow', err=True)
 gitlab_headers = {
   'Private-Token': gitlab_token,
 }
 gitlab_api = f"{gitlab_host}/api/v4"
 gitlab_project_id = quote(root_qatools_config['project']['name'], safe='')
 
 
+def check_gitlab_token():
+  if not gitlab_token:
+    click.secho("WARNING: GITLAB_ACCESS_TOKEN is not defined.", fg='yellow', bold=True, err=True)
+    click.secho("         Please provide it as an environment variable: https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html", fg='yellow', err=True)
+
 
 def ci_commit_data(commit_id):
+  check_gitlab_token()
   url = f"{gitlab_api}/projects/{gitlab_project_id}/repository/commits/{commit_id}"
   r = requests.get(url, headers=gitlab_headers)
   r.raise_for_status()
   return r.json()
 
 def ci_commit_statuses(commit_id, **kwargs):
+  check_gitlab_token()
   url = f"{gitlab_api}/projects/{gitlab_project_id}/repository/commits/{commit_id}/statuses"
   r = requests.get(url, headers=gitlab_headers, params=kwargs)
   r.raise_for_status()
@@ -39,6 +43,7 @@ def ci_commit_statuses(commit_id, **kwargs):
 
 
 def update_gitlab_status(commit_id, state='success'):
+  check_gitlab_token()
   url = f"{gitlab_api}/projects/{gitlab_project_id}/statuses/{commit_id}"
   name = f"QA {subproject.name}" if subproject else 'QA'
   params = {
@@ -63,8 +68,10 @@ def update_gitlab_status(commit_id, state='success'):
 
 
 def lastest_successful_ci_commit(commit_id: str, max_parents_depth=config.get('bit_accuracy', {}).get('max_parents_depth', 5)):
-  from .git import git_parents
+  if not gitlab_token:
+    return commit_id
 
+  from .git import git_parents
   if max_parents_depth < 0:
     click.secho(f'Could not find a commit that passed CI', fg='red', bold=True, err=True)
     exit(1)

qaboard/qa.py

@@ -190,21 +190,13 @@ def run(ctx, input_path, output_path, keep_previous, no_postprocess, forwarded_a
 
       start = time.time()
       cwd = os.getcwd() 
-      try:
-        # TODO: remove, it's only there for backward compatibility with HW_ALG tuning 
-        if 'ENV' in ctx.obj['extra_parameters']:
-          ctx.obj['ENV'] = ctx.obj['extra_parameters']
-          del ctx.obj['extra_parameters']
+      # TODO: remove, it's only there for backward compatibility with HW_ALG tuning 
+      if 'ENV' in ctx.obj['extra_parameters']:
+        ctx.obj['ENV'] = ctx.obj['extra_parameters']
+        del ctx.obj['extra_parameters']
 
+      try:
         runtime_metrics = entrypoint_module(config).run(ctx)
-        if not isinstance(runtime_metrics, dict):
-          click.secho(f'[ERROR] Your `run` function did not return a dict, but {runtime_metrics}', fg='red', bold=True)
-          runtime_metrics = {'is_failed': True}
-
-        if not runtime_metrics:
-          runtime_metrics = {}
-        runtime_metrics['compute_time'] = time.time() - start
-
       except Exception as e:
         exc_type, exc_value, exc_traceback = sys.exc_info()
         click.secho(f'[ERROR] Your `run` function raised an exception: {e}', fg='red', bold=True)
@@ -215,6 +207,14 @@ def run(ctx, input_path, output_path, keep_previous, no_postprocess, forwarded_a
           print(f"ERROR: {e}")
         runtime_metrics = {'is_failed': True}
 
+      if not isinstance(runtime_metrics, dict):
+        click.secho(f'[ERROR] Your `run` function did not return a dict, but {runtime_metrics}', fg='red', bold=True)
+        runtime_metrics = {'is_failed': True}
+
+      if not runtime_metrics:
+        runtime_metrics = {}
+      runtime_metrics['compute_time'] = time.time() - start
+
       # TODO: remove, it's only there for backward compatibility with HW_ALG tuning 
       if 'ENV' in ctx.obj:
         ctx.obj['extra_parameters'].update(ctx.obj['ENV'])
@@ -223,7 +223,7 @@ def run(ctx, input_path, output_path, keep_previous, no_postprocess, forwarded_a
       # avoid issues if code in run() changes cwd
       if os.getcwd() != cwd:
         os.chdir(cwd)
-      metrics = postprocess_(runtime_metrics, ctx, skip=no_postprocess, save_manifests_in_database=save_manifests_in_database)
+      metrics = postprocess_(runtime_metrics, ctx, skip=no_postprocess or runtime_metrics['is_failed'], save_manifests_in_database=save_manifests_in_database)
       if not metrics:
         metrics = runtime_metrics
 
@@ -549,9 +549,9 @@ def batch(ctx, batches, batches_files, tuning_search_dict, tuning_search_file, n
       qa_context=ctx.obj,
     )
 
-    from .gitlab import update_gitlab_status
+    from .gitlab import gitlab_token, update_gitlab_status
     always_update = getenvs(('QATOOLS_ALWAYS_UPDATE_GITLAB', 'QA_ALWAYS_UPDATE_GITLAB'))
-    if jobs and is_ci and (ctx.obj['batch_label']=='default' or always_update):
+    if gitlab_token and jobs and is_ci and (ctx.obj['batch_label']=='default' or always_update):
       update_gitlab_status(commit_id, 'failed' if is_failed else 'success')
 
     if is_failed and not no_wait: