Merge pull request #63 from opensafely-core/update-dependencies

Update to opensafely-jobrunner v2.26.3

opensafely/__init__.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 import sys
 
-from opensafely._vendor.jobrunner import local_run
+from opensafely._vendor.jobrunner.cli import local_run
 from opensafely import codelists, pull, upgrade
 
 

opensafely/_vendor/jobrunner/add_job.py → opensafely/_vendor/jobrunner/cli/add_job.py

@@ -6,17 +6,17 @@
 import base64
 import dataclasses
 import pprint
-from pathlib import Path
-from urllib.parse import urlparse
 import secrets
 import textwrap
+from pathlib import Path
+from urllib.parse import urlparse
 
-from .log_utils import configure_logging
-from .sync import job_request_from_remote_format
-from .database import find_where
-from .models import Job
-from .create_or_update_jobs import create_or_update_jobs
-from .git import get_sha_from_remote_ref
+from opensafely._vendor.jobrunner.create_or_update_jobs import create_or_update_jobs
+from opensafely._vendor.jobrunner.lib.database import find_where
+from opensafely._vendor.jobrunner.lib.git import get_sha_from_remote_ref
+from opensafely._vendor.jobrunner.lib.log_utils import configure_logging
+from opensafely._vendor.jobrunner.models import Job
+from opensafely._vendor.jobrunner.sync import job_request_from_remote_format
 
 
 def main(

opensafely/_vendor/jobrunner/extract_stats.py → opensafely/_vendor/jobrunner/cli/extract_stats.py

@@ -3,10 +3,10 @@
 """
 import argparse
 import gzip
-from pathlib import Path
 import sqlite3
+from pathlib import Path
 
-from . import config
+from opensafely._vendor.jobrunner import config
 
 
 def main(output_file, since, gz=False):

opensafely/_vendor/jobrunner/kill_job.py → opensafely/_vendor/jobrunner/cli/kill_job.py

@@ -3,10 +3,10 @@
 """
 import argparse
 
-from .database import find_where
-from .models import Job, State
-from .run import mark_job_as_failed
-from .manage_jobs import docker, container_name, volume_name
+from opensafely._vendor.jobrunner.lib.database import find_where
+from opensafely._vendor.jobrunner.manage_jobs import container_name, docker, volume_name
+from opensafely._vendor.jobrunner.models import Job, State
+from opensafely._vendor.jobrunner.run import mark_job_as_failed
 
 
 def main(partial_job_ids, cleanup=False):

opensafely/_vendor/jobrunner/local_run.py → opensafely/_vendor/jobrunner/cli/local_run.py

@@ -19,11 +19,8 @@
 away afterwards.
 """
 import argparse
-from datetime import datetime, timedelta
 import getpass
-import json
 import os
-from pathlib import Path
 import platform
 import random
 import shlex
@@ -33,24 +30,25 @@
 import sys
 import tempfile
 import textwrap
+from datetime import datetime, timedelta
+from pathlib import Path
 
-from .run import main as run_main
-from . import config
-from . import docker
-from .database import find_where
-from .manage_jobs import METADATA_DIR
-from .models import JobRequest, Job, State, StatusCode
-from .create_or_update_jobs import (
-    create_jobs,
-    ProjectValidationError,
+from opensafely._vendor.jobrunner import config
+from opensafely._vendor.jobrunner.create_or_update_jobs import (
+    RUN_ALL_COMMAND,
     JobRequestError,
     NothingToDoError,
-    RUN_ALL_COMMAND,
+    ProjectValidationError,
+    create_jobs,
 )
-from .log_utils import configure_logging
-from .subprocess_utils import subprocess_run
-from .string_utils import tabulate
-
+from opensafely._vendor.jobrunner.lib import docker
+from opensafely._vendor.jobrunner.lib.database import find_where
+from opensafely._vendor.jobrunner.lib.log_utils import configure_logging
+from opensafely._vendor.jobrunner.lib.string_utils import tabulate
+from opensafely._vendor.jobrunner.lib.subprocess_utils import subprocess_run
+from opensafely._vendor.jobrunner.manage_jobs import METADATA_DIR
+from opensafely._vendor.jobrunner.models import Job, JobRequest, State, StatusCode
+from opensafely._vendor.jobrunner.run import main as run_main
 
 # First paragraph of docstring
 DESCRIPTION = __doc__.partition("\n\n")[0]
@@ -255,15 +253,27 @@ def create_and_run_jobs(
 
     jobs = find_where(Job)
 
-    for image in get_docker_images(jobs):
-        is_stata = "stata-mp" in image
+    docker_images = get_docker_images(jobs)
 
-        if is_stata and config.STATA_LICENSE is None:
-            config.STATA_LICENSE = get_stata_license()
-            if config.STATA_LICENSE is None:
-                # TODO switch this to failing when the stata image requires it
-                print("WARNING: no STATA_LICENSE found")
+    stata_images = [i for i in docker_images if i.startswith("stata-mp")]
+    if stata_images and config.STATA_LICENSE is None:
+        config.STATA_LICENSE = get_stata_license()
+        if config.STATA_LICENSE is None:
+            print(
+                f"The docker image '{stata_images[0]}' requires a license to function.\n"
+                f"\n"
+                f"If you are a member of OpenSAFELY we should have been able to fetch\n"
+                f"the license automatically, so something has gone wrong.  Please open\n"
+                f"a new discussion here so we can help:\n"
+                f"  https://github.com/opensafely/documentation/discussions\n"
+                f"\n"
+                f"If you are not a member of OpenSAFELY you will have to provide your\n"
+                f"own license. See the dicussion here for pointers:\n"
+                f" https://github.com/opensafely/documentation/discussions/299"
+            )
+            return False
 
+    for image in docker_images:
         if not docker.image_exists_locally(image):
             print(f"Fetching missing docker image: docker pull {image}")
             try:
@@ -272,14 +282,9 @@ def create_and_run_jobs(
                 # layer download noise
                 docker.pull(image, quiet=not sys.stdout.isatty())
             except docker.DockerPullError as e:
-                success = False
-                if is_stata:
-                    # best effort retry hack
-                    success = temporary_stata_workaround(image)
-                if not success:
-                    print("Failed with error:")
-                    print(e)
-                    return False
+                print("Failed with error:")
+                print(e)
+                return False
 
     action_names = [job.action for job in jobs]
     print(f"\nRunning actions: {', '.join(action_names)}\n")
@@ -463,44 +468,6 @@ def get_log_file_snippet(log_file, max_lines):
     return "\n".join(log_lines).strip(), truncated
 
 
-def temporary_stata_workaround(image):
-    """
-    This is a temporary workaround for the fact that the current crop of Github
-    Actions have credentials for the old docker.opensafely.org registry but not
-    for ghcr.io. These are only needed for the one private image we have
-    (Stata) so we detect when we are in this situation and pull from the old
-    registry instead.
-
-    We'll shortly be updating the Github Actions to use an entirely new set of
-    commands in any case, so this will hopefully be a short-lived hack.
-    """
-    if not os.environ.get("GITHUB_WORKFLOW"):
-        return False
-
-    docker_config = os.environ.get("DOCKER_CONFIG", os.path.expanduser("~/.docker"))
-    config_path = Path(docker_config) / "config.json"
-    try:
-        config = json.loads(config_path.read_text())
-        auths = config["auths"]
-    except Exception:
-        return False
-
-    if "ghcr.io" in auths or "docker.opensafely.org" not in auths:
-        return False
-
-    print("Applying Docker authentication workaround...")
-    alt_image = image.replace("ghcr.io/opensafely-core/", "docker.opensafely.org/")
-
-    try:
-        docker.pull(alt_image, quiet=True)
-        print(f"Retagging '{alt_image}' as '{image}'")
-        subprocess_run(["docker", "tag", alt_image, image])
-    except Exception:
-        return False
-
-    return True
-
-
 def get_stata_license(repo=config.STATA_LICENSE_REPO):
     """Load a stata license from local cache or remote repo."""
     cached = Path(f"{tempfile.gettempdir()}/opensafely-stata.lic")

opensafely/_vendor/jobrunner/retry_job.py → opensafely/_vendor/jobrunner/cli/retry_job.py

@@ -17,10 +17,10 @@
 import argparse
 import time
 
-from .sync import api_post, job_to_remote_format
-from .database import find_where, update
-from .models import Job, State
-from .manage_jobs import docker, container_name
+from opensafely._vendor.jobrunner.lib.database import find_where, update
+from opensafely._vendor.jobrunner.manage_jobs import container_name, docker
+from opensafely._vendor.jobrunner.models import Job, State
+from opensafely._vendor.jobrunner.sync import api_post, job_to_remote_format
 
 
 def main(partial_job_id):

opensafely/_vendor/jobrunner/config.py

@@ -1,8 +1,8 @@
 import configparser
 import os
-from pathlib import Path
 import re
 from multiprocessing import cpu_count
+from pathlib import Path
 
 
 class ConfigException(Exception):
@@ -18,6 +18,7 @@ class ConfigException(Exception):
 GIT_REPO_DIR = WORK_DIR / "repos"
 
 DATABASE_FILE = WORK_DIR / "db.sqlite"
+DATABASE_SCHEMA_FILE = Path(__file__).parent / "schema.sql"
 
 HIGH_PRIVACY_STORAGE_BASE = Path(
     os.environ.get("HIGH_PRIVACY_STORAGE_BASE", WORK_DIR / "high_privacy")
@@ -50,7 +51,14 @@ class ConfigException(Exception):
 else:
     USING_DUMMY_DATA_BACKEND = BACKEND == "expectations"
 
-ALLOWED_IMAGES = {"cohortextractor", "cohortextractor-v2", "stata-mp", "r", "jupyter", "python"}
+ALLOWED_IMAGES = {
+    "cohortextractor",
+    "cohortextractor-v2",
+    "stata-mp",
+    "r",
+    "jupyter",
+    "python",
+}
 
 DOCKER_REGISTRY = "ghcr.io/opensafely-core"