Fix: Replace v1 dockerhub api with v2 json files

Signed-off-by: Bengt Thuree <[email protected]>
Change-Id: Iac2e23671bb421745099a1b4cbc56187ab39474f

lftools/nexus/release_docker_hub.py

@@ -40,6 +40,7 @@
 """
 from __future__ import print_function
 
+import json
 import logging
 import multiprocessing
 import os
@@ -269,14 +270,18 @@ class DockerTagClass(TagClass):
     This class fetches and stores all docker tags for a repository.
 
     Doing this manually from command line, you will give this command:
-        curl -s https://registry.hub.docker.com/v1/repositories/onap/base_sdc-sanity/tags
-    which gives you the following output:
-        {"layer": "", "name": "latest"}, {"layer": "", "name": "1.3.0"},
-        {"layer": "", "name": "1.3.1"}, {"layer": "", "name": "1.4.0"},
-        {"layer": "", "name": "1.4.1"}, {"layer": "", "name": "v1.0.0"}]
+        curl -s https://registry.hub.docker.com:443/v2/namespaces/onap/repositories/base_sdc-sanity/tags
+    which gives you a json output. Just looking for the tag names we do this
+        curl -s https://registry.hub.docker.com:443/v2/namespaces/onap/repositories/base_sdc-sanity/tags | jq -r ".results[].name"
+            latest
+            1.7.0
+            1.6.0
+            1.4.1
+            1.4.0
+            1.3.1
+            1.3.0
+            v1.0.0
 
-    When we fetch the tags from the docker repository url, they are returned like
-        [{"layer": "", "name": "latest"}, {"layer": "", "name": "v1.0.0"}]
     Hence, we need to extract all the tags, and add them to our list of valid or
     invalid tags.
     If we fail to collect the tags, we set the repository_exist flag to false.
@@ -292,7 +297,7 @@ class DockerTagClass(TagClass):
         If no repository is found, self.repository_exist will be set to False.
     """
 
-    _docker_base = "https://registry.hub.docker.com/v1/repositories"
+    _docker_base_start = "https://registry.hub.docker.com/v2/namespaces/"
 
     def __init__(self, org_name, repo_name, repo_from_file):
         """Initialize this class."""
@@ -302,49 +307,55 @@ def __init__(self, org_name, repo_name, repo_from_file):
         else:
             combined_repo_name = "{}/{}".format(org_name, repo_name)
         log.debug("Fetching docker tags for {}".format(combined_repo_name))
-        retries = 0
-        while retries < 20:
-            try:
-                r = _request_get(self._docker_base + "/" + combined_repo_name + "/tags")
-                if r.status_code == 429:
-                    # Docker returns 429 if we access it too fast too many times.
-                    # If it happends, delay 60 seconds, and try again, up to 19 times.
-                    log.debug(
-                        "Too many docker gets too fast, wait 1 min: {}, repo {}".format(retries, combined_repo_name)
-                    )
-                    time.sleep(60)
+        _docker_base = self._docker_base_start + "{}/repositories".format(org_name)
+        still_more = True
+        docker_tag_url = _docker_base + "/" + repo_name + "/tags"
+        while still_more:
+            retries = 0
+            while retries < 20:
+                try:
+                    log.debug("URL={}".format(docker_tag_url))
+                    r = _request_get(docker_tag_url)
+                    if r.status_code == 429:
+                        # Docker returns 429 if we access it too fast too many times.
+                        # If it happends, delay 60 seconds, and try again, up to 19 times.
+                        log.debug(
+                            "Too many docker gets too fast, wait 1 min: {}, repo {}".format(retries, combined_repo_name)
+                        )
+                        time.sleep(60)
+                        retries = retries + 1
+                    else:
+                        break
+                except requests.HTTPError as excinfo:
+                    log.debug("Fetching Docker Hub tags. {}".format(excinfo))
                     retries = retries + 1
-                else:
-                    break
-            except requests.HTTPError as excinfo:
-                log.debug("Fetching Docker Hub tags. {}".format(excinfo))
-                retries = retries + 1
-                if retries > 19:
-                    self.repository_exist = False
-                    return
-
-        log.debug("r.status_code = {}, ok={}".format(r.status_code, r.status_code == requests.codes.ok))
-        if r.status_code == 429:
-            # Speed throttling in effect. Cancel program
-            raise requests.HTTPError(
-                "Speed throttling in effect. To fast accessing dockerhub for tags.\n {}".format(r.text)
-            )
-        if r.status_code == requests.codes.ok:
-            raw_tags = r.text
-            raw_tags = raw_tags.replace("}]", "")
-            raw_tags = raw_tags.replace("[{", "")
-            raw_tags = raw_tags.replace("{", "")
-            raw_tags = raw_tags.replace('"', "")
-            raw_tags = raw_tags.replace(" ", "")
-            TmpSplittedTuple = raw_tags.split("}")
-            if len(TmpSplittedTuple) > 0:
-                for tuple in TmpSplittedTuple:
-                    tmp_tuple = tuple.split(":")
-                    if len(tmp_tuple) > 1:
-                        self.add_tag(tmp_tuple[2].strip())
-                        log.debug("Docker {} has tag {}".format(combined_repo_name, tmp_tuple[2]))
-        else:
-            self.repository_exist = False
+                    if retries > 19:
+                        self.repository_exist = False
+                        return
+
+            log.debug("r.status_code = {}, ok={}".format(r.status_code, r.status_code == requests.codes.ok))
+            if r.status_code == 429:
+                # Speed throttling in effect. Cancel program
+                raise requests.HTTPError(
+                    "Dockerhub throttling at tag fetching.\n {}".format(r.text)
+                )
+            if r.status_code == requests.codes.ok:
+                raw_json = json.loads(r.text)
+
+                try:
+                    for result in raw_json["results"]:
+                        tag_name = result["name"]
+                        self.add_tag(tag_name)
+                        log.debug("Docker {} has tag {}".format(combined_repo_name, tag_name))
+                except:
+                    log.debug("Issue fetching tags for {}".format(combined_repo_name))
+            else:
+                self.repository_exist = False
+            if raw_json["next"]:
+                docker_tag_url = raw_json["next"]
+                still_more = True
+            else:
+                still_more = False
 
 
 class ProjectClass:

releasenotes/notes/Move-to-v2-on-dockerhub-tag-api-62834fe85a7808f8.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Dockerhub retired the v1 version of the dockerhub api, which was used
+    to collect the existing tag information. This patch replaces v1 with v2
+    which is using json files.

tests/fixtures/nexus/releasedockerhub_dockertags-gizmo.json

@@ -0,0 +1,224 @@
+{
+  "count": 7,
+  "next": null,
+  "previous": null,
+  "results": [
+    {
+      "creator": 4339606,
+      "id": 68582206,
+      "images": [
+        {
+          "architecture": "amd64",
+          "features": "",
+          "variant": null,
+          "digest": "sha256:b5394b4697cfeaae2494a5c8ac9254c9415213f4a1241b5d63137110f54ab8d0",
+          "os": "linux",
+          "os_features": "",
+          "os_version": null,
+          "size": 236420979,
+          "status": "active",
+          "last_pulled": "2022-08-14T13:55:25.099592Z",
+          "last_pushed": "2022-11-16T12:02:42.108521Z"
+        }
+      ],
+      "last_updated": "2022-11-16T12:02:42.108521Z",
+      "last_updater": 2325232,
+      "last_updater_username": "onapdockerhub",
+      "name": "1.5.2",
+      "repository": 6189598,
+      "full_size": 236420979,
+      "v2": true,
+      "tag_status": "active",
+      "tag_last_pulled": "2022-08-14T13:55:25.099592Z",
+      "tag_last_pushed": "2022-11-16T12:02:42.108521Z",
+      "media_type": "application/vnd.docker.container.image.v1+json",
+      "digest": "sha256:b5394b4697cfeaae2494a5c8ac9254c9415213f4a1241b5d63137110f54ab8d0"
+    },
+    {
+      "creator": 4339606,
+      "id": 54419291,
+      "images": [
+        {
+          "architecture": "amd64",
+          "features": "",
+          "variant": null,
+          "digest": "sha256:04bcb7034020c3740d57abff5bc205270c27f0023a093f2fd5d902487a3e01d2",
+          "os": "linux",
+          "os_features": "",
+          "os_version": null,
+          "size": 330956612,
+          "status": "active",
+          "last_pulled": "2022-01-18T13:34:50.997188Z",
+          "last_pushed": "2022-11-16T12:02:22.142022Z"
+        }
+      ],
+      "last_updated": "2022-11-16T12:02:22.142022Z",
+      "last_updater": 2325232,
+      "last_updater_username": "onapdockerhub",
+      "name": "1.4.0",
+      "repository": 6189598,
+      "full_size": 330956612,
+      "v2": true,
+      "tag_status": "active",
+      "tag_last_pulled": "2022-01-18T13:34:50.997188Z",
+      "tag_last_pushed": "2022-11-16T12:02:22.142022Z",
+      "media_type": "application/vnd.docker.container.image.v1+json",
+      "digest": "sha256:04bcb7034020c3740d57abff5bc205270c27f0023a093f2fd5d902487a3e01d2"
+    },
+    {
+      "creator": 4339606,
+      "id": 44037046,
+      "images": [
+        {
+          "architecture": "amd64",
+          "features": "",
+          "variant": null,
+          "digest": "sha256:5579c0f5f46f75fd23effc319a8d8169253c720fc3aa83fcb5ef2cc44a9f79a7",
+          "os": "linux",
+          "os_features": "",
+          "os_version": null,
+          "size": 306777861,
+          "status": "active",
+          "last_pulled": "2022-01-18T13:34:47.196389Z",
+          "last_pushed": "2022-11-16T12:01:51.181469Z"
+        }
+      ],
+      "last_updated": "2022-11-16T12:01:51.181469Z",
+      "last_updater": 2325232,
+      "last_updater_username": "onapdockerhub",
+      "name": "1.3.2",
+      "repository": 6189598,
+      "full_size": 306777861,
+      "v2": true,
+      "tag_status": "active",
+      "tag_last_pulled": "2022-01-18T13:34:47.196389Z",
+      "tag_last_pushed": "2022-11-16T12:01:51.181469Z",
+      "media_type": "application/vnd.docker.container.image.v1+json",
+      "digest": "sha256:5579c0f5f46f75fd23effc319a8d8169253c720fc3aa83fcb5ef2cc44a9f79a7"
+    },
+    {
+      "creator": 4339606,
+      "id": 39041406,
+      "images": [
+        {
+          "architecture": "amd64",
+          "features": "",
+          "variant": null,
+          "digest": "sha256:39c30a61fe8705536dd193bbfbd96820901700a7380c5d75f081d1f8312aa0e3",
+          "os": "linux",
+          "os_features": "",
+          "os_version": null,
+          "size": 309691887,
+          "status": "active",
+          "last_pulled": "2022-08-14T13:55:40.256178Z",
+          "last_pushed": "2022-11-16T12:01:22.802907Z"
+        }
+      ],
+      "last_updated": "2022-11-16T12:01:22.802907Z",
+      "last_updater": 2325232,
+      "last_updater_username": "onapdockerhub",
+      "name": "1.3.1",
+      "repository": 6189598,
+      "full_size": 309691887,
+      "v2": true,
+      "tag_status": "active",
+      "tag_last_pulled": "2022-08-14T13:55:40.256178Z",
+      "tag_last_pushed": "2022-11-16T12:01:22.802907Z",
+      "media_type": "application/vnd.docker.container.image.v1+json",
+      "digest": "sha256:39c30a61fe8705536dd193bbfbd96820901700a7380c5d75f081d1f8312aa0e3"
+    },
+    {
+      "creator": 4339606,
+      "id": 38206299,
+      "images": [
+        {
+          "architecture": "amd64",
+          "features": "",
+          "variant": null,
+          "digest": "sha256:3945e98997fb875b6d5f8db21967f1b3c05023ff5b5986f14b99a3ada7c6bf90",
+          "os": "linux",
+          "os_features": "",
+          "os_version": null,
+          "size": 334469313,
+          "status": "active",
+          "last_pulled": "2022-08-14T13:55:44.929781Z",
+          "last_pushed": "2022-11-16T12:00:57.436924Z"
+        }
+      ],
+      "last_updated": "2022-11-16T12:00:57.436924Z",
+      "last_updater": 2325232,
+      "last_updater_username": "onapdockerhub",
+      "name": "1.3.0",
+      "repository": 6189598,
+      "full_size": 334469313,
+      "v2": true,
+      "tag_status": "active",
+      "tag_last_pulled": "2022-08-14T13:55:44.929781Z",
+      "tag_last_pushed": "2022-11-16T12:00:57.436924Z",
+      "media_type": "application/vnd.docker.container.image.v1+json",
+      "digest": "sha256:3945e98997fb875b6d5f8db21967f1b3c05023ff5b5986f14b99a3ada7c6bf90"
+    },
+    {
+      "creator": 4339606,
+      "id": 38205809,
+      "images": [
+        {
+          "architecture": "amd64",
+          "features": "",
+          "variant": null,
+          "digest": "sha256:af7298b66c480407d7a8bc650b3122b3c0180038f59c57eaa6903786823ded4b",
+          "os": "linux",
+          "os_features": "",
+          "os_version": null,
+          "size": 335766308,
+          "status": "active",
+          "last_pulled": "2022-01-18T13:34:41.188991Z",
+          "last_pushed": "2022-11-16T12:00:32.417426Z"
+        }
+      ],
+      "last_updated": "2022-11-16T12:00:32.417426Z",
+      "last_updater": 2325232,
+      "last_updater_username": "onapdockerhub",
+      "name": "1.2.1",
+      "repository": 6189598,
+      "full_size": 335766308,
+      "v2": true,
+      "tag_status": "active",
+      "tag_last_pulled": "2022-01-18T13:34:41.188991Z",
+      "tag_last_pushed": "2022-11-16T12:00:32.417426Z",
+      "media_type": "application/vnd.docker.container.image.v1+json",
+      "digest": "sha256:af7298b66c480407d7a8bc650b3122b3c0180038f59c57eaa6903786823ded4b"
+    },
+    {
+      "creator": 4339606,
+      "id": 38205256,
+      "images": [
+        {
+          "architecture": "amd64",
+          "features": "",
+          "variant": null,
+          "digest": "sha256:a09ec6b5443e94aeab2b61491e56af124a2d50c9eab3e555f7c6fdd22309598b",
+          "os": "linux",
+          "os_features": "",
+          "os_version": null,
+          "size": 335764567,
+          "status": "active",
+          "last_pulled": "2022-01-18T13:34:40.069568Z",
+          "last_pushed": "2022-11-16T11:59:51.061067Z"
+        }
+      ],
+      "last_updated": "2022-11-16T11:59:51.061067Z",
+      "last_updater": 2325232,
+      "last_updater_username": "onapdockerhub",
+      "name": "1.2.0",
+      "repository": 6189598,
+      "full_size": 335764567,
+      "v2": true,
+      "tag_status": "active",
+      "tag_last_pulled": "2022-01-18T13:34:40.069568Z",
+      "tag_last_pushed": "2022-11-16T11:59:51.061067Z",
+      "media_type": "application/vnd.docker.container.image.v1+json",
+      "digest": "sha256:a09ec6b5443e94aeab2b61491e56af124a2d50c9eab3e555f7c6fdd22309598b"
+    }
+  ]
+}