misc.python.materialize.docker

  1# Copyright Materialize, Inc. and contributors. All rights reserved.
  2#
  3# Use of this software is governed by the Business Source License
  4# included in the LICENSE file at the root of this repository.
  5#
  6# As of the Change Date specified in that file, in accordance with
  7# the Business Source License, use of this software will be governed
  8# by the Apache License, Version 2.0.
  9
 10"""Docker utilities."""
 11import re
 12import subprocess
 13import time
 14
 15import requests
 16
 17from materialize.mz_version import MzVersion
 18
 19CACHED_IMAGE_NAME_BY_COMMIT_HASH: dict[str, str] = dict()
 20EXISTENCE_OF_IMAGE_NAMES_FROM_EARLIER_CHECK: dict[str, bool] = dict()
 21
 22IMAGE_TAG_OF_DEV_VERSION_METADATA_SEPARATOR = "--"
 23LATEST_IMAGE_TAG = "latest"
 24LEGACY_IMAGE_TAG_COMMIT_PREFIX = "devel-"
 25
 26# Examples:
 27# * v0.114.0
 28# * v0.114.0-dev
 29# * v0.114.0-dev.0--pr.g3d565dd11ba1224a41beb6a584215d99e6b3c576
 30VERSION_IN_IMAGE_TAG_PATTERN = re.compile(r"^(v\d+\.\d+\.\d+(-dev)?)")
 31
 32
 33def image_of_release_version_exists(version: MzVersion) -> bool:
 34    if version.is_dev_version():
 35        raise ValueError(f"Version {version} is a dev version, not a release version")
 36
 37    return _mz_image_tag_exists(release_version_to_image_tag(version))
 38
 39
 40def image_of_commit_exists(commit_hash: str) -> bool:
 41    return _mz_image_tag_exists(commit_to_image_tag(commit_hash))
 42
 43
 44def _mz_image_tag_exists(image_tag: str) -> bool:
 45    image_name = f"materialize/materialized:{image_tag}"
 46
 47    if image_name in EXISTENCE_OF_IMAGE_NAMES_FROM_EARLIER_CHECK:
 48        image_exists = EXISTENCE_OF_IMAGE_NAMES_FROM_EARLIER_CHECK[image_name]
 49        print(
 50            f"Status of image {image_name} known from earlier check: {'exists' if image_exists else 'does not exist'}"
 51        )
 52        return image_exists
 53
 54    print(f"Checking existence of image manifest: {image_name}")
 55
 56    command_local = ["docker", "images", "--quiet", image_name]
 57
 58    output = subprocess.check_output(command_local, stderr=subprocess.STDOUT, text=True)
 59    if output:
 60        # image found locally, can skip querying remote Docker Hub
 61        EXISTENCE_OF_IMAGE_NAMES_FROM_EARLIER_CHECK[image_name] = True
 62        return True
 63
 64    # docker manifest inspect counts against the Docker Hub rate limits, even
 65    # when the image doesn't exist, see https://www.docker.com/increase-rate-limits/,
 66    # so use the API instead.
 67
 68    try:
 69        response = requests.get(
 70            f"https://hub.docker.com/v2/repositories/materialize/materialized/tags/{image_tag}"
 71        )
 72        result = response.json()
 73    except (requests.exceptions.ConnectionError, requests.exceptions.JSONDecodeError):
 74        command = [
 75            "docker",
 76            "manifest",
 77            "inspect",
 78            image_name,
 79        ]
 80        try:
 81            subprocess.check_output(command, stderr=subprocess.STDOUT, text=True)
 82            EXISTENCE_OF_IMAGE_NAMES_FROM_EARLIER_CHECK[image_name] = True
 83            return True
 84        except subprocess.CalledProcessError as e:
 85            if "no such manifest:" in e.output:
 86                print(f"Failed to fetch image manifest '{image_name}' (does not exist)")
 87                EXISTENCE_OF_IMAGE_NAMES_FROM_EARLIER_CHECK[image_name] = False
 88            else:
 89                print(f"Failed to fetch image manifest '{image_name}' ({e.output})")
 90                # do not cache the result of unknown error messages
 91            return False
 92
 93    if result.get("images"):
 94        EXISTENCE_OF_IMAGE_NAMES_FROM_EARLIER_CHECK[image_name] = True
 95        return True
 96    if "not found" in result.get("message", ""):
 97        EXISTENCE_OF_IMAGE_NAMES_FROM_EARLIER_CHECK[image_name] = False
 98        return False
 99    print(f"Failed to fetch image info from API: {result}")
100    # do not cache the result of unknown error messages
101    return False
102
103
104def commit_to_image_tag(commit_hash: str) -> str:
105    return _resolve_image_name_by_commit_hash(commit_hash)
106
107
108def release_version_to_image_tag(version: MzVersion) -> str:
109    return str(version)
110
111
112def is_image_tag_of_release_version(image_tag: str) -> bool:
113    return (
114        IMAGE_TAG_OF_DEV_VERSION_METADATA_SEPARATOR not in image_tag
115        and not image_tag.startswith(LEGACY_IMAGE_TAG_COMMIT_PREFIX)
116        and image_tag != LATEST_IMAGE_TAG
117    )
118
119
120def is_image_tag_of_commit(image_tag: str) -> bool:
121    return (
122        IMAGE_TAG_OF_DEV_VERSION_METADATA_SEPARATOR in image_tag
123        or image_tag.startswith(LEGACY_IMAGE_TAG_COMMIT_PREFIX)
124    )
125
126
127def get_version_from_image_tag(image_tag: str) -> str:
128    match = VERSION_IN_IMAGE_TAG_PATTERN.match(image_tag)
129    assert match is not None, f"Invalid image tag: {image_tag}"
130
131    return match.group(1)
132
133
134def get_mz_version_from_image_tag(image_tag: str) -> MzVersion:
135    return MzVersion.parse_mz(get_version_from_image_tag(image_tag))
136
137
138def _resolve_image_name_by_commit_hash(commit_hash: str) -> str:
139    if commit_hash in CACHED_IMAGE_NAME_BY_COMMIT_HASH.keys():
140        return CACHED_IMAGE_NAME_BY_COMMIT_HASH[commit_hash]
141
142    image_name_candidates = _search_docker_hub_for_image_name(search_value=commit_hash)
143    image_name = _select_image_name_from_candidates(image_name_candidates, commit_hash)
144
145    CACHED_IMAGE_NAME_BY_COMMIT_HASH[commit_hash] = image_name
146    EXISTENCE_OF_IMAGE_NAMES_FROM_EARLIER_CHECK[image_name] = True
147
148    return image_name
149
150
151def _search_docker_hub_for_image_name(
152    search_value: str, remaining_retries: int = 10
153) -> list[str]:
154    try:
155        json_response = requests.get(
156            f"https://hub.docker.com/v2/repositories/materialize/materialized/tags?name={search_value}"
157        ).json()
158    except (
159        requests.exceptions.ConnectionError,
160        requests.exceptions.JSONDecodeError,
161    ) as _:
162        if remaining_retries > 0:
163            print("Searching Docker Hub for image name failed, retrying in 5 seconds")
164            time.sleep(5)
165            return _search_docker_hub_for_image_name(
166                search_value, remaining_retries - 1
167            )
168
169        raise
170
171    json_results = json_response.get("results")
172
173    image_names = []
174
175    for entry in json_results:
176        image_name = entry.get("name")
177
178        if image_name.startswith("unstable-"):
179            # for images with the old version scheme favor "devel-" over "unstable-"
180            continue
181
182        image_names.append(image_name)
183
184    return image_names
185
186
187def _select_image_name_from_candidates(
188    image_name_candidates: list[str], commit_hash: str
189) -> str:
190    if len(image_name_candidates) == 0:
191        raise RuntimeError(f"No image found for commit hash {commit_hash}")
192
193    if len(image_name_candidates) > 1:
194        print(
195            f"Multiple images found for commit hash {commit_hash}: {image_name_candidates}, picking first"
196        )
197
198    return image_name_candidates[0]