misc.python.materialize.mzbuild

   1# Copyright Materialize, Inc. and contributors. All rights reserved.
   2#
   3# Use of this software is governed by the Business Source License
   4# included in the LICENSE file at the root of this repository.
   5#
   6# As of the Change Date specified in that file, in accordance with
   7# the Business Source License, use of this software will be governed
   8# by the Apache License, Version 2.0.
   9
  10"""The implementation of the mzbuild system for Docker images.
  11
  12For an overview of what mzbuild is and why it exists, see the [user-facing
  13documentation][user-docs].
  14
  15[user-docs]: https://github.com/MaterializeInc/materialize/blob/main/doc/developer/mzbuild.md
  16"""
  17
  18import argparse
  19import base64
  20import collections
  21import hashlib
  22import json
  23import multiprocessing
  24import os
  25import re
  26import shlex
  27import shutil
  28import stat
  29import subprocess
  30import sys
  31import tarfile
  32import time
  33from collections import OrderedDict
  34from collections.abc import Callable, Iterable, Iterator, Sequence
  35from concurrent.futures import ThreadPoolExecutor
  36from enum import Enum, auto
  37from functools import cache
  38from pathlib import Path
  39from tempfile import TemporaryFile
  40from typing import IO, Any, cast
  41
  42import yaml
  43
  44from materialize import bazel as bazel_utils
  45from materialize import cargo, git, rustc_flags, spawn, ui, xcompile
  46from materialize.rustc_flags import Sanitizer
  47from materialize.xcompile import Arch, target
  48
  49
  50class Fingerprint(bytes):
  51    """A SHA-1 hash of the inputs to an `Image`.
  52
  53    The string representation uses base32 encoding to distinguish mzbuild
  54    fingerprints from Git's hex encoded SHA-1 hashes while still being
  55    URL safe.
  56    """
  57
  58    def __str__(self) -> str:
  59        return base64.b32encode(self).decode()
  60
  61
  62class Profile(Enum):
  63    RELEASE = auto()
  64    OPTIMIZED = auto()
  65    DEV = auto()
  66
  67
  68class RepositoryDetails:
  69    """Immutable details about a `Repository`.
  70
  71    Used internally by mzbuild.
  72
  73    Attributes:
  74        root: The path to the root of the repository.
  75        arch: The CPU architecture to build for.
  76        profile: What profile the repository is being built with.
  77        coverage: Whether the repository has code coverage instrumentation
  78            enabled.
  79        sanitizer: Whether to use a sanitizer (address, hwaddress, cfi, thread, leak, memory, none)
  80        cargo_workspace: The `cargo.Workspace` associated with the repository.
  81        image_registry: The Docker image registry to pull images from and push
  82            images to.
  83        image_prefix: A prefix to apply to all Docker image names.
  84        bazel: Whether or not to use Bazel as the build system instead of Cargo.
  85        bazel_remote_cache: URL of a Bazel Remote Cache that we can build with.
  86    """
  87
  88    def __init__(
  89        self,
  90        root: Path,
  91        arch: Arch,
  92        profile: Profile,
  93        coverage: bool,
  94        sanitizer: Sanitizer,
  95        image_registry: str,
  96        image_prefix: str,
  97        bazel: bool,
  98        bazel_remote_cache: str | None,
  99    ):
 100        self.root = root
 101        self.arch = arch
 102        self.profile = profile
 103        self.coverage = coverage
 104        self.sanitizer = sanitizer
 105        self.cargo_workspace = cargo.Workspace(root)
 106        self.image_registry = image_registry
 107        self.image_prefix = image_prefix
 108        self.bazel = bazel
 109        self.bazel_remote_cache = bazel_remote_cache
 110
 111    def build(
 112        self,
 113        subcommand: str,
 114        rustflags: list[str],
 115        channel: str | None = None,
 116        extra_env: dict[str, str] = {},
 117    ) -> list[str]:
 118        """Start a build invocation for the configured architecture."""
 119        if self.bazel:
 120            assert not channel, "Bazel doesn't support building for multiple channels."
 121            return xcompile.bazel(
 122                arch=self.arch,
 123                subcommand=subcommand,
 124                rustflags=rustflags,
 125                extra_env=extra_env,
 126            )
 127        else:
 128            return xcompile.cargo(
 129                arch=self.arch,
 130                channel=channel,
 131                subcommand=subcommand,
 132                rustflags=rustflags,
 133                extra_env=extra_env,
 134            )
 135
 136    def tool(self, name: str) -> list[str]:
 137        """Start a binutils tool invocation for the configured architecture."""
 138        if self.bazel:
 139            return ["bazel", "run", f"@//misc/bazel/tools:{name}", "--"]
 140        else:
 141            return xcompile.tool(self.arch, name)
 142
 143    def cargo_target_dir(self) -> Path:
 144        """Determine the path to the target directory for Cargo."""
 145        return self.root / "target-xcompile" / xcompile.target(self.arch)
 146
 147    def bazel_workspace_dir(self) -> Path:
 148        """Determine the path to the root of the Bazel workspace."""
 149        return self.root
 150
 151    def bazel_config(self) -> list[str]:
 152        """Returns a set of Bazel config flags to set for the build."""
 153        flags = []
 154
 155        if self.profile == Profile.RELEASE:
 156            # If we're a tagged build, then we'll use stamping to update our
 157            # build info, otherwise we'll use our side channel/best-effort
 158            # approach to update it.
 159            if ui.env_is_truthy("BUILDKITE_TAG"):
 160                flags.append("--config=release-tagged")
 161            else:
 162                flags.append("--config=release-dev")
 163                bazel_utils.write_git_hash()
 164        elif self.profile == Profile.OPTIMIZED:
 165            flags.append("--config=optimized")
 166
 167        if self.bazel_remote_cache:
 168            flags.append(f"--remote_cache={self.bazel_remote_cache}")
 169
 170        if ui.env_is_truthy("CI"):
 171            flags.append("--config=ci")
 172
 173        return flags
 174
 175    def rewrite_builder_path_for_host(self, path: Path) -> Path:
 176        """Rewrite a path that is relative to the target directory inside the
 177        builder to a path that is relative to the target directory on the host.
 178
 179        If path does is not relative to the target directory inside the builder,
 180        it is returned unchanged.
 181        """
 182        builder_target_dir = Path("/mnt/build") / xcompile.target(self.arch)
 183        try:
 184            return self.cargo_target_dir() / path.relative_to(builder_target_dir)
 185        except ValueError:
 186            return path
 187
 188
 189def docker_images() -> set[str]:
 190    """List the Docker images available on the local machine."""
 191    return set(
 192        spawn.capture(["docker", "images", "--format", "{{.Repository}}:{{.Tag}}"])
 193        .strip()
 194        .split("\n")
 195    )
 196
 197
 198def is_docker_image_pushed(name: str) -> bool:
 199    """Check whether the named image is pushed to Docker Hub.
 200
 201    Note that this operation requires a rather slow network request.
 202    """
 203    proc = subprocess.run(
 204        ["docker", "manifest", "inspect", name],
 205        stdout=subprocess.DEVNULL,
 206        stderr=subprocess.DEVNULL,
 207        env=dict(os.environ, DOCKER_CLI_EXPERIMENTAL="enabled"),
 208    )
 209    return proc.returncode == 0
 210
 211
 212def chmod_x(path: Path) -> None:
 213    """Set the executable bit on a file or directory."""
 214    # https://stackoverflow.com/a/30463972/1122351
 215    mode = os.stat(path).st_mode
 216    mode |= (mode & 0o444) >> 2  # copy R bits to X
 217    os.chmod(path, mode)
 218
 219
 220class PreImage:
 221    """An action to run before building a Docker image.
 222
 223    Args:
 224        rd: The `RepositoryDetails` for the repository.
 225        path: The path to the `Image` associated with this action.
 226    """
 227
 228    def __init__(self, rd: RepositoryDetails, path: Path):
 229        self.rd = rd
 230        self.path = path
 231
 232    @classmethod
 233    def prepare_batch(cls, instances: list["PreImage"]) -> Any:
 234        """Prepare a batch of actions.
 235
 236        This is useful for `PreImage` actions that are more efficient when
 237        their actions are applied to several images in bulk.
 238
 239        Returns an arbitrary output that is passed to `PreImage.run`.
 240        """
 241        pass
 242
 243    def run(self, prep: Any) -> None:
 244        """Perform the action.
 245
 246        Args:
 247            prep: Any prep work returned by `prepare_batch`.
 248        """
 249        pass
 250
 251    def inputs(self) -> set[str]:
 252        """Return the files which are considered inputs to the action."""
 253        raise NotImplementedError
 254
 255    def extra(self) -> str:
 256        """Returns additional data for incorporation in the fingerprint."""
 257        return ""
 258
 259
 260class Copy(PreImage):
 261    """A `PreImage` action which copies files from a directory.
 262
 263    See doc/developer/mzbuild.md for an explanation of the user-facing
 264    parameters.
 265    """
 266
 267    def __init__(self, rd: RepositoryDetails, path: Path, config: dict[str, Any]):
 268        super().__init__(rd, path)
 269
 270        self.source = config.pop("source", None)
 271        if self.source is None:
 272            raise ValueError("mzbuild config is missing 'source' argument")
 273
 274        self.destination = config.pop("destination", None)
 275        if self.destination is None:
 276            raise ValueError("mzbuild config is missing 'destination' argument")
 277
 278        self.matching = config.pop("matching", "*")
 279
 280    def run(self, prep: Any) -> None:
 281        super().run(prep)
 282        for src in self.inputs():
 283            dst = self.path / self.destination / src
 284            dst.parent.mkdir(parents=True, exist_ok=True)
 285            shutil.copy(self.rd.root / self.source / src, dst)
 286
 287    def inputs(self) -> set[str]:
 288        return set(git.expand_globs(self.rd.root / self.source, self.matching))
 289
 290
 291class CargoPreImage(PreImage):
 292    """A `PreImage` action that uses Cargo."""
 293
 294    def inputs(self) -> set[str]:
 295        inputs = {
 296            "ci/builder",
 297            "Cargo.toml",
 298            # TODO(benesch): we could in theory fingerprint only the subset of
 299            # Cargo.lock that applies to the crates at hand, but that is a
 300            # *lot* of work.
 301            "Cargo.lock",
 302            ".cargo/config",
 303            # Even though we are not always building with Bazel, consider its
 304            # inputs so that developers with CI_BAZEL_BUILD=0 can still
 305            # download the images from Dockerhub
 306            ".bazelrc",
 307            "WORKSPACE",
 308        }
 309
 310        # Bazel has some rules and additive files that aren't directly
 311        # associated with a crate, but can change how it's built.
 312        additive_path = self.rd.root / "misc" / "bazel"
 313        additive_files = ["*.bazel", "*.bzl"]
 314        inputs |= {
 315            f"misc/bazel/{path}"
 316            for path in git.expand_globs(additive_path, *additive_files)
 317        }
 318
 319        return inputs
 320
 321    def extra(self) -> str:
 322        # Cargo images depend on the release mode and whether
 323        # coverage/sanitizer is enabled.
 324        flags: list[str] = []
 325        if self.rd.profile == Profile.RELEASE:
 326            flags += "release"
 327        if self.rd.profile == Profile.OPTIMIZED:
 328            flags += "optimized"
 329        if self.rd.coverage:
 330            flags += "coverage"
 331        if self.rd.sanitizer != Sanitizer.none:
 332            flags += self.rd.sanitizer.value
 333        flags.sort()
 334        return ",".join(flags)
 335
 336
 337class CargoBuild(CargoPreImage):
 338    """A `PreImage` action that builds a single binary with Cargo.
 339
 340    See doc/developer/mzbuild.md for an explanation of the user-facing
 341    parameters.
 342    """
 343
 344    def __init__(self, rd: RepositoryDetails, path: Path, config: dict[str, Any]):
 345        super().__init__(rd, path)
 346        bin = config.pop("bin", [])
 347        self.bins = bin if isinstance(bin, list) else [bin]
 348        example = config.pop("example", [])
 349        self.examples = example if isinstance(example, list) else [example]
 350        self.strip = config.pop("strip", True)
 351        self.extract = config.pop("extract", {})
 352
 353        bazel_bins = config.pop("bazel-bin")
 354        self.bazel_bins = (
 355            bazel_bins if isinstance(bazel_bins, dict) else {self.bins[0]: bazel_bins}
 356        )
 357        self.bazel_tars = config.pop("bazel-tar", {})
 358
 359        if len(self.bins) == 0 and len(self.examples) == 0:
 360            raise ValueError("mzbuild config is missing pre-build target")
 361        for bin in self.bins:
 362            if bin not in self.bazel_bins:
 363                raise ValueError(
 364                    f"need to specify a 'bazel-bin' for '{bin}' at '{path}'"
 365                )
 366
 367    @staticmethod
 368    def generate_bazel_build_command(
 369        rd: RepositoryDetails,
 370        bins: list[str],
 371        examples: list[str],
 372        bazel_bins: dict[str, str],
 373        bazel_tars: dict[str, str],
 374    ) -> list[str]:
 375        assert (
 376            rd.bazel
 377        ), "Programming error, tried to invoke Bazel when it is not enabled."
 378        assert not rd.coverage, "Bazel doesn't support building with coverage."
 379
 380        rustflags = []
 381        if rd.sanitizer == Sanitizer.none:
 382            rustflags += ["--cfg=tokio_unstable"]
 383
 384        extra_env = {
 385            "TSAN_OPTIONS": "report_bugs=0",  # build-scripts fail
 386        }
 387
 388        bazel_build = rd.build(
 389            "build",
 390            channel=None,
 391            rustflags=rustflags,
 392            extra_env=extra_env,
 393        )
 394
 395        for bin in bins:
 396            bazel_build.append(bazel_bins[bin])
 397        for tar in bazel_tars:
 398            bazel_build.append(tar)
 399        # TODO(parkmycar): Make sure cargo-gazelle generates rust_binary targets for examples.
 400        assert len(examples) == 0, "Bazel doesn't support building examples."
 401
 402        # Add extra Bazel config flags.
 403        bazel_build.extend(rd.bazel_config())
 404        # Add flags for the Sanitizer
 405        bazel_build.extend(rd.sanitizer.bazel_flags())
 406
 407        return bazel_build
 408
 409    @staticmethod
 410    def generate_cargo_build_command(
 411        rd: RepositoryDetails,
 412        bins: list[str],
 413        examples: list[str],
 414    ) -> list[str]:
 415        assert (
 416            not rd.bazel
 417        ), "Programming error, tried to invoke Cargo when Bazel is enabled."
 418
 419        rustflags = (
 420            rustc_flags.coverage
 421            if rd.coverage
 422            else (
 423                rustc_flags.sanitizer[rd.sanitizer]
 424                if rd.sanitizer != Sanitizer.none
 425                else ["--cfg=tokio_unstable"]
 426            )
 427        )
 428        cflags = (
 429            [
 430                f"--target={target(rd.arch)}",
 431                f"--gcc-toolchain=/opt/x-tools/{target(rd.arch)}/",
 432                "-fuse-ld=lld",
 433                f"--sysroot=/opt/x-tools/{target(rd.arch)}/{target(rd.arch)}/sysroot",
 434                f"-L/opt/x-tools/{target(rd.arch)}/{target(rd.arch)}/lib64",
 435            ]
 436            + rustc_flags.sanitizer_cflags[rd.sanitizer]
 437            if rd.sanitizer != Sanitizer.none
 438            else []
 439        )
 440        extra_env = (
 441            {
 442                "CFLAGS": " ".join(cflags),
 443                "CXXFLAGS": " ".join(cflags),
 444                "LDFLAGS": " ".join(cflags),
 445                "CXXSTDLIB": "stdc++",
 446                "CC": "cc",
 447                "CXX": "c++",
 448                "CPP": "clang-cpp-15",
 449                "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER": "cc",
 450                "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER": "cc",
 451                "PATH": f"/sanshim:/opt/x-tools/{target(rd.arch)}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
 452                "TSAN_OPTIONS": "report_bugs=0",  # build-scripts fail
 453            }
 454            if rd.sanitizer != Sanitizer.none
 455            else {}
 456        )
 457
 458        cargo_build = rd.build(
 459            "build", channel=None, rustflags=rustflags, extra_env=extra_env
 460        )
 461
 462        packages = set()
 463        for bin in bins:
 464            cargo_build.extend(["--bin", bin])
 465            packages.add(rd.cargo_workspace.crate_for_bin(bin).name)
 466        for example in examples:
 467            cargo_build.extend(["--example", example])
 468            packages.add(rd.cargo_workspace.crate_for_example(example).name)
 469        cargo_build.extend(f"--package={p}" for p in packages)
 470
 471        if rd.profile == Profile.RELEASE:
 472            cargo_build.append("--release")
 473        if rd.profile == Profile.OPTIMIZED:
 474            cargo_build.extend(["--profile", "optimized"])
 475        if rd.sanitizer != Sanitizer.none:
 476            # ASan doesn't work with jemalloc
 477            cargo_build.append("--no-default-features")
 478            # Uses more memory, so reduce the number of jobs
 479            cargo_build.extend(
 480                ["--jobs", str(round(multiprocessing.cpu_count() * 2 / 3))]
 481            )
 482
 483        return cargo_build
 484
 485    @classmethod
 486    def prepare_batch(cls, cargo_builds: list["PreImage"]) -> dict[str, Any]:
 487        super().prepare_batch(cargo_builds)
 488
 489        if not cargo_builds:
 490            return {}
 491
 492        # Building all binaries and examples in the same `cargo build` command
 493        # allows Cargo to link in parallel with other work, which can
 494        # meaningfully speed up builds.
 495
 496        rd: RepositoryDetails | None = None
 497        builds = cast(list[CargoBuild], cargo_builds)
 498        bins = set()
 499        examples = set()
 500        bazel_bins = dict()
 501        bazel_tars = dict()
 502        for build in builds:
 503            if not rd:
 504                rd = build.rd
 505            bins.update(build.bins)
 506            examples.update(build.examples)
 507            bazel_bins.update(build.bazel_bins)
 508            bazel_tars.update(build.bazel_tars)
 509        assert rd
 510
 511        ui.section(f"Common build for: {', '.join(bins | examples)}")
 512
 513        if rd.bazel:
 514            cargo_build = cls.generate_bazel_build_command(
 515                rd, list(bins), list(examples), bazel_bins, bazel_tars
 516            )
 517        else:
 518            cargo_build = cls.generate_cargo_build_command(
 519                rd, list(bins), list(examples)
 520            )
 521
 522        spawn.runv(cargo_build, cwd=rd.root)
 523
 524        # Re-run with JSON-formatted messages and capture the output so we can
 525        # later analyze the build artifacts in `run`. This should be nearly
 526        # instantaneous since we just compiled above with the same crates and
 527        # features. (We don't want to do the compile above with JSON-formatted
 528        # messages because it wouldn't be human readable.)
 529        if rd.bazel:
 530            # TODO(parkmycar): Having to assign the same compilation flags as the build process
 531            # is a bit brittle. It would be better if the Bazel build process itself could
 532            # output the file to a known location.
 533            options = rd.bazel_config()
 534            paths_to_binaries = {}
 535            for bin in bins:
 536                paths = bazel_utils.output_paths(bazel_bins[bin], options)
 537                assert len(paths) == 1, f"{bazel_bins[bin]} output more than 1 file"
 538                paths_to_binaries[bin] = paths[0]
 539            for tar in bazel_tars:
 540                paths = bazel_utils.output_paths(tar, options)
 541                assert len(paths) == 1, f"more than one output path found for '{tar}'"
 542                paths_to_binaries[tar] = paths[0]
 543            prep = {"bazel": paths_to_binaries}
 544        else:
 545            json_output = spawn.capture(
 546                cargo_build + ["--message-format=json"],
 547                cwd=rd.root,
 548            )
 549            prep = {"cargo": json_output}
 550
 551        return prep
 552
 553    def build(self, build_output: dict[str, Any]) -> None:
 554        cargo_profile = (
 555            "release"
 556            if self.rd.profile == Profile.RELEASE
 557            else "optimized" if self.rd.profile == Profile.OPTIMIZED else "debug"
 558        )
 559
 560        def copy(src: Path, relative_dst: Path) -> None:
 561            exe_path = self.path / relative_dst
 562            exe_path.parent.mkdir(parents=True, exist_ok=True)
 563            shutil.copy(src, exe_path)
 564
 565            # Bazel doesn't add write or exec permissions for built binaries
 566            # but `strip` and `objcopy` need write permissions and we add exec
 567            # permissions for the built Docker images.
 568            current_perms = os.stat(exe_path).st_mode
 569            new_perms = (
 570                current_perms
 571                # chmod +wx
 572                | stat.S_IWUSR
 573                | stat.S_IWGRP
 574                | stat.S_IWOTH
 575                | stat.S_IXUSR
 576                | stat.S_IXGRP
 577                | stat.S_IXOTH
 578            )
 579            os.chmod(exe_path, new_perms)
 580
 581            if self.strip:
 582                # The debug information is large enough that it slows down CI,
 583                # since we're packaging these binaries up into Docker images and
 584                # shipping them around.
 585                spawn.runv(
 586                    [*self.rd.tool("strip"), "--strip-debug", exe_path],
 587                    cwd=self.rd.root,
 588                )
 589            else:
 590                # Even if we've been asked not to strip the binary, remove the
 591                # `.debug_pubnames` and `.debug_pubtypes` sections. These are just
 592                # indexes that speed up launching a debugger against the binary,
 593                # and we're happy to have slower debugger start up in exchange for
 594                # smaller binaries. Plus the sections have been obsoleted by a
 595                # `.debug_names` section in DWARF 5, and so debugger support for
 596                # `.debug_pubnames`/`.debug_pubtypes` is minimal anyway.
 597                # See: https://github.com/rust-lang/rust/issues/46034
 598                spawn.runv(
 599                    [
 600                        *self.rd.tool("objcopy"),
 601                        "-R",
 602                        ".debug_pubnames",
 603                        "-R",
 604                        ".debug_pubtypes",
 605                        exe_path,
 606                    ],
 607                    cwd=self.rd.root,
 608                )
 609
 610        for bin in self.bins:
 611            if "bazel" in build_output:
 612                src_path = self.rd.bazel_workspace_dir() / build_output["bazel"][bin]
 613            else:
 614                src_path = self.rd.cargo_target_dir() / cargo_profile / bin
 615            copy(src_path, bin)
 616        for example in self.examples:
 617            src_path = (
 618                self.rd.cargo_target_dir() / cargo_profile / Path("examples") / example
 619            )
 620            copy(src_path, Path("examples") / example)
 621
 622        # Bazel doesn't support 'extract', instead you need to use 'bazel-tar'
 623        if self.extract and "bazel" not in build_output:
 624            cargo_build_json_output = build_output["cargo"]
 625
 626            target_dir = self.rd.cargo_target_dir()
 627            for line in cargo_build_json_output.split("\n"):
 628                if line.strip() == "" or not line.startswith("{"):
 629                    continue
 630                message = json.loads(line)
 631                if message["reason"] != "build-script-executed":
 632                    continue
 633                out_dir = self.rd.rewrite_builder_path_for_host(
 634                    Path(message["out_dir"])
 635                )
 636                if not out_dir.is_relative_to(target_dir):
 637                    # Some crates are built for both the host and the target.
 638                    # Ignore the built-for-host out dir.
 639                    continue
 640                # parse the package name from a package_id that looks like one of:
 641                # git+https://github.com/MaterializeInc/rust-server-sdk#launchdarkly-server-sdk@1.0.0
 642                # path+file:///Users/roshan/materialize/src/catalog#mz-catalog@0.0.0
 643                # registry+https://github.com/rust-lang/crates.io-index#num-rational@0.4.0
 644                # file:///path/to/my-package#0.1.0
 645                package_id = message["package_id"]
 646                if "@" in package_id:
 647                    package = package_id.split("@")[0].split("#")[-1]
 648                else:
 649                    package = message["package_id"].split("#")[0].split("/")[-1]
 650                for src, dst in self.extract.get(package, {}).items():
 651                    spawn.runv(["cp", "-R", out_dir / src, self.path / dst])
 652
 653        if self.bazel_tars and "bazel" in build_output:
 654            ui.section("Extracing 'bazel-tar'")
 655            for tar in self.bazel_tars:
 656                # Where Bazel built the tarball.
 657                tar_path = self.rd.bazel_workspace_dir() / build_output["bazel"][tar]
 658                # Where we need to extract it into.
 659                tar_dest = self.path / self.bazel_tars[tar]
 660                ui.say(f"extracing {tar_path} to {tar_dest}")
 661
 662                with tarfile.open(tar_path, "r") as tar_file:
 663                    os.makedirs(tar_dest, exist_ok=True)
 664                    tar_file.extractall(path=tar_dest)
 665
 666        self.acquired = True
 667
 668    def run(self, prep: dict[str, Any]) -> None:
 669        super().run(prep)
 670        self.build(prep)
 671
 672    def inputs(self) -> set[str]:
 673        deps = set()
 674
 675        for bin in self.bins:
 676            crate = self.rd.cargo_workspace.crate_for_bin(bin)
 677            deps |= self.rd.cargo_workspace.transitive_path_dependencies(crate)
 678        for example in self.examples:
 679            crate = self.rd.cargo_workspace.crate_for_example(example)
 680            deps |= self.rd.cargo_workspace.transitive_path_dependencies(
 681                crate, dev=True
 682            )
 683        inputs = super().inputs() | set(inp for dep in deps for inp in dep.inputs())
 684        # Even though we are not always building with Bazel, consider its
 685        # inputs so that developers with CI_BAZEL_BUILD=0 can still
 686        # download the images from Dockerhub
 687        inputs |= {"BUILD.bazel"}
 688
 689        return inputs
 690
 691
 692class Image:
 693    """A Docker image whose build and dependencies are managed by mzbuild.
 694
 695    An image corresponds to a directory in a repository that contains a
 696    `mzbuild.yml` file. This directory is called an "mzbuild context."
 697
 698    Attributes:
 699        name: The name of the image.
 700        publish: Whether the image should be pushed to Docker Hub.
 701        depends_on: The names of the images upon which this image depends.
 702        root: The path to the root of the associated `Repository`.
 703        path: The path to the directory containing the `mzbuild.yml`
 704            configuration file.
 705        pre_images: Optional actions to perform before running `docker build`.
 706        build_args: An optional list of --build-arg to pass to the dockerfile
 707    """
 708
 709    _DOCKERFILE_MZFROM_RE = re.compile(rb"^MZFROM\s*(\S+)")
 710
 711    def __init__(self, rd: RepositoryDetails, path: Path):
 712        self.rd = rd
 713        self.path = path
 714        self.pre_images: list[PreImage] = []
 715        with open(self.path / "mzbuild.yml") as f:
 716            data = yaml.safe_load(f)
 717            self.name: str = data.pop("name")
 718            self.publish: bool = data.pop("publish", True)
 719            self.description: str | None = data.pop("description", None)
 720            self.mainline: bool = data.pop("mainline", True)
 721            for pre_image in data.pop("pre-image", []):
 722                typ = pre_image.pop("type", None)
 723                if typ == "cargo-build":
 724                    self.pre_images.append(CargoBuild(self.rd, self.path, pre_image))
 725                elif typ == "copy":
 726                    self.pre_images.append(Copy(self.rd, self.path, pre_image))
 727                else:
 728                    raise ValueError(
 729                        f"mzbuild config in {self.path} has unknown pre-image type"
 730                    )
 731            self.build_args = data.pop("build-args", {})
 732
 733        if re.search(r"[^A-Za-z0-9\-]", self.name):
 734            raise ValueError(
 735                f"mzbuild image name {self.name} contains invalid character; only alphanumerics and hyphens allowed"
 736            )
 737
 738        self.depends_on: list[str] = []
 739        with open(self.path / "Dockerfile", "rb") as f:
 740            for line in f:
 741                match = self._DOCKERFILE_MZFROM_RE.match(line)
 742                if match:
 743                    self.depends_on.append(match.group(1).decode())
 744
 745    def sync_description(self) -> None:
 746        """Sync the description to Docker Hub if the image is publishable
 747        and a README.md file exists."""
 748
 749        if not self.publish:
 750            ui.say(f"{self.name} is not publishable")
 751            return
 752
 753        readme_path = self.path / "README.md"
 754        has_readme = readme_path.exists()
 755        if not has_readme:
 756            ui.say(f"{self.name} has no README.md or description")
 757            return
 758
 759        docker_config = os.getenv("DOCKER_CONFIG")
 760        spawn.runv(
 761            [
 762                "docker",
 763                "pushrm",
 764                f"--file={readme_path}",
 765                *([f"--config={docker_config}/config.json"] if docker_config else []),
 766                *([f"--short={self.description}"] if self.description else []),
 767                self.docker_name(),
 768            ]
 769        )
 770
 771    def docker_name(self, tag: str | None = None) -> str:
 772        """Return the name of the image on Docker Hub at the given tag."""
 773        name = f"{self.rd.image_registry}/{self.rd.image_prefix}{self.name}"
 774        if tag:
 775            name += f":{tag}"
 776        return name
 777
 778
 779class ResolvedImage:
 780    """An `Image` whose dependencies have been resolved.
 781
 782    Attributes:
 783        image: The underlying `Image`.
 784        acquired: Whether the image is available locally.
 785        dependencies: A mapping from dependency name to `ResolvedImage` for
 786            each of the images that `image` depends upon.
 787    """
 788
 789    def __init__(self, image: Image, dependencies: Iterable["ResolvedImage"]):
 790        self.image = image
 791        self.acquired = False
 792        self.dependencies = {}
 793        for d in dependencies:
 794            self.dependencies[d.name] = d
 795
 796    def __repr__(self) -> str:
 797        return f"ResolvedImage<{self.spec()}>"
 798
 799    @property
 800    def name(self) -> str:
 801        """The name of the underlying image."""
 802        return self.image.name
 803
 804    @property
 805    def publish(self) -> bool:
 806        """Whether the underlying image should be pushed to Docker Hub."""
 807        return self.image.publish
 808
 809    def spec(self) -> str:
 810        """Return the "spec" for the image.
 811
 812        A spec is the unique identifier for the image given its current
 813        fingerprint. It is a valid Docker Hub name.
 814        """
 815        return self.image.docker_name(tag=f"mzbuild-{self.fingerprint()}")
 816
 817    def write_dockerfile(self) -> IO[bytes]:
 818        """Render the Dockerfile without mzbuild directives.
 819
 820        Returns:
 821            file: A handle to a temporary file containing the adjusted
 822                Dockerfile."""
 823        with open(self.image.path / "Dockerfile", "rb") as f:
 824            lines = f.readlines()
 825        f = TemporaryFile()
 826        for line in lines:
 827            match = Image._DOCKERFILE_MZFROM_RE.match(line)
 828            if match:
 829                image = match.group(1).decode()
 830                spec = self.dependencies[image].spec()
 831                line = Image._DOCKERFILE_MZFROM_RE.sub(b"FROM %b" % spec.encode(), line)
 832            f.write(line)
 833        f.seek(0)
 834        return f
 835
 836    def build(self, prep: dict[type[PreImage], Any]) -> None:
 837        """Build the image from source.
 838
 839        Requires that the caller has already acquired all dependencies and
 840        prepared all `PreImage` actions via `PreImage.prepare_batch`.
 841        """
 842        ui.section(f"Building {self.spec()}")
 843        spawn.runv(["git", "clean", "-ffdX", self.image.path])
 844
 845        for pre_image in self.image.pre_images:
 846            pre_image.run(prep[type(pre_image)])
 847        build_args = {
 848            **self.image.build_args,
 849            "ARCH_GCC": str(self.image.rd.arch),
 850            "ARCH_GO": self.image.rd.arch.go_str(),
 851            "CI_SANITIZER": str(self.image.rd.sanitizer),
 852        }
 853        f = self.write_dockerfile()
 854        cmd: Sequence[str] = [
 855            "docker",
 856            "build",
 857            "-f",
 858            "-",
 859            *(f"--build-arg={k}={v}" for k, v in build_args.items()),
 860            "-t",
 861            self.spec(),
 862            f"--platform=linux/{self.image.rd.arch.go_str()}",
 863            str(self.image.path),
 864        ]
 865        spawn.runv(cmd, stdin=f, stdout=sys.stderr.buffer)
 866
 867    def try_pull(self, max_retries: int) -> bool:
 868        """Download the image if it does not exist locally. Returns whether it was found."""
 869        ui.header(f"Acquiring {self.spec()}")
 870        command = ["docker", "pull"]
 871        # --quiet skips printing the progress bar, which does not display well in CI.
 872        if ui.env_is_truthy("CI"):
 873            command.append("--quiet")
 874        command.append(self.spec())
 875        if not self.acquired:
 876            sleep_time = 1
 877            for retry in range(1, max_retries + 1):
 878                try:
 879                    spawn.runv(
 880                        command,
 881                        stdout=sys.stderr.buffer,
 882                    )
 883                    self.acquired = True
 884                except subprocess.CalledProcessError:
 885                    if retry < max_retries:
 886                        # There seems to be no good way to tell what error
 887                        # happened based on error code
 888                        # (https://github.com/docker/cli/issues/538) and we
 889                        # want to print output directly to terminal.
 890                        print(f"Retrying in {sleep_time}s ...")
 891                        time.sleep(sleep_time)
 892                        sleep_time *= 2
 893                        continue
 894                    else:
 895                        break
 896        return self.acquired
 897
 898    def is_published_if_necessary(self) -> bool:
 899        """Report whether the image exists on Docker Hub if it is publishable."""
 900        if self.publish and is_docker_image_pushed(self.spec()):
 901            ui.say(f"{self.spec()} already exists")
 902            return True
 903        return False
 904
 905    def run(
 906        self,
 907        args: list[str] = [],
 908        docker_args: list[str] = [],
 909        env: dict[str, str] = {},
 910    ) -> None:
 911        """Run a command in the image.
 912
 913        Creates a container from the image and runs the command described by
 914        `args` in the image.
 915        """
 916        envs = []
 917        for key, val in env.items():
 918            envs.extend(["--env", f"{key}={val}"])
 919        spawn.runv(
 920            [
 921                "docker",
 922                "run",
 923                "--tty",
 924                "--rm",
 925                *envs,
 926                "--init",
 927                *docker_args,
 928                self.spec(),
 929                *args,
 930            ],
 931        )
 932
 933    def list_dependencies(self, transitive: bool = False) -> set[str]:
 934        out = set()
 935        for dep in self.dependencies.values():
 936            out.add(dep.name)
 937            if transitive:
 938                out |= dep.list_dependencies(transitive)
 939        return out
 940
 941    def inputs(self, transitive: bool = False) -> set[str]:
 942        """List the files tracked as inputs to the image.
 943
 944        These files are used to compute the fingerprint for the image. See
 945        `ResolvedImage.fingerprint` for details.
 946
 947        Returns:
 948            inputs: A list of input files, relative to the root of the
 949                repository.
 950        """
 951        paths = set(git.expand_globs(self.image.rd.root, f"{self.image.path}/**"))
 952        if not paths:
 953            # While we could find an `mzbuild.yml` file for this service, expland_globs didn't
 954            # return any files that matched this service. At the very least, the `mzbuild.yml`
 955            # file itself should have been returned. We have a bug if paths is empty.
 956            raise AssertionError(
 957                f"{self.image.name} mzbuild exists but its files are unknown to git"
 958            )
 959        for pre_image in self.image.pre_images:
 960            paths |= pre_image.inputs()
 961        if transitive:
 962            for dep in self.dependencies.values():
 963                paths |= dep.inputs(transitive)
 964        return paths
 965
 966    @cache
 967    def fingerprint(self) -> Fingerprint:
 968        """Fingerprint the inputs to the image.
 969
 970        Compute the fingerprint of the image. Changing the contents of any of
 971        the files or adding or removing files to the image will change the
 972        fingerprint, as will modifying the inputs to any of its dependencies.
 973
 974        The image considers all non-gitignored files in its mzbuild context to
 975        be inputs. If it has a pre-image action, that action may add additional
 976        inputs via `PreImage.inputs`.
 977        """
 978        self_hash = hashlib.sha1()
 979        for rel_path in sorted(
 980            set(git.expand_globs(self.image.rd.root, *self.inputs()))
 981        ):
 982            abs_path = self.image.rd.root / rel_path
 983            file_hash = hashlib.sha1()
 984            raw_file_mode = os.lstat(abs_path).st_mode
 985            # Compute a simplified file mode using the same rules as Git.
 986            # https://github.com/git/git/blob/3bab5d562/Documentation/git-fast-import.txt#L610-L616
 987            if stat.S_ISLNK(raw_file_mode):
 988                file_mode = 0o120000
 989            elif raw_file_mode & stat.S_IXUSR:
 990                file_mode = 0o100755
 991            else:
 992                file_mode = 0o100644
 993            with open(abs_path, "rb") as f:
 994                file_hash.update(f.read())
 995            self_hash.update(file_mode.to_bytes(2, byteorder="big"))
 996            self_hash.update(rel_path.encode())
 997            self_hash.update(file_hash.digest())
 998            self_hash.update(b"\0")
 999
1000        for pre_image in self.image.pre_images:
1001            self_hash.update(pre_image.extra().encode())
1002            self_hash.update(b"\0")
1003
1004        self_hash.update(f"arch={self.image.rd.arch}".encode())
1005        self_hash.update(f"coverage={self.image.rd.coverage}".encode())
1006        self_hash.update(f"sanitizer={self.image.rd.sanitizer}".encode())
1007
1008        full_hash = hashlib.sha1()
1009        full_hash.update(self_hash.digest())
1010        for dep in sorted(self.dependencies.values(), key=lambda d: d.name):
1011            full_hash.update(dep.name.encode())
1012            full_hash.update(dep.fingerprint())
1013            full_hash.update(b"\0")
1014
1015        return Fingerprint(full_hash.digest())
1016
1017
1018class DependencySet:
1019    """A set of `ResolvedImage`s.
1020
1021    Iterating over a dependency set yields the contained images in an arbitrary
1022    order. Indexing a dependency set yields the image with the specified name.
1023    """
1024
1025    def __init__(self, dependencies: Iterable[Image]):
1026        """Construct a new `DependencySet`.
1027
1028        The provided `dependencies` must be topologically sorted.
1029        """
1030        self._dependencies: dict[str, ResolvedImage] = {}
1031        known_images = docker_images()
1032        for d in dependencies:
1033            image = ResolvedImage(
1034                image=d,
1035                dependencies=(self._dependencies[d0] for d0 in d.depends_on),
1036            )
1037            image.acquired = image.spec() in known_images
1038            self._dependencies[d.name] = image
1039
1040    def _prepare_batch(self, images: list[ResolvedImage]) -> dict[type[PreImage], Any]:
1041        pre_images = collections.defaultdict(list)
1042        for image in images:
1043            for pre_image in image.image.pre_images:
1044                pre_images[type(pre_image)].append(pre_image)
1045        pre_image_prep = {}
1046        for cls, instances in pre_images.items():
1047            pre_image = cast(PreImage, cls)
1048            pre_image_prep[cls] = pre_image.prepare_batch(instances)
1049        return pre_image_prep
1050
1051    def acquire(self, max_retries: int | None = None) -> None:
1052        """Download or build all of the images in the dependency set that do not
1053        already exist locally.
1054
1055        Args:
1056            max_retries: Number of retries on failure.
1057        """
1058
1059        # Only retry in CI runs since we struggle with flaky docker pulls there
1060        if not max_retries:
1061            max_retries = 5 if ui.env_is_truthy("CI") else 1
1062        assert max_retries > 0
1063
1064        deps_to_build = [
1065            dep for dep in self if not dep.publish or not dep.try_pull(max_retries)
1066        ]
1067
1068        # Don't attempt to build in CI, as our timeouts and small machines won't allow it anyway
1069        if ui.env_is_truthy("CI"):
1070            expected_deps = [dep for dep in deps_to_build if dep.publish]
1071            if expected_deps:
1072                print(
1073                    f"+++ Expected builds to be available, the build probably failed, so not proceeding: {expected_deps}"
1074                )
1075                sys.exit(5)
1076
1077        prep = self._prepare_batch(deps_to_build)
1078        for dep in deps_to_build:
1079            dep.build(prep)
1080
1081    def ensure(self, post_build: Callable[[ResolvedImage], None] | None = None):
1082        """Ensure all publishable images in this dependency set exist on Docker
1083        Hub.
1084
1085        Images are pushed using their spec as their tag.
1086
1087        Args:
1088            post_build: A callback to invoke with each dependency that was built
1089                locally.
1090        """
1091        deps_to_build = [dep for dep in self if not dep.is_published_if_necessary()]
1092        prep = self._prepare_batch(deps_to_build)
1093
1094        images_to_push = []
1095        for dep in deps_to_build:
1096            dep.build(prep)
1097            if post_build:
1098                post_build(dep)
1099            if dep.publish:
1100                images_to_push.append(dep.spec())
1101
1102        # Push all Docker images in parallel to minimize build time.
1103        ui.section("Pushing images")
1104        # Attempt to upload images a maximum of 3 times before giving up.
1105        for attempts_remaining in reversed(range(3)):
1106            pushes: list[subprocess.Popen] = []
1107            for image in images_to_push:
1108                # Piping through `cat` disables terminal control codes, and so the
1109                # interleaved progress output from multiple pushes is less hectic.
1110                # We don't use `docker push --quiet`, as that disables progress
1111                # output entirely. Use `set -o pipefail` so the return code of
1112                # `docker push` is passed through.
1113                push = subprocess.Popen(
1114                    [
1115                        "/bin/bash",
1116                        "-c",
1117                        f"set -o pipefail; docker push {shlex.quote(image)} | cat",
1118                    ]
1119                )
1120                pushes.append(push)
1121
1122            for i, push in reversed(list(enumerate(pushes))):
1123                returncode = push.wait()
1124                if returncode:
1125                    if attempts_remaining == 0:
1126                        # Last attempt, fail
1127                        raise subprocess.CalledProcessError(returncode, push.args)
1128                    else:
1129                        print(f"docker push {push.args} failed: {returncode}")
1130                else:
1131                    del images_to_push[i]
1132
1133            if images_to_push:
1134                time.sleep(10)
1135                print("Retrying in 10 seconds")
1136
1137    def check(self) -> bool:
1138        """Check all publishable images in this dependency set exist on Docker
1139        Hub. Don't try to download or build them."""
1140        num_deps = len(list(self))
1141        if num_deps == 0:
1142            return True
1143        with ThreadPoolExecutor(max_workers=num_deps) as executor:
1144            results = list(
1145                executor.map(lambda dep: dep.is_published_if_necessary(), list(self))
1146            )
1147        return all(results)
1148
1149    def __iter__(self) -> Iterator[ResolvedImage]:
1150        return iter(self._dependencies.values())
1151
1152    def __getitem__(self, key: str) -> ResolvedImage:
1153        return self._dependencies[key]
1154
1155
1156class Repository:
1157    """A collection of mzbuild `Image`s.
1158
1159    Creating a repository will walk the filesystem beneath `root` to
1160    automatically discover all contained `Image`s.
1161
1162    Iterating over a repository yields the contained images in an arbitrary
1163    order.
1164
1165    Args:
1166        root: The path to the root of the repository.
1167        arch: The CPU architecture to build for.
1168        profile: What profile to build the repository in.
1169        coverage: Whether to enable code coverage instrumentation.
1170        sanitizer: Whether to a sanitizer (address, thread, leak, memory, none)
1171        image_registry: The Docker image registry to pull images from and push
1172            images to.
1173        image_prefix: A prefix to apply to all Docker image names.
1174
1175    Attributes:
1176        images: A mapping from image name to `Image` for all contained images.
1177        compose_dirs: The set of directories containing a `mzcompose.py` file.
1178    """
1179
1180    def __init__(
1181        self,
1182        root: Path,
1183        arch: Arch = Arch.host(),
1184        profile: Profile = Profile.RELEASE,
1185        coverage: bool = False,
1186        sanitizer: Sanitizer = Sanitizer.none,
1187        image_registry: str = "materialize",
1188        image_prefix: str = "",
1189        bazel: bool = False,
1190        bazel_remote_cache: str | None = None,
1191    ):
1192        self.rd = RepositoryDetails(
1193            root,
1194            arch,
1195            profile,
1196            coverage,
1197            sanitizer,
1198            image_registry,
1199            image_prefix,
1200            bazel,
1201            bazel_remote_cache,
1202        )
1203        self.images: dict[str, Image] = {}
1204        self.compositions: dict[str, Path] = {}
1205        for path, dirs, files in os.walk(self.root, topdown=True):
1206            if path == str(root / "misc"):
1207                dirs.remove("python")
1208            # Filter out some particularly massive ignored directories to keep
1209            # things snappy. Not required for correctness.
1210            dirs[:] = set(dirs) - {
1211                ".git",
1212                ".mypy_cache",
1213                "target",
1214                "target-ra",
1215                "target-xcompile",
1216                "mzdata",
1217                "node_modules",
1218                "venv",
1219            }
1220            if "mzbuild.yml" in files:
1221                image = Image(self.rd, Path(path))
1222                if not image.name:
1223                    raise ValueError(f"config at {path} missing name")
1224                if image.name in self.images:
1225                    raise ValueError(f"image {image.name} exists twice")
1226                self.images[image.name] = image
1227            if "mzcompose.py" in files:
1228                name = Path(path).name
1229                if name in self.compositions:
1230                    raise ValueError(f"composition {name} exists twice")
1231                self.compositions[name] = Path(path)
1232
1233        # Validate dependencies.
1234        for image in self.images.values():
1235            for d in image.depends_on:
1236                if d not in self.images:
1237                    raise ValueError(
1238                        f"image {image.name} depends on non-existent image {d}"
1239                    )
1240
1241    @staticmethod
1242    def install_arguments(parser: argparse.ArgumentParser) -> None:
1243        """Install options to configure a repository into an argparse parser.
1244
1245        This function installs the following options:
1246
1247          * The mutually-exclusive `--dev`/`--optimized`/`--release` options to control the
1248            `profile` repository attribute.
1249          * The `--coverage` boolean option to control the `coverage` repository
1250            attribute.
1251
1252        Use `Repository.from_arguments` to construct a repository from the
1253        parsed command-line arguments.
1254        """
1255        build_mode = parser.add_mutually_exclusive_group()
1256        build_mode.add_argument(
1257            "--dev",
1258            action="store_true",
1259            help="build Rust binaries with the dev profile",
1260        )
1261        build_mode.add_argument(
1262            "--release",
1263            action="store_true",
1264            help="build Rust binaries with the release profile (default)",
1265        )
1266        build_mode.add_argument(
1267            "--optimized",
1268            action="store_true",
1269            help="build Rust binaries with the optimized profile (optimizations, no LTO, no debug symbols)",
1270        )
1271        parser.add_argument(
1272            "--coverage",
1273            help="whether to enable code coverage compilation flags",
1274            default=ui.env_is_truthy("CI_COVERAGE_ENABLED"),
1275            action="store_true",
1276        )
1277        parser.add_argument(
1278            "--sanitizer",
1279            help="whether to enable a sanitizer",
1280            default=Sanitizer[os.getenv("CI_SANITIZER", "none")],
1281            type=Sanitizer,
1282            choices=Sanitizer,
1283        )
1284        parser.add_argument(
1285            "--arch",
1286            default=Arch.host(),
1287            help="the CPU architecture to build for",
1288            type=Arch,
1289            choices=Arch,
1290        )
1291        parser.add_argument(
1292            "--image-registry",
1293            default="materialize",
1294            help="the Docker image registry to pull images from and push images to",
1295        )
1296        parser.add_argument(
1297            "--image-prefix",
1298            default="",
1299            help="a prefix to apply to all Docker image names",
1300        )
1301        parser.add_argument(
1302            "--bazel",
1303            default=ui.env_is_truthy("CI_BAZEL_BUILD"),
1304            action="store_true",
1305        )
1306        parser.add_argument(
1307            "--bazel-remote-cache",
1308            default=os.getenv("CI_BAZEL_REMOTE_CACHE"),
1309            action="store",
1310        )
1311
1312    @classmethod
1313    def from_arguments(cls, root: Path, args: argparse.Namespace) -> "Repository":
1314        """Construct a repository from command-line arguments.
1315
1316        The provided namespace must contain the options installed by
1317        `Repository.install_arguments`.
1318        """
1319        if args.release:
1320            profile = Profile.RELEASE
1321        elif args.optimized:
1322            profile = Profile.OPTIMIZED
1323        elif args.dev:
1324            profile = Profile.DEV
1325        else:
1326            profile = Profile.RELEASE
1327
1328        return cls(
1329            root,
1330            profile=profile,
1331            coverage=args.coverage,
1332            sanitizer=args.sanitizer,
1333            image_registry=args.image_registry,
1334            image_prefix=args.image_prefix,
1335            arch=args.arch,
1336            bazel=args.bazel,
1337            bazel_remote_cache=args.bazel_remote_cache,
1338        )
1339
1340    @property
1341    def root(self) -> Path:
1342        """The path to the root directory for the repository."""
1343        return self.rd.root
1344
1345    def resolve_dependencies(self, targets: Iterable[Image]) -> DependencySet:
1346        """Compute the dependency set necessary to build target images.
1347
1348        The dependencies of `targets` will be crawled recursively until the
1349        complete set of transitive dependencies is determined or a circular
1350        dependency is discovered. The returned dependency set will be sorted
1351        in topological order.
1352
1353        Raises:
1354           ValueError: A circular dependency was discovered in the images
1355               in the repository.
1356        """
1357        resolved = OrderedDict()
1358        visiting = set()
1359
1360        def visit(image: Image, path: list[str] = []) -> None:
1361            if image.name in resolved:
1362                return
1363            if image.name in visiting:
1364                diagram = " -> ".join(path + [image.name])
1365                raise ValueError(f"circular dependency in mzbuild: {diagram}")
1366
1367            visiting.add(image.name)
1368            for d in sorted(image.depends_on):
1369                visit(self.images[d], path + [image.name])
1370            resolved[image.name] = image
1371
1372        for target_image in sorted(targets, key=lambda image: image.name):
1373            visit(target_image)
1374
1375        return DependencySet(resolved.values())
1376
1377    def __iter__(self) -> Iterator[Image]:
1378        return iter(self.images.values())
1379
1380
1381def publish_multiarch_images(
1382    tag: str, dependency_sets: Iterable[Iterable[ResolvedImage]]
1383) -> None:
1384    """Publishes a set of docker images under a given tag."""
1385    for images in zip(*dependency_sets):
1386        names = set(image.image.name for image in images)
1387        assert len(names) == 1, "dependency sets did not contain identical images"
1388        name = images[0].image.docker_name(tag)
1389        spawn.runv(
1390            ["docker", "manifest", "create", name, *(image.spec() for image in images)]
1391        )
1392        spawn.runv(["docker", "manifest", "push", name])
1393    print(f"--- Nofifying for tag {tag}")
1394    markdown = f"""Pushed images with Docker tag `{tag}`"""
1395    spawn.runv(
1396        [
1397            "buildkite-agent",
1398            "annotate",
1399            "--style=info",
1400            f"--context=build-tags-{tag}",
1401        ],
1402        stdin=markdown.encode(),
1403    )