misc.python.materialize.ci_util

  1# Copyright Materialize, Inc. and contributors. All rights reserved.
  2#
  3# Use of this software is governed by the Business Source License
  4# included in the LICENSE file at the root of this repository.
  5#
  6# As of the Change Date specified in that file, in accordance with
  7# the Business Source License, use of this software will be governed
  8# by the Apache License, Version 2.0.
  9
 10"""Utility functions only useful in CI."""
 11
 12import os
 13import time
 14from pathlib import Path
 15from typing import Any
 16
 17import requests
 18from semver.version import VersionInfo
 19
 20from materialize import MZ_ROOT, buildkite, cargo, git, ui
 21from materialize.rustc_flags import Sanitizer
 22
 23
 24def junit_report_filename(suite: str) -> Path:
 25    """Compute the JUnit report filename for the specified test suite.
 26
 27    See also `upload_test_report`. In CI, the filename will include the
 28    Buildkite job ID.
 29
 30    Args:
 31        suite: The identifier for the test suite in Buildkite Test Analytics.
 32    """
 33    filename = f"junit_{suite}"
 34    if "BUILDKITE_JOB_ID" in os.environ:
 35        filename += "_" + os.environ["BUILDKITE_JOB_ID"]
 36    return Path(f"{filename}.xml")
 37
 38
 39def get_artifacts() -> Any:
 40    """Get artifact informations from Buildkite. Outside of CI, this function does nothing."""
 41
 42    if not buildkite.is_in_buildkite():
 43        return []
 44
 45    ui.section("Getting artifact informations from Buildkite")
 46    build = os.environ["BUILDKITE_BUILD_NUMBER"]
 47    build_id = os.environ["BUILDKITE_BUILD_ID"]
 48    job = os.environ["BUILDKITE_JOB_ID"]
 49    token = os.environ["BUILDKITE_AGENT_ACCESS_TOKEN"]
 50
 51    payload = {
 52        "query": "*",
 53        "step": job,
 54        "build": build,
 55        "state": "finished",
 56        "includeRetriedJobs": "false",
 57        "includeDuplicates": "false",
 58    }
 59
 60    attempts = 10
 61    res = None
 62    for attempt in range(attempts):
 63        try:
 64            res = requests.get(
 65                f"https://agent.buildkite.com/v3/builds/{build_id}/artifacts/search",
 66                params=payload,
 67                headers={"Authorization": f"Token {token}"},
 68                timeout=30,
 69            )
 70            res.raise_for_status()
 71            break
 72        except Exception as e:
 73            # Artifact info only supplies links for annotations, so on repeated
 74            # failure degrade to no artifacts rather than crashing the caller
 75            # and losing all annotations. A bare except would also swallow
 76            # KeyboardInterrupt.
 77            print(f"Failed to get artifacts (attempt {attempt + 1}/{attempts}): {e}")
 78            if attempt == attempts - 1:
 79                return []
 80            time.sleep(5)
 81
 82    assert res
 83    if res.status_code != 200:
 84        print(f"Failed to get artifacts: {res.status_code} {res.text}")
 85        return []
 86
 87    return res.json()
 88
 89
 90def get_mz_version(workspace: cargo.Workspace | None = None) -> VersionInfo:
 91    """Get the current Materialize version from Cargo.toml."""
 92
 93    if not workspace:
 94        workspace = cargo.Workspace(MZ_ROOT)
 95    return VersionInfo.parse(workspace.crates["mz-environmentd"].version_string)
 96
 97
 98def dev_docker_tag() -> str:
 99    """The Docker tag under which this commit's images are published.
100
101    Both the publishing side (`ci/test/dev_tag.py`) and anything that pulls
102    those images have to derive the tag the same way, or the pull looks for an
103    image that was never pushed.
104    """
105    # Ideally we'd use SemVer metadata (e.g., `v1.0.0+metadata`), but `+` is not
106    # a valid character in Docker tags, so we use `--` instead.
107    sanitizer = Sanitizer[os.getenv("CI_SANITIZER", "none")]
108    suffix = "pr" if sanitizer == Sanitizer.none else f"pr-{sanitizer}"
109    return f"v{get_mz_version()}--{suffix}.g{git.rev_parse('HEAD')}"