misc.python.materialize.ci_util

Utility functions only useful in CI.

View Source

 1# Copyright Materialize, Inc. and contributors. All rights reserved.
 2#
 3# Use of this software is governed by the Business Source License
 4# included in the LICENSE file at the root of this repository.
 5#
 6# As of the Change Date specified in that file, in accordance with
 7# the Business Source License, use of this software will be governed
 8# by the Apache License, Version 2.0.
 9
10"""Utility functions only useful in CI."""
11
12import os
13import time
14from pathlib import Path
15from typing import Any
16
17import requests
18from semver.version import VersionInfo
19
20from materialize import MZ_ROOT, buildkite, cargo, ui
21
22
23def junit_report_filename(suite: str) -> Path:
24    """Compute the JUnit report filename for the specified test suite.
25
26    See also `upload_test_report`. In CI, the filename will include the
27    Buildkite job ID.
28
29    Args:
30        suite: The identifier for the test suite in Buildkite Test Analytics.
31    """
32    filename = f"junit_{suite}"
33    if "BUILDKITE_JOB_ID" in os.environ:
34        filename += "_" + os.environ["BUILDKITE_JOB_ID"]
35    return Path(f"{filename}.xml")
36
37
38def get_artifacts() -> Any:
39    """Get artifact informations from Buildkite. Outside of CI, this function does nothing."""
40
41    if not buildkite.is_in_buildkite():
42        return []
43
44    ui.section("Getting artifact informations from Buildkite")
45    build = os.environ["BUILDKITE_BUILD_NUMBER"]
46    build_id = os.environ["BUILDKITE_BUILD_ID"]
47    job = os.environ["BUILDKITE_JOB_ID"]
48    token = os.environ["BUILDKITE_AGENT_ACCESS_TOKEN"]
49
50    payload = {
51        "query": "*",
52        "step": job,
53        "build": build,
54        "state": "finished",
55        "includeRetriedJobs": "false",
56        "includeDuplicates": "false",
57    }
58
59    attempts = 10
60    res = None
61    for attempt in range(attempts):
62        try:
63            res = requests.get(
64                f"https://agent.buildkite.com/v3/builds/{build_id}/artifacts/search",
65                params=payload,
66                headers={"Authorization": f"Token {token}"},
67            )
68            res.raise_for_status()
69            break
70        except:
71            if attempt == attempts - 1:
72                raise
73            time.sleep(5)
74
75    assert res
76    if res.status_code != 200:
77        print(f"Failed to get artifacts: {res.status_code} {res.text}")
78        return []
79
80    return res.json()
81
82
83def get_mz_version(workspace: cargo.Workspace | None = None) -> VersionInfo:
84    """Get the current Materialize version from Cargo.toml."""
85
86    if not workspace:
87        workspace = cargo.Workspace(MZ_ROOT)
88    return VersionInfo.parse(workspace.crates["mz-environmentd"].version_string)

def junit_report_filename(suite: str) -> pathlib._local.Path: View Source

24def junit_report_filename(suite: str) -> Path:
25    """Compute the JUnit report filename for the specified test suite.
26
27    See also `upload_test_report`. In CI, the filename will include the
28    Buildkite job ID.
29
30    Args:
31        suite: The identifier for the test suite in Buildkite Test Analytics.
32    """
33    filename = f"junit_{suite}"
34    if "BUILDKITE_JOB_ID" in os.environ:
35        filename += "_" + os.environ["BUILDKITE_JOB_ID"]
36    return Path(f"{filename}.xml")

Compute the JUnit report filename for the specified test suite.

See also upload_test_report. In CI, the filename will include the Buildkite job ID.

Args: suite: The identifier for the test suite in Buildkite Test Analytics.

def get_artifacts() -> Any: View Source

39def get_artifacts() -> Any:
40    """Get artifact informations from Buildkite. Outside of CI, this function does nothing."""
41
42    if not buildkite.is_in_buildkite():
43        return []
44
45    ui.section("Getting artifact informations from Buildkite")
46    build = os.environ["BUILDKITE_BUILD_NUMBER"]
47    build_id = os.environ["BUILDKITE_BUILD_ID"]
48    job = os.environ["BUILDKITE_JOB_ID"]
49    token = os.environ["BUILDKITE_AGENT_ACCESS_TOKEN"]
50
51    payload = {
52        "query": "*",
53        "step": job,
54        "build": build,
55        "state": "finished",
56        "includeRetriedJobs": "false",
57        "includeDuplicates": "false",
58    }
59
60    attempts = 10
61    res = None
62    for attempt in range(attempts):
63        try:
64            res = requests.get(
65                f"https://agent.buildkite.com/v3/builds/{build_id}/artifacts/search",
66                params=payload,
67                headers={"Authorization": f"Token {token}"},
68            )
69            res.raise_for_status()
70            break
71        except:
72            if attempt == attempts - 1:
73                raise
74            time.sleep(5)
75
76    assert res
77    if res.status_code != 200:
78        print(f"Failed to get artifacts: {res.status_code} {res.text}")
79        return []
80
81    return res.json()

Get artifact informations from Buildkite. Outside of CI, this function does nothing.

def get_mz_version( workspace: materialize.cargo.Workspace | None = None) -> semver.version.Version: View Source

84def get_mz_version(workspace: cargo.Workspace | None = None) -> VersionInfo:
85    """Get the current Materialize version from Cargo.toml."""
86
87    if not workspace:
88        workspace = cargo.Workspace(MZ_ROOT)
89    return VersionInfo.parse(workspace.crates["mz-environmentd"].version_string)

Get the current Materialize version from Cargo.toml.