misc.python.materialize.ci_util

  1# Copyright Materialize, Inc. and contributors. All rights reserved.
  2#
  3# Use of this software is governed by the Business Source License
  4# included in the LICENSE file at the root of this repository.
  5#
  6# As of the Change Date specified in that file, in accordance with
  7# the Business Source License, use of this software will be governed
  8# by the Apache License, Version 2.0.
  9
 10"""Utility functions only useful in CI."""
 11
 12import os
 13from pathlib import Path
 14from typing import Any
 15
 16import requests
 17from semver.version import VersionInfo
 18
 19from materialize import MZ_ROOT, buildkite, cargo, ui
 20
 21
 22def junit_report_filename(suite: str) -> Path:
 23    """Compute the JUnit report filename for the specified test suite.
 24
 25    See also `upload_test_report`. In CI, the filename will include the
 26    Buildkite job ID.
 27
 28    Args:
 29        suite: The identifier for the test suite in Buildkite Test Analytics.
 30    """
 31    filename = f"junit_{suite}"
 32    if "BUILDKITE_JOB_ID" in os.environ:
 33        filename += "_" + os.environ["BUILDKITE_JOB_ID"]
 34    return Path(f"{filename}.xml")
 35
 36
 37def upload_junit_report(suite: str, junit_report: Path) -> None:
 38    """Upload a JUnit report to Buildkite Test Analytics.
 39
 40    Outside of CI, this function does nothing. Inside of CI, the API key for
 41    Buildkite Test Analytics is expected to be in the environment variable
 42    `BUILDKITE_TEST_ANALYTICS_API_KEY_{SUITE}`, where `{SUITE}` is the
 43    upper-snake-cased rendition of the `suite` parameter.
 44
 45    Args:
 46        suite: The identifier for the test suite in Buildkite Test Analytics.
 47        junit_report: The path to the JUnit XML-formatted report file.
 48    """
 49    if not buildkite.is_in_buildkite():
 50        return
 51    suite = suite.upper().replace("-", "_")
 52    token = os.getenv(f"BUILDKITE_TEST_ANALYTICS_API_KEY_{suite}")
 53    if not token:
 54        return
 55    try:
 56        res = requests.post(
 57            "https://analytics-api.buildkite.com/v1/uploads",
 58            headers={"Authorization": f"Token {token}"},
 59            json={
 60                "format": "junit",
 61                "run_env": {
 62                    "key": os.environ["BUILDKITE_BUILD_ID"],
 63                    "CI": "buildkite",
 64                    "number": os.environ["BUILDKITE_BUILD_NUMBER"],
 65                    "job_id": os.environ["BUILDKITE_JOB_ID"],
 66                    "branch": os.environ["BUILDKITE_BRANCH"],
 67                    "commit_sha": os.environ["BUILDKITE_COMMIT"],
 68                    "message": os.environ["BUILDKITE_MESSAGE"],
 69                    "url": os.environ["BUILDKITE_BUILD_URL"],
 70                },
 71                "data": junit_report.read_text(),
 72            },
 73        )
 74    except Exception as e:
 75        print(f"Got exception when uploading analytics: {e}")
 76    else:
 77        print(res.status_code, res.text)
 78
 79
 80def get_artifacts() -> Any:
 81    """Get artifact informations from Buildkite. Outside of CI, this function does nothing."""
 82
 83    if not buildkite.is_in_buildkite():
 84        return []
 85
 86    ui.section("Getting artifact informations from Buildkite")
 87    build = os.environ["BUILDKITE_BUILD_NUMBER"]
 88    build_id = os.environ["BUILDKITE_BUILD_ID"]
 89    job = os.environ["BUILDKITE_JOB_ID"]
 90    token = os.environ["BUILDKITE_AGENT_ACCESS_TOKEN"]
 91
 92    payload = {
 93        "query": "*",
 94        "step": job,
 95        "build": build,
 96        "state": "finished",
 97        "includeRetriedJobs": "false",
 98        "includeDuplicates": "false",
 99    }
100
101    res = requests.get(
102        f"https://agent.buildkite.com/v3/builds/{build_id}/artifacts/search",
103        params=payload,
104        headers={"Authorization": f"Token {token}"},
105    )
106
107    if res.status_code != 200:
108        print(f"Failed to get artifacts: {res.status_code} {res.text}")
109        return []
110
111    return res.json()
112
113
114def get_mz_version(workspace: cargo.Workspace | None = None) -> VersionInfo:
115    """Get the current Materialize version from Cargo.toml."""
116
117    if not workspace:
118        workspace = cargo.Workspace(MZ_ROOT)
119    return VersionInfo.parse(workspace.crates["mz-environmentd"].version_string)