misc.python.materialize.git

  1# Copyright Materialize, Inc. and contributors. All rights reserved.
  2#
  3# Use of this software is governed by the Business Source License
  4# included in the LICENSE file at the root of this repository.
  5#
  6# As of the Change Date specified in that file, in accordance with
  7# the Business Source License, use of this software will be governed
  8# by the Apache License, Version 2.0.
  9
 10"""Git utilities."""
 11
 12import functools
 13import os
 14import subprocess
 15import sys
 16from pathlib import Path
 17from typing import TypeVar
 18
 19import requests
 20
 21from materialize import spawn
 22from materialize.mz_version import MzVersion, TypedVersionBase
 23from materialize.util import YesNoOnce
 24
 25VERSION_TYPE = TypeVar("VERSION_TYPE", bound=TypedVersionBase)
 26
 27MATERIALIZE_REMOTE_URL = "https://github.com/MaterializeInc/materialize"
 28
 29fetched_tags_in_remotes: set[str | None] = set()
 30
 31
 32def get_config(key: str) -> str | None:
 33    """Read a git config value, returning None if unset."""
 34    try:
 35        return spawn.capture(["git", "config", key]).strip()
 36    except subprocess.CalledProcessError:
 37        return None
 38
 39
 40def get_user_name() -> str | None:
 41    """Get the configured git user.name."""
 42    return get_config("user.name")
 43
 44
 45def get_user_email() -> str | None:
 46    """Get the configured git user.email."""
 47    return get_config("user.email")
 48
 49
 50def rev_count(rev: str) -> int:
 51    """Count the commits up to a revision.
 52
 53    Args:
 54        rev: A Git revision in any format know to the Git CLI.
 55
 56    Returns:
 57        count: The number of commits in the Git repository starting from the
 58            initial commit and ending with the specified commit, inclusive.
 59    """
 60    return int(spawn.capture(["git", "rev-list", "--count", rev, "--"]).strip())
 61
 62
 63def get_first_parent_commits(rev: str, limit: int) -> list[str]:
 64    """Get commit hashes along the first-parent chain starting from rev.
 65
 66    Returns up to `limit` commit hashes (including rev itself), following
 67    only first parents (i.e., staying on the main branch).
 68    """
 69    return (
 70        spawn.capture(["git", "rev-list", "--first-parent", f"-{limit}", rev])
 71        .strip()
 72        .splitlines()
 73    )
 74
 75
 76def rev_parse(rev: str, *, abbrev: bool = False) -> str:
 77    """Compute the hash for a revision.
 78
 79    Args:
 80        rev: A Git revision in any format known to the Git CLI.
 81        abbrev: Return a branch or tag name instead of a git sha
 82
 83    Returns:
 84        ref: A 40 character hex-encoded SHA-1 hash representing the ID of the
 85            named revision in Git's object database.
 86
 87            With "abbrev=True" this will return an abbreviated ref, or throw an
 88            error if there is no abbrev.
 89    """
 90    a = ["--abbrev-ref"] if abbrev else []
 91    out = spawn.capture(["git", "rev-parse", *a, "--verify", rev]).strip()
 92    if not out:
 93        raise RuntimeError(f"No parsed rev for {rev}")
 94    return out
 95
 96
 97@functools.cache
 98def expand_globs(root: Path, *specs: Path | str) -> set[str]:
 99    """Find unignored files within the specified paths."""
100    # The goal here is to find all files in the working tree that are not
101    # ignored by .gitignore. Naively using `git ls-files` doesn't work, because
102    # it reports files that have been deleted in the working tree if they are
103    # still present in the index. Using `os.walkdir` doesn't work because there
104    # is no good way to evaluate .gitignore rules from Python. So we use a
105    # combination of `git diff` and `git ls-files`.
106
107    # `git diff` against the empty tree surfaces all tracked files that have
108    # not been deleted.
109    empty_tree = (
110        "4b825dc642cb6eb9a060e54bf8d69288fbee4904"  # git hash-object -t tree /dev/null
111    )
112    diff_files = spawn.capture(
113        ["git", "diff", "--name-only", "-z", "--relative", empty_tree, "--", *specs],
114        cwd=root,
115    )
116
117    # `git ls-files --others --exclude-standard` surfaces any non-ignored,
118    # untracked files, which are not included in the `git diff` output above.
119    ls_files = spawn.capture(
120        ["git", "ls-files", "--others", "--exclude-standard", "-z", "--", *specs],
121        cwd=root,
122    )
123
124    return set(f for f in (diff_files + ls_files).split("\0") if f.strip() != "")
125
126
127def get_version_tags(
128    *,
129    version_type: type[VERSION_TYPE],
130    newest_first: bool = True,
131    fetch: bool = True,
132    remote_url: str = MATERIALIZE_REMOTE_URL,
133) -> list[VERSION_TYPE]:
134    """List all the version-like tags in the repo
135
136    Args:
137        fetch: If false, don't automatically run `git fetch --tags`.
138        prefix: A prefix to strip from each tag before attempting to parse the
139            tag as a version.
140    """
141    if fetch:
142        _fetch(
143            remote=get_remote(remote_url),
144            include_tags=YesNoOnce.ONCE,
145            force=True,
146            only_tags=True,
147        )
148    tags = []
149    for t in spawn.capture(["git", "tag"]).splitlines():
150        if not t.startswith(version_type.get_prefix()):
151            continue
152        try:
153            tags.append(version_type.parse(t))
154        except ValueError as e:
155            print(f"WARN: {e}", file=sys.stderr)
156
157    return sorted(tags, reverse=newest_first)
158
159
160def get_latest_version(
161    version_type: type[VERSION_TYPE],
162    excluded_versions: set[VERSION_TYPE] | None = None,
163    current_version: VERSION_TYPE | None = None,
164) -> VERSION_TYPE:
165    all_version_tags: list[VERSION_TYPE] = get_version_tags(
166        version_type=version_type, fetch=True
167    )
168
169    if excluded_versions is not None:
170        all_version_tags = [
171            v
172            for v in all_version_tags
173            if v not in excluded_versions
174            and (not current_version or v < current_version)
175        ]
176
177    return max(all_version_tags)
178
179
180def get_tags_of_current_commit(include_tags: YesNoOnce = YesNoOnce.ONCE) -> list[str]:
181    if include_tags:
182        fetch(get_remote(), include_tags=include_tags, only_tags=True)
183
184    result = spawn.capture(["git", "tag", "--points-at", "HEAD"])
185
186    if len(result) == 0:
187        return []
188
189    return result.splitlines()
190
191
192def is_ancestor(earlier: str, later: str) -> bool:
193    """True if earlier is in an ancestor of later"""
194    try:
195        headers = {"Accept": "application/vnd.github+json"}
196        if token := os.getenv("GITHUB_TOKEN"):
197            headers["Authorization"] = f"Bearer {token}"
198
199        resp = requests.get(
200            f"https://api.github.com/repos/materializeinc/materialize/compare/{earlier}...{later}",
201            headers=headers,
202        )
203        resp.raise_for_status()
204        data = resp.json()
205        return data.get("status") in ("ahead", "identical")
206    except Exception as e:
207        # Try locally if Github is down or the change has not been pushed yet when running locally
208        print(f"Failed to get ancestor status from Github, running locally: {e}")
209
210        # Make sure we have an up to date view of main.
211        command = ["git", "fetch"]
212        if spawn.capture(["git", "rev-parse", "--is-shallow-repository"]) == "true":
213            command.append("--unshallow")
214        spawn.runv(command + [get_remote(), earlier, later])
215
216        return (
217            spawn.run_and_get_return_code(
218                ["git", "merge-base", "--is-ancestor", earlier, later]
219            )
220            == 0
221        )
222
223
224def is_dirty() -> bool:
225    """Check if the working directory has modifications to tracked files"""
226    proc = subprocess.run("git diff --no-ext-diff --quiet --exit-code".split())
227    idx = subprocess.run("git diff --cached --no-ext-diff --quiet --exit-code".split())
228    return proc.returncode != 0 or idx.returncode != 0
229
230
231def describe() -> str:
232    """Describe the relationship between the current commit and the most recent tag"""
233    return spawn.capture(["git", "describe"]).strip()
234
235
236def fetch(
237    remote: str | None = None,
238    all_remotes: bool = False,
239    include_tags: YesNoOnce = YesNoOnce.NO,
240    force: bool = False,
241    branch: str | None = None,
242    only_tags: bool = False,
243) -> str:
244    """Fetch from remotes"""
245
246    if remote is not None and all_remotes:
247        raise RuntimeError("all_remotes must be false when a remote is specified")
248
249    if branch is not None and remote is None:
250        raise RuntimeError("remote must be specified when a branch is specified")
251
252    if branch is not None and only_tags:
253        raise RuntimeError("branch must not be specified if only_tags is set")
254
255    command = ["git", "fetch"]
256    if spawn.capture(["git", "rev-parse", "--is-shallow-repository"]) == "true":
257        command.append("--unshallow")
258
259    if remote:
260        command.append(remote)
261
262    if branch:
263        command.append(branch)
264
265    if all_remotes:
266        command.append("--all")
267
268    fetch_tags = (
269        include_tags == YesNoOnce.YES
270        # fetch tags again if used with force (tags might have changed)
271        or (include_tags == YesNoOnce.ONCE and force)
272        or (
273            include_tags == YesNoOnce.ONCE
274            and remote not in fetched_tags_in_remotes
275            and "*" not in fetched_tags_in_remotes
276        )
277    )
278
279    if fetch_tags:
280        command.append("--tags")
281
282    if force:
283        command.append("--force")
284
285    if not fetch_tags and only_tags:
286        return ""
287
288    output = spawn.capture(command).strip()
289
290    if fetch_tags:
291        fetched_tags_in_remotes.add(remote)
292
293        if all_remotes:
294            fetched_tags_in_remotes.add("*")
295
296    return output
297
298
299_fetch = fetch  # renamed because an argument shadows the fetch name in get_tags
300
301
302def try_get_remote_name_by_url(url: str) -> str | None:
303    result = spawn.capture(["git", "remote", "--verbose"])
304    for line in result.splitlines():
305        remote, desc = line.split("\t")
306        if desc.lower() in (f"{url} (fetch)".lower(), f"{url}.git (fetch)".lower()):
307            return remote
308    return None
309
310
311def get_remote(
312    url: str = MATERIALIZE_REMOTE_URL,
313    default_remote_name: str = "origin",
314) -> str:
315    # Alternative syntax
316    remote = try_get_remote_name_by_url(url) or try_get_remote_name_by_url(
317        url.replace("https://github.com/", "git@github.com:")
318    )
319    if not remote:
320        remote = default_remote_name
321        print(f"Remote for URL {url} not found, using {remote}")
322
323    return remote
324
325
326def get_common_ancestor_commit(remote: str, branch: str) -> str:
327    try:
328        head = spawn.capture(["git", "rev-parse", "HEAD"]).strip()
329        headers = {"Accept": "application/vnd.github+json"}
330        if token := os.getenv("GITHUB_TOKEN"):
331            headers["Authorization"] = f"Bearer {token}"
332
333        resp = requests.get(
334            f"https://api.github.com/repos/materializeinc/materialize/compare/{head}...{branch}",
335            headers=headers,
336        )
337        resp.raise_for_status()
338        data = resp.json()
339        return data["merge_base_commit"]["sha"]
340    except Exception as e:
341        # Try locally if Github is down or the change has not been pushed yet when running locally
342        print(f"Failed to get ancestor commit from Github, running locally: {e}")
343
344        # Make sure we have an up to date view
345        command = ["git", "fetch"]
346        if spawn.capture(["git", "rev-parse", "--is-shallow-repository"]) == "true":
347            command.append("--unshallow")
348        spawn.runv(command + [remote, branch])
349
350        return spawn.capture(
351            ["git", "merge-base", "HEAD", f"{remote}/{branch}"]
352        ).strip()
353
354
355def is_on_release_version() -> bool:
356    git_tags = get_tags_of_current_commit()
357    return any(MzVersion.is_valid_version_string(git_tag) for git_tag in git_tags)
358
359
360def contains_commit(
361    commit_sha: str,
362    target: str = "HEAD",
363    remote_url: str = MATERIALIZE_REMOTE_URL,
364) -> bool:
365    return is_ancestor(commit_sha, target)
366
367
368def get_tagged_release_version(version_type: type[VERSION_TYPE]) -> VERSION_TYPE | None:
369    """
370    This returns the release version if exactly this commit is tagged.
371    If multiple release versions are present, the highest one will be returned.
372    None will be returned if the commit is not tagged.
373    """
374    git_tags = get_tags_of_current_commit()
375
376    versions: list[VERSION_TYPE] = []
377
378    for git_tag in git_tags:
379        if version_type.is_valid_version_string(git_tag):
380            versions.append(version_type.parse(git_tag))
381
382    if len(versions) == 0:
383        return None
384
385    if len(versions) > 1:
386        print(
387            "Warning! Commit is tagged with multiple release versions! Returning the highest."
388        )
389
390    return max(versions)
391
392
393def get_commit_message(commit_sha: str) -> str | None:
394    try:
395        command = ["git", "log", "-1", "--pretty=format:%s", commit_sha]
396        return spawn.capture(command, stderr=subprocess.DEVNULL).strip()
397    except subprocess.CalledProcessError:
398        # Sometimes mz_version() will report a Git SHA that is not available
399        # in the current repository
400        return None
401
402
403def get_branch_name() -> str:
404    """This may not work on Buildkite; consider using the same function from build_context."""
405    command = ["git", "branch", "--show-current"]
406    return spawn.capture(command).strip()
407
408
409# Work tree mutation
410
411
412def create_branch(name: str) -> None:
413    spawn.runv(["git", "checkout", "-b", name])
414
415
416def checkout(rev: str, path: str | None = None) -> None:
417    """Git checkout the rev"""
418    cmd = ["git", "checkout", rev]
419    if path:
420        cmd.extend(["--", path])
421    spawn.runv(cmd)
422
423
424def add_file(file: str) -> None:
425    """Git add a file"""
426    spawn.runv(["git", "add", file])
427
428
429def commit_all_changed(message: str) -> None:
430    """Commit all changed files with the given message"""
431    spawn.runv(["git", "commit", "-a", "-m", message])
432
433
434def tag_annotated(tag: str) -> None:
435    """Create an annotated tag on HEAD"""
436    spawn.runv(["git", "tag", "-a", "-m", tag, tag])