misc.python.materialize.git

  1# Copyright Materialize, Inc. and contributors. All rights reserved.
  2#
  3# Use of this software is governed by the Business Source License
  4# included in the LICENSE file at the root of this repository.
  5#
  6# As of the Change Date specified in that file, in accordance with
  7# the Business Source License, use of this software will be governed
  8# by the Apache License, Version 2.0.
  9
 10"""Git utilities."""
 11
 12import functools
 13import os
 14import subprocess
 15import sys
 16from pathlib import Path
 17from typing import TypeVar
 18
 19import requests
 20
 21from materialize import spawn
 22from materialize.mz_version import MzVersion, TypedVersionBase
 23from materialize.util import YesNoOnce
 24
 25VERSION_TYPE = TypeVar("VERSION_TYPE", bound=TypedVersionBase)
 26
 27MATERIALIZE_REMOTE_URL = "https://github.com/MaterializeInc/materialize"
 28
 29fetched_tags_in_remotes: set[str | None] = set()
 30
 31
 32def rev_count(rev: str) -> int:
 33    """Count the commits up to a revision.
 34
 35    Args:
 36        rev: A Git revision in any format know to the Git CLI.
 37
 38    Returns:
 39        count: The number of commits in the Git repository starting from the
 40            initial commit and ending with the specified commit, inclusive.
 41    """
 42    return int(spawn.capture(["git", "rev-list", "--count", rev, "--"]).strip())
 43
 44
 45def rev_parse(rev: str, *, abbrev: bool = False) -> str:
 46    """Compute the hash for a revision.
 47
 48    Args:
 49        rev: A Git revision in any format known to the Git CLI.
 50        abbrev: Return a branch or tag name instead of a git sha
 51
 52    Returns:
 53        ref: A 40 character hex-encoded SHA-1 hash representing the ID of the
 54            named revision in Git's object database.
 55
 56            With "abbrev=True" this will return an abbreviated ref, or throw an
 57            error if there is no abbrev.
 58    """
 59    a = ["--abbrev-ref"] if abbrev else []
 60    out = spawn.capture(["git", "rev-parse", *a, "--verify", rev]).strip()
 61    if not out:
 62        raise RuntimeError(f"No parsed rev for {rev}")
 63    return out
 64
 65
 66@functools.cache
 67def expand_globs(root: Path, *specs: Path | str) -> set[str]:
 68    """Find unignored files within the specified paths."""
 69    # The goal here is to find all files in the working tree that are not
 70    # ignored by .gitignore. Naively using `git ls-files` doesn't work, because
 71    # it reports files that have been deleted in the working tree if they are
 72    # still present in the index. Using `os.walkdir` doesn't work because there
 73    # is no good way to evaluate .gitignore rules from Python. So we use a
 74    # combination of `git diff` and `git ls-files`.
 75
 76    # `git diff` against the empty tree surfaces all tracked files that have
 77    # not been deleted.
 78    empty_tree = (
 79        "4b825dc642cb6eb9a060e54bf8d69288fbee4904"  # git hash-object -t tree /dev/null
 80    )
 81    diff_files = spawn.capture(
 82        ["git", "diff", "--name-only", "-z", "--relative", empty_tree, "--", *specs],
 83        cwd=root,
 84    )
 85
 86    # `git ls-files --others --exclude-standard` surfaces any non-ignored,
 87    # untracked files, which are not included in the `git diff` output above.
 88    ls_files = spawn.capture(
 89        ["git", "ls-files", "--others", "--exclude-standard", "-z", "--", *specs],
 90        cwd=root,
 91    )
 92
 93    return set(f for f in (diff_files + ls_files).split("\0") if f.strip() != "")
 94
 95
 96def get_version_tags(
 97    *,
 98    version_type: type[VERSION_TYPE],
 99    newest_first: bool = True,
100    fetch: bool = True,
101    remote_url: str = MATERIALIZE_REMOTE_URL,
102) -> list[VERSION_TYPE]:
103    """List all the version-like tags in the repo
104
105    Args:
106        fetch: If false, don't automatically run `git fetch --tags`.
107        prefix: A prefix to strip from each tag before attempting to parse the
108            tag as a version.
109    """
110    if fetch:
111        _fetch(
112            remote=get_remote(remote_url),
113            include_tags=YesNoOnce.ONCE,
114            force=True,
115            only_tags=True,
116        )
117    tags = []
118    for t in spawn.capture(["git", "tag"]).splitlines():
119        if not t.startswith(version_type.get_prefix()):
120            continue
121        try:
122            tags.append(version_type.parse(t))
123        except ValueError as e:
124            print(f"WARN: {e}", file=sys.stderr)
125
126    return sorted(tags, reverse=newest_first)
127
128
129def get_latest_version(
130    version_type: type[VERSION_TYPE],
131    excluded_versions: set[VERSION_TYPE] | None = None,
132    current_version: VERSION_TYPE | None = None,
133) -> VERSION_TYPE:
134    all_version_tags: list[VERSION_TYPE] = get_version_tags(
135        version_type=version_type, fetch=True
136    )
137
138    if excluded_versions is not None:
139        all_version_tags = [
140            v
141            for v in all_version_tags
142            if v not in excluded_versions
143            and (not current_version or v < current_version)
144        ]
145
146    return max(all_version_tags)
147
148
149def get_tags_of_current_commit(include_tags: YesNoOnce = YesNoOnce.ONCE) -> list[str]:
150    if include_tags:
151        fetch(get_remote(), include_tags=include_tags, only_tags=True)
152
153    result = spawn.capture(["git", "tag", "--points-at", "HEAD"])
154
155    if len(result) == 0:
156        return []
157
158    return result.splitlines()
159
160
161def is_ancestor(earlier: str, later: str) -> bool:
162    """True if earlier is in an ancestor of later"""
163    try:
164        headers = {"Accept": "application/vnd.github+json"}
165        if token := os.getenv("GITHUB_TOKEN"):
166            headers["Authorization"] = f"Bearer {token}"
167
168        resp = requests.get(
169            f"https://api.github.com/repos/materializeinc/materialize/compare/{earlier}...{later}",
170            headers=headers,
171        )
172        resp.raise_for_status()
173        data = resp.json()
174        return data.get("status") in ("ahead", "identical")
175    except Exception as e:
176        # Try locally if Github is down or the change has not been pushed yet when running locally
177        print(f"Failed to get ancestor status from Github, running locally: {e}")
178
179        # Make sure we have an up to date view of main.
180        command = ["git", "fetch"]
181        if spawn.capture(["git", "rev-parse", "--is-shallow-repository"]) == "true":
182            command.append("--unshallow")
183        spawn.runv(command + [get_remote(), earlier, later])
184
185        return (
186            spawn.run_and_get_return_code(
187                ["git", "merge-base", "--is-ancestor", earlier, later]
188            )
189            == 0
190        )
191
192
193def is_dirty() -> bool:
194    """Check if the working directory has modifications to tracked files"""
195    proc = subprocess.run("git diff --no-ext-diff --quiet --exit-code".split())
196    idx = subprocess.run("git diff --cached --no-ext-diff --quiet --exit-code".split())
197    return proc.returncode != 0 or idx.returncode != 0
198
199
200def first_remote_matching(pattern: str) -> str | None:
201    """Get the name of the remote that matches the pattern"""
202    remotes = spawn.capture(["git", "remote", "-v"])
203    for remote in remotes.splitlines():
204        if pattern in remote:
205            return remote.split()[0]
206
207    return None
208
209
210def describe() -> str:
211    """Describe the relationship between the current commit and the most recent tag"""
212    return spawn.capture(["git", "describe"]).strip()
213
214
215def fetch(
216    remote: str | None = None,
217    all_remotes: bool = False,
218    include_tags: YesNoOnce = YesNoOnce.NO,
219    force: bool = False,
220    branch: str | None = None,
221    only_tags: bool = False,
222    include_submodules: bool = False,
223) -> str:
224    """Fetch from remotes"""
225
226    if remote is not None and all_remotes:
227        raise RuntimeError("all_remotes must be false when a remote is specified")
228
229    if branch is not None and remote is None:
230        raise RuntimeError("remote must be specified when a branch is specified")
231
232    if branch is not None and only_tags:
233        raise RuntimeError("branch must not be specified if only_tags is set")
234
235    command = ["git", "fetch"]
236    if spawn.capture(["git", "rev-parse", "--is-shallow-repository"]) == "true":
237        command.append("--unshallow")
238
239    if remote:
240        command.append(remote)
241
242    if branch:
243        command.append(branch)
244
245    if all_remotes:
246        command.append("--all")
247
248    # explicitly specify both cases to be independent of the git config
249    if include_submodules:
250        command.append("--recurse-submodules")
251    else:
252        command.append("--no-recurse-submodules")
253
254    fetch_tags = (
255        include_tags == YesNoOnce.YES
256        # fetch tags again if used with force (tags might have changed)
257        or (include_tags == YesNoOnce.ONCE and force)
258        or (
259            include_tags == YesNoOnce.ONCE
260            and remote not in fetched_tags_in_remotes
261            and "*" not in fetched_tags_in_remotes
262        )
263    )
264
265    if fetch_tags:
266        command.append("--tags")
267
268    if force:
269        command.append("--force")
270
271    if not fetch_tags and only_tags:
272        return ""
273
274    output = spawn.capture(command).strip()
275
276    if fetch_tags:
277        fetched_tags_in_remotes.add(remote)
278
279        if all_remotes:
280            fetched_tags_in_remotes.add("*")
281
282    return output
283
284
285_fetch = fetch  # renamed because an argument shadows the fetch name in get_tags
286
287
288def try_get_remote_name_by_url(url: str) -> str | None:
289    result = spawn.capture(["git", "remote", "--verbose"])
290    for line in result.splitlines():
291        remote, desc = line.split("\t")
292        if desc.lower() in (f"{url} (fetch)".lower(), f"{url}.git (fetch)".lower()):
293            return remote
294    return None
295
296
297def get_remote(
298    url: str = MATERIALIZE_REMOTE_URL,
299    default_remote_name: str = "origin",
300) -> str:
301    # Alternative syntax
302    remote = try_get_remote_name_by_url(url) or try_get_remote_name_by_url(
303        url.replace("https://github.com/", "git@github.com:")
304    )
305    if not remote:
306        remote = default_remote_name
307        print(f"Remote for URL {url} not found, using {remote}")
308
309    return remote
310
311
312def get_common_ancestor_commit(remote: str, branch: str) -> str:
313    try:
314        head = spawn.capture(["git", "rev-parse", "HEAD"]).strip()
315        headers = {"Accept": "application/vnd.github+json"}
316        if token := os.getenv("GITHUB_TOKEN"):
317            headers["Authorization"] = f"Bearer {token}"
318
319        resp = requests.get(
320            f"https://api.github.com/repos/materializeinc/materialize/compare/{head}...{branch}",
321            headers=headers,
322        )
323        resp.raise_for_status()
324        data = resp.json()
325        return data["merge_base_commit"]["sha"]
326    except Exception as e:
327        # Try locally if Github is down or the change has not been pushed yet when running locally
328        print(f"Failed to get ancestor commit from Github, running locally: {e}")
329
330        # Make sure we have an up to date view
331        command = ["git", "fetch"]
332        if spawn.capture(["git", "rev-parse", "--is-shallow-repository"]) == "true":
333            command.append("--unshallow")
334        spawn.runv(command + [remote, branch])
335
336        return spawn.capture(
337            ["git", "merge-base", "HEAD", f"{remote}/{branch}"]
338        ).strip()
339
340
341def is_on_release_version() -> bool:
342    git_tags = get_tags_of_current_commit()
343    return any(MzVersion.is_valid_version_string(git_tag) for git_tag in git_tags)
344
345
346def contains_commit(
347    commit_sha: str,
348    target: str = "HEAD",
349    remote_url: str = MATERIALIZE_REMOTE_URL,
350) -> bool:
351    return is_ancestor(commit_sha, target)
352
353
354def get_tagged_release_version(version_type: type[VERSION_TYPE]) -> VERSION_TYPE | None:
355    """
356    This returns the release version if exactly this commit is tagged.
357    If multiple release versions are present, the highest one will be returned.
358    None will be returned if the commit is not tagged.
359    """
360    git_tags = get_tags_of_current_commit()
361
362    versions: list[VERSION_TYPE] = []
363
364    for git_tag in git_tags:
365        if version_type.is_valid_version_string(git_tag):
366            versions.append(version_type.parse(git_tag))
367
368    if len(versions) == 0:
369        return None
370
371    if len(versions) > 1:
372        print(
373            "Warning! Commit is tagged with multiple release versions! Returning the highest."
374        )
375
376    return max(versions)
377
378
379def get_commit_message(commit_sha: str) -> str | None:
380    try:
381        command = ["git", "log", "-1", "--pretty=format:%s", commit_sha]
382        return spawn.capture(command, stderr=subprocess.DEVNULL).strip()
383    except subprocess.CalledProcessError:
384        # Sometimes mz_version() will report a Git SHA that is not available
385        # in the current repository
386        return None
387
388
389def get_branch_name() -> str:
390    """This may not work on Buildkite; consider using the same function from build_context."""
391    command = ["git", "branch", "--show-current"]
392    return spawn.capture(command).strip()
393
394
395# Work tree mutation
396
397
398def create_branch(name: str) -> None:
399    spawn.runv(["git", "checkout", "-b", name])
400
401
402def checkout(rev: str, path: str | None = None) -> None:
403    """Git checkout the rev"""
404    cmd = ["git", "checkout", rev]
405    if path:
406        cmd.extend(["--", path])
407    spawn.runv(cmd)
408
409
410def add_file(file: str) -> None:
411    """Git add a file"""
412    spawn.runv(["git", "add", file])
413
414
415def commit_all_changed(message: str) -> None:
416    """Commit all changed files with the given message"""
417    spawn.runv(["git", "commit", "-a", "-m", message])
418
419
420def tag_annotated(tag: str) -> None:
421    """Create an annotated tag on HEAD"""
422    spawn.runv(["git", "tag", "-a", "-m", tag, tag])