misc.python.materialize.git

  1# Copyright Materialize, Inc. and contributors. All rights reserved.
  2#
  3# Use of this software is governed by the Business Source License
  4# included in the LICENSE file at the root of this repository.
  5#
  6# As of the Change Date specified in that file, in accordance with
  7# the Business Source License, use of this software will be governed
  8# by the Apache License, Version 2.0.
  9
 10"""Git utilities."""
 11
 12import functools
 13import os
 14import subprocess
 15import sys
 16from pathlib import Path
 17from typing import TypeVar
 18
 19import requests
 20
 21from materialize import spawn
 22from materialize.mz_version import MzVersion, TypedVersionBase
 23from materialize.util import YesNoOnce
 24
 25VERSION_TYPE = TypeVar("VERSION_TYPE", bound=TypedVersionBase)
 26
 27MATERIALIZE_REMOTE_URL = "https://github.com/MaterializeInc/materialize"
 28
 29fetched_tags_in_remotes: set[str | None] = set()
 30
 31
 32def get_config(key: str) -> str | None:
 33    """Read a git config value, returning None if unset."""
 34    try:
 35        return spawn.capture(["git", "config", key]).strip()
 36    except subprocess.CalledProcessError:
 37        return None
 38
 39
 40def get_user_name() -> str | None:
 41    """Get the configured git user.name."""
 42    return get_config("user.name")
 43
 44
 45def get_user_email() -> str | None:
 46    """Get the configured git user.email."""
 47    return get_config("user.email")
 48
 49
 50def rev_count(rev: str) -> int:
 51    """Count the commits up to a revision.
 52
 53    Args:
 54        rev: A Git revision in any format know to the Git CLI.
 55
 56    Returns:
 57        count: The number of commits in the Git repository starting from the
 58            initial commit and ending with the specified commit, inclusive.
 59    """
 60    return int(spawn.capture(["git", "rev-list", "--count", rev, "--"]).strip())
 61
 62
 63def get_first_parent_commits(rev: str, limit: int) -> list[str]:
 64    """Get commit hashes along the first-parent chain starting from rev.
 65
 66    Returns up to `limit` commit hashes (including rev itself), following
 67    only first parents (i.e., staying on the main branch).
 68    """
 69    return (
 70        spawn.capture(["git", "rev-list", "--first-parent", f"-{limit}", rev])
 71        .strip()
 72        .splitlines()
 73    )
 74
 75
 76def rev_parse(rev: str, *, abbrev: bool = False) -> str:
 77    """Compute the hash for a revision.
 78
 79    Args:
 80        rev: A Git revision in any format known to the Git CLI.
 81        abbrev: Return a branch or tag name instead of a git sha
 82
 83    Returns:
 84        ref: A 40 character hex-encoded SHA-1 hash representing the ID of the
 85            named revision in Git's object database.
 86
 87            With "abbrev=True" this will return an abbreviated ref, or throw an
 88            error if there is no abbrev.
 89    """
 90    a = ["--abbrev-ref"] if abbrev else []
 91    out = spawn.capture(["git", "rev-parse", *a, "--verify", rev]).strip()
 92    if not out:
 93        raise RuntimeError(f"No parsed rev for {rev}")
 94    return out
 95
 96
 97@functools.cache
 98def expand_globs(root: Path, *specs: Path | str) -> set[str]:
 99    """Find unignored files within the specified paths."""
100    # The goal here is to find all files in the working tree that are not
101    # ignored by .gitignore. Naively using `git ls-files` doesn't work, because
102    # it reports files that have been deleted in the working tree if they are
103    # still present in the index. Using `os.walkdir` doesn't work because there
104    # is no good way to evaluate .gitignore rules from Python. So we use a
105    # combination of `git diff` and `git ls-files`.
106
107    # `git diff` against the empty tree surfaces all tracked files that have
108    # not been deleted.
109    empty_tree = (
110        "4b825dc642cb6eb9a060e54bf8d69288fbee4904"  # git hash-object -t tree /dev/null
111    )
112    diff_files = spawn.capture(
113        ["git", "diff", "--name-only", "-z", "--relative", empty_tree, "--", *specs],
114        cwd=root,
115    )
116
117    # `git ls-files --others --exclude-standard` surfaces any non-ignored,
118    # untracked files, which are not included in the `git diff` output above.
119    ls_files = spawn.capture(
120        ["git", "ls-files", "--others", "--exclude-standard", "-z", "--", *specs],
121        cwd=root,
122    )
123
124    return set(f for f in (diff_files + ls_files).split("\0") if f.strip() != "")
125
126
127def get_version_tags(
128    *,
129    version_type: type[VERSION_TYPE],
130    newest_first: bool = True,
131    fetch: bool = True,
132    remote_url: str = MATERIALIZE_REMOTE_URL,
133) -> list[VERSION_TYPE]:
134    """List all the version-like tags in the repo
135
136    Args:
137        fetch: If false, don't automatically run `git fetch --tags`.
138        prefix: A prefix to strip from each tag before attempting to parse the
139            tag as a version.
140    """
141    if fetch:
142        _fetch(
143            remote=get_remote(remote_url),
144            include_tags=YesNoOnce.ONCE,
145            force=True,
146            only_tags=True,
147        )
148    tags = []
149    for t in spawn.capture(["git", "tag"]).splitlines():
150        if not t.startswith(version_type.get_prefix()):
151            continue
152        try:
153            tags.append(version_type.parse(t))
154        except ValueError as e:
155            print(f"WARN: {e}", file=sys.stderr)
156
157    return sorted(tags, reverse=newest_first)
158
159
160def get_latest_version(
161    version_type: type[VERSION_TYPE],
162    excluded_versions: set[VERSION_TYPE] | None = None,
163    current_version: VERSION_TYPE | None = None,
164) -> VERSION_TYPE:
165    all_version_tags: list[VERSION_TYPE] = get_version_tags(
166        version_type=version_type, fetch=True
167    )
168
169    if excluded_versions is not None:
170        all_version_tags = [
171            v
172            for v in all_version_tags
173            if v not in excluded_versions
174            and (not current_version or v < current_version)
175        ]
176
177    return max(all_version_tags)
178
179
180def get_tags_of_current_commit(include_tags: YesNoOnce = YesNoOnce.ONCE) -> list[str]:
181    if include_tags:
182        fetch(get_remote(), include_tags=include_tags, only_tags=True)
183
184    result = spawn.capture(["git", "tag", "--points-at", "HEAD"])
185
186    if len(result) == 0:
187        return []
188
189    return result.splitlines()
190
191
192def is_ancestor(earlier: str, later: str) -> bool:
193    """True if earlier is in an ancestor of later"""
194    try:
195        headers = {"Accept": "application/vnd.github+json"}
196        if token := os.getenv("GITHUB_TOKEN"):
197            headers["Authorization"] = f"Bearer {token}"
198
199        # GitHub resolves the ref "HEAD" server-side to the repo's default
200        # branch, not the local checkout's HEAD. Resolve it to a concrete SHA
201        # first, otherwise e.g. compare/HEAD...main becomes main...main.
202        api_earlier = rev_parse(earlier) if earlier == "HEAD" else earlier
203        api_later = rev_parse(later) if later == "HEAD" else later
204
205        resp = requests.get(
206            f"https://api.github.com/repos/materializeinc/materialize/compare/{api_earlier}...{api_later}",
207            headers=headers,
208        )
209        resp.raise_for_status()
210        data = resp.json()
211        return data.get("status") in ("ahead", "identical")
212    except Exception as e:
213        # Try locally if Github is down or the change has not been pushed yet when running locally
214        print(f"Failed to get ancestor status from Github, running locally: {e}")
215
216        # Make sure we have an up to date view of main.
217        command = ["git", "fetch"]
218        if (
219            spawn.capture(["git", "rev-parse", "--is-shallow-repository"]).strip()
220            == "true"
221        ):
222            command.append("--unshallow")
223        spawn.runv(command + [get_remote(), earlier, later])
224
225        return (
226            spawn.run_and_get_return_code(
227                ["git", "merge-base", "--is-ancestor", earlier, later]
228            )
229            == 0
230        )
231
232
233def is_dirty() -> bool:
234    """Check if the working directory has modifications to tracked files"""
235    proc = subprocess.run("git diff --no-ext-diff --quiet --exit-code".split())
236    idx = subprocess.run("git diff --cached --no-ext-diff --quiet --exit-code".split())
237    return proc.returncode != 0 or idx.returncode != 0
238
239
240def describe() -> str:
241    """Describe the relationship between the current commit and the most recent tag"""
242    return spawn.capture(["git", "describe"]).strip()
243
244
245def fetch(
246    remote: str | None = None,
247    all_remotes: bool = False,
248    include_tags: YesNoOnce = YesNoOnce.NO,
249    force: bool = False,
250    branch: str | None = None,
251    only_tags: bool = False,
252) -> str:
253    """Fetch from remotes"""
254
255    if remote is not None and all_remotes:
256        raise RuntimeError("all_remotes must be false when a remote is specified")
257
258    if branch is not None and remote is None:
259        raise RuntimeError("remote must be specified when a branch is specified")
260
261    if branch is not None and only_tags:
262        raise RuntimeError("branch must not be specified if only_tags is set")
263
264    command = ["git", "fetch"]
265    if spawn.capture(["git", "rev-parse", "--is-shallow-repository"]).strip() == "true":
266        command.append("--unshallow")
267
268    if remote:
269        command.append(remote)
270
271    if branch:
272        command.append(branch)
273
274    if all_remotes:
275        command.append("--all")
276
277    fetch_tags = (
278        include_tags == YesNoOnce.YES
279        # fetch tags again if used with force (tags might have changed)
280        or (include_tags == YesNoOnce.ONCE and force)
281        or (
282            include_tags == YesNoOnce.ONCE
283            and remote not in fetched_tags_in_remotes
284            and "*" not in fetched_tags_in_remotes
285        )
286    )
287
288    if fetch_tags:
289        command.append("--tags")
290
291    if force:
292        command.append("--force")
293
294    if not fetch_tags and only_tags:
295        return ""
296
297    output = spawn.capture(command).strip()
298
299    if fetch_tags:
300        fetched_tags_in_remotes.add(remote)
301
302        if all_remotes:
303            fetched_tags_in_remotes.add("*")
304
305    return output
306
307
308_fetch = fetch  # renamed because an argument shadows the fetch name in get_tags
309
310
311def try_get_remote_name_by_url(url: str) -> str | None:
312    result = spawn.capture(["git", "remote", "--verbose"])
313    for line in result.splitlines():
314        remote, desc = line.split("\t")
315        if desc.lower() in (f"{url} (fetch)".lower(), f"{url}.git (fetch)".lower()):
316            return remote
317    return None
318
319
320def get_remote(
321    url: str = MATERIALIZE_REMOTE_URL,
322    default_remote_name: str = "origin",
323) -> str:
324    # Alternative syntax
325    remote = try_get_remote_name_by_url(url) or try_get_remote_name_by_url(
326        url.replace("https://github.com/", "git@github.com:")
327    )
328    if not remote:
329        remote = default_remote_name
330        print(f"Remote for URL {url} not found, using {remote}")
331
332    return remote
333
334
335def get_common_ancestor_commit(remote: str, branch: str) -> str:
336    try:
337        head = spawn.capture(["git", "rev-parse", "HEAD"]).strip()
338        headers = {"Accept": "application/vnd.github+json"}
339        if token := os.getenv("GITHUB_TOKEN"):
340            headers["Authorization"] = f"Bearer {token}"
341
342        resp = requests.get(
343            f"https://api.github.com/repos/materializeinc/materialize/compare/{head}...{branch}",
344            headers=headers,
345        )
346        resp.raise_for_status()
347        data = resp.json()
348        return data["merge_base_commit"]["sha"]
349    except Exception as e:
350        # Try locally if Github is down or the change has not been pushed yet when running locally
351        print(f"Failed to get ancestor commit from Github, running locally: {e}")
352
353        # Make sure we have an up to date view
354        command = ["git", "fetch"]
355        if (
356            spawn.capture(["git", "rev-parse", "--is-shallow-repository"]).strip()
357            == "true"
358        ):
359            command.append("--unshallow")
360        spawn.runv(command + [remote, branch])
361
362        return spawn.capture(
363            ["git", "merge-base", "HEAD", f"{remote}/{branch}"]
364        ).strip()
365
366
367def is_on_release_version() -> bool:
368    git_tags = get_tags_of_current_commit()
369    return any(MzVersion.is_valid_version_string(git_tag) for git_tag in git_tags)
370
371
372def contains_commit(
373    commit_sha: str,
374    target: str = "HEAD",
375    remote_url: str = MATERIALIZE_REMOTE_URL,
376) -> bool:
377    return is_ancestor(commit_sha, target)
378
379
380def get_tagged_release_version(version_type: type[VERSION_TYPE]) -> VERSION_TYPE | None:
381    """
382    This returns the release version if exactly this commit is tagged.
383    If multiple release versions are present, the highest one will be returned.
384    None will be returned if the commit is not tagged.
385    """
386    git_tags = get_tags_of_current_commit()
387
388    versions: list[VERSION_TYPE] = []
389
390    for git_tag in git_tags:
391        if version_type.is_valid_version_string(git_tag):
392            versions.append(version_type.parse(git_tag))
393
394    if len(versions) == 0:
395        return None
396
397    if len(versions) > 1:
398        print(
399            "Warning! Commit is tagged with multiple release versions! Returning the highest."
400        )
401
402    return max(versions)
403
404
405def get_commit_message(commit_sha: str) -> str | None:
406    try:
407        command = ["git", "log", "-1", "--pretty=format:%s", commit_sha]
408        return spawn.capture(command, stderr=subprocess.DEVNULL).strip()
409    except subprocess.CalledProcessError:
410        # Sometimes mz_version() will report a Git SHA that is not available
411        # in the current repository
412        return None
413
414
415def get_branch_name() -> str:
416    """This may not work on Buildkite; consider using the same function from build_context."""
417    command = ["git", "branch", "--show-current"]
418    return spawn.capture(command).strip()
419
420
421# Work tree mutation
422
423
424def create_branch(name: str) -> None:
425    spawn.runv(["git", "checkout", "-b", name])
426
427
428def checkout(rev: str, path: str | None = None) -> None:
429    """Git checkout the rev"""
430    cmd = ["git", "checkout", rev]
431    if path:
432        cmd.extend(["--", path])
433    spawn.runv(cmd)
434
435
436def add_file(file: str) -> None:
437    """Git add a file"""
438    spawn.runv(["git", "add", file])
439
440
441def commit_all_changed(message: str) -> None:
442    """Commit all changed files with the given message"""
443    spawn.runv(["git", "commit", "-a", "-m", message])
444
445
446def tag_annotated(tag: str) -> None:
447    """Create an annotated tag on HEAD"""
448    spawn.runv(["git", "tag", "-a", "-m", tag, tag])