misc.python.materialize.version_list

  1# Copyright Materialize, Inc. and contributors. All rights reserved.
  2#
  3# Use of this software is governed by the Business Source License
  4# included in the LICENSE file at the root of this repository.
  5#
  6# As of the Change Date specified in that file, in accordance with
  7# the Business Source License, use of this software will be governed
  8# by the Apache License, Version 2.0.
  9
 10
 11from __future__ import annotations
 12
 13import os
 14from collections.abc import Callable
 15from pathlib import Path
 16
 17import frontmatter
 18
 19from materialize import build_context, buildkite, docker, git
 20from materialize.docker import (
 21    commit_to_image_tag,
 22    image_of_commit_exists,
 23    release_version_to_image_tag,
 24)
 25from materialize.git import get_version_tags
 26from materialize.mz_version import MzVersion
 27
 28MZ_ROOT = Path(os.environ["MZ_ROOT"])
 29
 30LTS_VERSIONS = [
 31    MzVersion.parse_mz("v0.130.1"),  # v25.1.0
 32    MzVersion.parse_mz("v0.130.3"),  # v25.1.1
 33    # Put new versions at the bottom
 34]
 35
 36# not released on Docker
 37INVALID_VERSIONS = {
 38    MzVersion.parse_mz("v0.52.1"),
 39    MzVersion.parse_mz("v0.55.1"),
 40    MzVersion.parse_mz("v0.55.2"),
 41    MzVersion.parse_mz("v0.55.3"),
 42    MzVersion.parse_mz("v0.55.4"),
 43    MzVersion.parse_mz("v0.55.5"),
 44    MzVersion.parse_mz("v0.55.6"),
 45    MzVersion.parse_mz("v0.56.0"),
 46    MzVersion.parse_mz("v0.57.1"),
 47    MzVersion.parse_mz("v0.57.2"),
 48    MzVersion.parse_mz("v0.57.5"),
 49    MzVersion.parse_mz("v0.57.6"),
 50    MzVersion.parse_mz("v0.81.0"),  # incompatible for upgrades
 51    MzVersion.parse_mz("v0.81.1"),  # incompatible for upgrades
 52    MzVersion.parse_mz("v0.81.2"),  # incompatible for upgrades
 53    MzVersion.parse_mz("v0.89.7"),
 54    MzVersion.parse_mz("v0.92.0"),  # incompatible for upgrades
 55    MzVersion.parse_mz("v0.93.0"),  # accidental release
 56    MzVersion.parse_mz("v0.99.1"),  # incompatible for upgrades
 57    MzVersion.parse_mz("v0.113.1"),  # incompatible for upgrades
 58}
 59
 60_SKIP_IMAGE_CHECK_BELOW_THIS_VERSION = MzVersion.parse_mz("v0.77.0")
 61
 62
 63def resolve_ancestor_image_tag(ancestor_overrides: dict[str, MzVersion]) -> str:
 64    """
 65    Resolve the ancestor image tag.
 66    :param ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS
 67    :return: image of the ancestor
 68    """
 69
 70    manual_ancestor_override = os.getenv("COMMON_ANCESTOR_OVERRIDE")
 71    if manual_ancestor_override is not None:
 72        image_tag = _manual_ancestor_specification_to_image_tag(
 73            manual_ancestor_override
 74        )
 75        print(
 76            f"Using specified {image_tag} as image tag for ancestor (context: specified in $COMMON_ANCESTOR_OVERRIDE)"
 77        )
 78        return image_tag
 79
 80    ancestor_image_resolution = _create_ancestor_image_resolution(ancestor_overrides)
 81    image_tag, context = ancestor_image_resolution.resolve_image_tag()
 82    print(f"Using {image_tag} as image tag for ancestor (context: {context})")
 83    return image_tag
 84
 85
 86def _create_ancestor_image_resolution(
 87    ancestor_overrides: dict[str, MzVersion]
 88) -> AncestorImageResolutionBase:
 89    if buildkite.is_in_buildkite():
 90        return AncestorImageResolutionInBuildkite(ancestor_overrides)
 91    else:
 92        return AncestorImageResolutionLocal(ancestor_overrides)
 93
 94
 95def _manual_ancestor_specification_to_image_tag(ancestor_spec: str) -> str:
 96    if MzVersion.is_valid_version_string(ancestor_spec):
 97        return release_version_to_image_tag(MzVersion.parse_mz(ancestor_spec))
 98    else:
 99        return commit_to_image_tag(ancestor_spec)
100
101
102class AncestorImageResolutionBase:
103    def __init__(self, ancestor_overrides: dict[str, MzVersion]):
104        self.ancestor_overrides = ancestor_overrides
105
106    def resolve_image_tag(self) -> tuple[str, str]:
107        raise NotImplementedError
108
109    def _get_override_commit_instead_of_version(
110        self,
111        version: MzVersion,
112    ) -> str | None:
113        """
114        If a commit specifies a mz version as prerequisite (to avoid regressions) that is newer than the provided
115        version (i.e., prerequisite not satisfied by the latest version), then return that commit's hash if the commit
116        contained in the current state.
117        Otherwise, return none.
118        """
119        for (
120            commit_hash,
121            min_required_mz_version,
122        ) in self.ancestor_overrides.items():
123            if version >= min_required_mz_version:
124                continue
125
126            if git.contains_commit(commit_hash):
127                # commit would require at least min_required_mz_version
128                return commit_hash
129
130        return None
131
132    def _resolve_image_tag_of_previous_release(
133        self, context_prefix: str, previous_minor: bool
134    ) -> tuple[str, str]:
135        tagged_release_version = git.get_tagged_release_version(version_type=MzVersion)
136        assert tagged_release_version is not None
137        previous_release_version = get_previous_published_version(
138            tagged_release_version, previous_minor=previous_minor
139        )
140
141        override_commit = self._get_override_commit_instead_of_version(
142            previous_release_version
143        )
144
145        if override_commit is not None:
146            # use the commit instead of the previous release
147            return (
148                commit_to_image_tag(override_commit),
149                f"commit override instead of previous release ({previous_release_version})",
150            )
151
152        return (
153            release_version_to_image_tag(previous_release_version),
154            f"{context_prefix} {tagged_release_version}",
155        )
156
157    def _resolve_image_tag_of_previous_release_from_current(
158        self, context: str
159    ) -> tuple[str, str]:
160        # Even though we are on main we might be in an older state, pick the
161        # latest release that was before our current version.
162        current_version = MzVersion.parse_cargo()
163
164        previous_published_version = get_previous_published_version(
165            current_version, previous_minor=True
166        )
167        override_commit = self._get_override_commit_instead_of_version(
168            previous_published_version
169        )
170
171        if override_commit is not None:
172            # use the commit instead of the latest release
173            return (
174                commit_to_image_tag(override_commit),
175                f"commit override instead of latest release ({previous_published_version})",
176            )
177
178        return (
179            release_version_to_image_tag(previous_published_version),
180            context,
181        )
182
183    def _resolve_image_tag_of_merge_base(
184        self,
185        context_when_image_of_commit_exists: str,
186        context_when_falling_back_to_latest: str,
187    ) -> tuple[str, str]:
188        # If the current PR has a known and accepted regression, don't compare
189        # against merge base of it
190        override_commit = self._get_override_commit_instead_of_version(
191            MzVersion.parse_cargo()
192        )
193        common_ancestor_commit = buildkite.get_merge_base()
194
195        if override_commit is not None:
196            return (
197                commit_to_image_tag(override_commit),
198                f"commit override instead of merge base ({common_ancestor_commit})",
199            )
200
201        if image_of_commit_exists(common_ancestor_commit):
202            return (
203                commit_to_image_tag(common_ancestor_commit),
204                context_when_image_of_commit_exists,
205            )
206        else:
207            return (
208                release_version_to_image_tag(get_latest_published_version()),
209                context_when_falling_back_to_latest,
210            )
211
212
213class AncestorImageResolutionLocal(AncestorImageResolutionBase):
214    def resolve_image_tag(self) -> tuple[str, str]:
215        if build_context.is_on_release_version():
216            return self._resolve_image_tag_of_previous_release(
217                "previous minor release because on local release branch",
218                previous_minor=True,
219            )
220        elif build_context.is_on_main_branch():
221            return self._resolve_image_tag_of_previous_release_from_current(
222                "previous release from current because on local main branch"
223            )
224        else:
225            return self._resolve_image_tag_of_merge_base(
226                "merge base of local non-main branch",
227                "latest release because image of merge base of local non-main branch not available",
228            )
229
230
231class AncestorImageResolutionInBuildkite(AncestorImageResolutionBase):
232    def resolve_image_tag(self) -> tuple[str, str]:
233        if buildkite.is_in_pull_request():
234            return self._resolve_image_tag_of_merge_base(
235                "merge base of pull request",
236                "latest release because image of merge base of pull request not available",
237            )
238        elif build_context.is_on_release_version():
239            return self._resolve_image_tag_of_previous_release(
240                "previous minor release because on release branch", previous_minor=True
241            )
242        else:
243            return self._resolve_image_tag_of_previous_release_from_current(
244                "previous release from current because not in a pull request and not on a release branch",
245            )
246
247
248def get_latest_published_version() -> MzVersion:
249    """Get the latest mz version for which an image is published."""
250    excluded_versions = set()
251
252    while True:
253        latest_published_version = git.get_latest_version(
254            version_type=MzVersion, excluded_versions=excluded_versions
255        )
256
257        if is_valid_release_image(latest_published_version):
258            return latest_published_version
259        else:
260            print(
261                f"Skipping version {latest_published_version} (image not found), trying earlier version"
262            )
263            excluded_versions.add(latest_published_version)
264
265
266def get_previous_published_version(
267    release_version: MzVersion, previous_minor: bool
268) -> MzVersion:
269    """Get the highest preceding mz version to the specified version for which an image is published."""
270    excluded_versions = set()
271
272    while True:
273        previous_published_version = get_previous_mz_version(
274            release_version,
275            previous_minor=previous_minor,
276            excluded_versions=excluded_versions,
277        )
278
279        if is_valid_release_image(previous_published_version):
280            return previous_published_version
281        else:
282            print(f"Skipping version {previous_published_version} (image not found)")
283            excluded_versions.add(previous_published_version)
284
285
286def get_published_minor_mz_versions(
287    newest_first: bool = True,
288    limit: int | None = None,
289    include_filter: Callable[[MzVersion], bool] | None = None,
290    exclude_current_minor_version: bool = False,
291) -> list[MzVersion]:
292    """
293    Get the latest patch version for every minor version.
294    Use this version if it is NOT important whether a tag was introduced before or after creating this branch.
295
296    See also: #get_minor_mz_versions_listed_in_docs()
297    """
298
299    # sorted in descending order
300    all_versions = get_all_mz_versions(newest_first=True)
301    minor_versions: dict[str, MzVersion] = {}
302
303    version = MzVersion.parse_cargo()
304    current_version = f"{version.major}.{version.minor}"
305
306    # Note that this method must not apply limit_to_published_versions to a created list
307    # because in that case minor versions may get lost.
308    for version in all_versions:
309        if include_filter is not None and not include_filter(version):
310            # this version shall not be included
311            continue
312
313        minor_version = f"{version.major}.{version.minor}"
314
315        if exclude_current_minor_version and minor_version == current_version:
316            continue
317
318        if minor_version in minor_versions.keys():
319            # we already have a more recent version for this minor version
320            continue
321
322        if not is_valid_release_image(version):
323            # this version is not considered valid
324            continue
325
326        minor_versions[minor_version] = version
327
328        if limit is not None and len(minor_versions.keys()) == limit:
329            # collected enough versions
330            break
331
332    assert len(minor_versions) > 0
333    return sorted(minor_versions.values(), reverse=newest_first)
334
335
336def get_minor_mz_versions_listed_in_docs(respect_released_tag: bool) -> list[MzVersion]:
337    """
338    Get the latest patch version for every minor version in ascending order.
339    Use this version if it is important whether a tag was introduced before or after creating this branch.
340
341    See also: #get_published_minor_mz_versions()
342    """
343    return VersionsFromDocs(respect_released_tag).minor_versions()
344
345
346def get_all_mz_versions(
347    newest_first: bool = True,
348) -> list[MzVersion]:
349    """
350    Get all mz versions based on git tags. Versions known to be invalid are excluded.
351
352    See also: #get_all_mz_versions_listed_in_docs
353    """
354    return [
355        version
356        for version in get_version_tags(
357            version_type=MzVersion, newest_first=newest_first
358        )
359        if version not in INVALID_VERSIONS
360    ]
361
362
363def get_all_mz_versions_listed_in_docs(
364    respect_released_tag: bool,
365) -> list[MzVersion]:
366    """
367    Get all mz versions based on docs. Versions known to be invalid are excluded.
368
369    See also: #get_all_mz_versions()
370    """
371    return VersionsFromDocs(respect_released_tag).all_versions()
372
373
374def get_all_published_mz_versions(
375    newest_first: bool = True, limit: int | None = None
376) -> list[MzVersion]:
377    """Get all mz versions based on git tags. This method ensures that images of the versions exist."""
378    return limit_to_published_versions(
379        get_all_mz_versions(newest_first=newest_first), limit
380    )
381
382
383def limit_to_published_versions(
384    all_versions: list[MzVersion], limit: int | None = None
385) -> list[MzVersion]:
386    """Remove versions for which no image is published."""
387    versions = []
388
389    for v in all_versions:
390        if is_valid_release_image(v):
391            versions.append(v)
392
393        if limit is not None and len(versions) == limit:
394            break
395
396    return versions
397
398
399def get_previous_mz_version(
400    version: MzVersion,
401    previous_minor: bool,
402    excluded_versions: set[MzVersion] | None = None,
403) -> MzVersion:
404    """Get the predecessor of the specified version based on git tags."""
405    if excluded_versions is None:
406        excluded_versions = set()
407
408    if previous_minor:
409        version = MzVersion.create(version.major, version.minor, 0)
410
411    if version.prerelease is not None and len(version.prerelease) > 0:
412        # simply drop the prerelease, do not try to find a decremented version
413        found_version = MzVersion.create(version.major, version.minor, version.patch)
414
415        if found_version not in excluded_versions:
416            return found_version
417        else:
418            # start searching with this version
419            version = found_version
420
421    all_versions: list[MzVersion] = get_version_tags(version_type=type(version))
422    all_suitable_previous_versions = [
423        v
424        for v in all_versions
425        if v < version
426        and (v.prerelease is None or len(v.prerelease) == 0)
427        and v not in INVALID_VERSIONS
428        and v not in excluded_versions
429    ]
430    return max(all_suitable_previous_versions)
431
432
433def is_valid_release_image(version: MzVersion) -> bool:
434    """
435    Checks if a version is not known as an invalid version and has a published image.
436    Note that this method may take shortcuts on older versions.
437    """
438    if version in INVALID_VERSIONS:
439        return False
440
441    if version < _SKIP_IMAGE_CHECK_BELOW_THIS_VERSION:
442        # optimization: assume that all versions older than this one are either valid or listed in INVALID_VERSIONS
443        return True
444
445    # This is a potentially expensive operation which pulls an image if it hasn't been pulled yet.
446    return docker.image_of_release_version_exists(version)
447
448
449def get_commits_of_accepted_regressions_between_versions(
450    ancestor_overrides: dict[str, MzVersion],
451    since_version_exclusive: MzVersion,
452    to_version_inclusive: MzVersion,
453) -> list[str]:
454    """
455    Get commits of accepted regressions between both versions.
456    :param ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS
457    :return: commits
458    """
459
460    assert since_version_exclusive <= to_version_inclusive
461
462    commits = []
463
464    for (
465        regression_introducing_commit,
466        first_version_with_regression,
467    ) in ancestor_overrides.items():
468        if (
469            since_version_exclusive
470            < first_version_with_regression
471            <= to_version_inclusive
472        ):
473            commits.append(regression_introducing_commit)
474
475    return commits
476
477
478class VersionsFromDocs:
479    """Materialize versions as listed in doc/user/content/releases
480
481    Only versions that declare `versiond: true` in their
482    frontmatter are considered.
483
484    >>> len(VersionsFromDocs(respect_released_tag=True).all_versions()) > 0
485    True
486
487    >>> len(VersionsFromDocs(respect_released_tag=True).minor_versions()) > 0
488    True
489
490    >>> len(VersionsFromDocs(respect_released_tag=True).patch_versions(minor_version=MzVersion.parse_mz("v0.52.0")))
491    4
492
493    >>> min(VersionsFromDocs(respect_released_tag=True).all_versions())
494    MzVersion(major=0, minor=27, patch=0, prerelease=None, build=None)
495    """
496
497    def __init__(self, respect_released_tag: bool) -> None:
498        files = Path(MZ_ROOT / "doc" / "user" / "content" / "releases").glob("v*.md")
499        self.versions = []
500        current_version = MzVersion.parse_cargo()
501        for f in files:
502            base = f.stem
503            metadata = frontmatter.load(f)
504            if respect_released_tag and not metadata.get("released", False):
505                continue
506
507            current_patch = metadata.get("patch", 0)
508
509            for patch in range(current_patch + 1):
510                version = MzVersion.parse_mz(f"{base}.{patch}")
511                if not respect_released_tag and version >= current_version:
512                    continue
513                if version not in INVALID_VERSIONS:
514                    self.versions.append(version)
515
516        assert len(self.versions) > 0
517        self.versions.sort()
518
519    def all_versions(self) -> list[MzVersion]:
520        return self.versions
521
522    def minor_versions(self) -> list[MzVersion]:
523        """Return the latest patch version for every minor version."""
524        minor_versions = {}
525        for version in self.versions:
526            minor_versions[f"{version.major}.{version.minor}"] = version
527
528        assert len(minor_versions) > 0
529        return sorted(minor_versions.values())
530
531    def patch_versions(self, minor_version: MzVersion) -> list[MzVersion]:
532        """Return all patch versions within the given minor version."""
533        patch_versions = []
534        for version in self.versions:
535            if (
536                version.major == minor_version.major
537                and version.minor == minor_version.minor
538            ):
539                patch_versions.append(version)
540
541        assert len(patch_versions) > 0
542        return sorted(patch_versions)