misc.python.materialize.version_list
1# Copyright Materialize, Inc. and contributors. All rights reserved. 2# 3# Use of this software is governed by the Business Source License 4# included in the LICENSE file at the root of this repository. 5# 6# As of the Change Date specified in that file, in accordance with 7# the Business Source License, use of this software will be governed 8# by the Apache License, Version 2.0. 9 10 11from __future__ import annotations 12 13import datetime 14import os 15from collections.abc import Callable 16from dataclasses import dataclass 17from pathlib import Path 18 19import frontmatter 20import requests 21import yaml 22 23from materialize import build_context, buildkite, docker, git 24from materialize.docker import ( 25 commit_to_image_tag, 26 image_of_commit_exists, 27 release_version_to_image_tag, 28) 29from materialize.git import get_version_tags 30from materialize.mz_version import MzVersion 31 32MZ_ROOT = Path(os.environ["MZ_ROOT"]) 33 34 35@dataclass 36class SelfManagedVersion: 37 helm_version: MzVersion 38 version: MzVersion 39 40 41def fetch_self_managed_versions() -> list[SelfManagedVersion]: 42 result: list[SelfManagedVersion] = [] 43 for entry in yaml.safe_load( 44 requests.get("https://materializeinc.github.io/materialize/index.yaml").text 45 )["entries"]["materialize-operator"]: 46 self_managed_version = SelfManagedVersion( 47 MzVersion.parse_mz(entry["version"]), 48 MzVersion.parse_mz(entry["appVersion"]), 49 ) 50 if ( 51 not self_managed_version.version.prerelease 52 and self_managed_version.version not in BAD_SELF_MANAGED_VERSIONS 53 ): 54 result.append(self_managed_version) 55 return result 56 57 58def get_all_self_managed_versions() -> list[MzVersion]: 59 return sorted([version.version for version in fetch_self_managed_versions()]) 60 61 62def get_self_managed_versions( 63 max_version: MzVersion | None = None, 64) -> list[MzVersion]: 65 prefixes = set() 66 result = set() 67 self_managed_versions = fetch_self_managed_versions() 68 for version_info in self_managed_versions: 69 if max_version is not None and version_info.version >= max_version: 70 continue 71 prefix = (version_info.version.major, version_info.version.minor) 72 if ( 73 not version_info.version.prerelease 74 and prefix not in prefixes 75 and not version_info.helm_version.prerelease 76 ): 77 result.add(version_info.version) 78 prefixes.add(prefix) 79 return sorted(result) 80 81 82# Gets the range of versions we can "upgrade from" to the current version, sorted in ascending order. 83def get_compatible_upgrade_from_versions() -> list[MzVersion]: 84 85 # Determine the current MzVersion from the environment, or from a version constant 86 current_version = MzVersion.parse_cargo() 87 88 published_versions_within_one_major_version = { 89 v 90 for v in get_published_mz_versions_within_one_major_version() 91 if abs(v.major - current_version.major) <= 1 and v <= current_version 92 } 93 94 if current_version.major <= 26: 95 # For versions <= 26, we can only upgrade from 25.2 self-managed versions 96 self_managed_25_2_versions = { 97 v.version 98 for v in fetch_self_managed_versions() 99 if v.helm_version.major == 25 and v.helm_version.minor == 2 100 } 101 102 return sorted( 103 self_managed_25_2_versions.union( 104 published_versions_within_one_major_version 105 ) 106 ) 107 else: 108 # For versions > 26, get all mz versions within 1 major version of current_version 109 return sorted(published_versions_within_one_major_version) 110 111 112def keep_latest_patch_per_minor(versions: list[MzVersion]) -> list[MzVersion]: 113 """Thin a version list down to the latest patch of each (major, minor). 114 115 A linear upgrade path that steps through every version grows unbounded as 116 releases accumulate, eventually exceeding the test's time budget. Keeping 117 only the latest patch of each minor preserves coverage of every minor 118 boundary, no minor is skipped, while dropping the redundant intra-minor 119 patch hops that dominate the runtime. The latest patch is the version a 120 user on that minor would actually upgrade from. 121 """ 122 latest_per_minor: dict[tuple[int, int], MzVersion] = {} 123 for version in versions: 124 key = (version.major, version.minor) 125 current = latest_per_minor.get(key) 126 if current is None or version > current: 127 latest_per_minor[key] = version 128 return sorted(latest_per_minor.values()) 129 130 131BAD_SELF_MANAGED_VERSIONS = { 132 MzVersion.parse_mz("v0.130.0"), 133 MzVersion.parse_mz("v0.130.1"), 134 MzVersion.parse_mz("v0.130.2"), 135 MzVersion.parse_mz("v0.130.3"), 136 MzVersion.parse_mz("v0.130.4"), 137 MzVersion.parse_mz( 138 "v0.147.7" 139 ), # Incompatible for upgrades because it clears login attribute for roles due to catalog migration 140 MzVersion.parse_mz( 141 "v0.147.14" 142 ), # Incompatible for upgrades because it clears login attribute for roles due to catalog migration 143 MzVersion.parse_mz("v0.157.0"), 144} 145 146# not released on Docker 147INVALID_VERSIONS = { 148 MzVersion.parse_mz("v0.52.1"), 149 MzVersion.parse_mz("v0.55.1"), 150 MzVersion.parse_mz("v0.55.2"), 151 MzVersion.parse_mz("v0.55.3"), 152 MzVersion.parse_mz("v0.55.4"), 153 MzVersion.parse_mz("v0.55.5"), 154 MzVersion.parse_mz("v0.55.6"), 155 MzVersion.parse_mz("v0.56.0"), 156 MzVersion.parse_mz("v0.57.1"), 157 MzVersion.parse_mz("v0.57.2"), 158 MzVersion.parse_mz("v0.57.5"), 159 MzVersion.parse_mz("v0.57.6"), 160 MzVersion.parse_mz("v0.81.0"), # incompatible for upgrades 161 MzVersion.parse_mz("v0.81.1"), # incompatible for upgrades 162 MzVersion.parse_mz("v0.81.2"), # incompatible for upgrades 163 MzVersion.parse_mz("v0.89.7"), 164 MzVersion.parse_mz("v0.92.0"), # incompatible for upgrades 165 MzVersion.parse_mz("v0.93.0"), # accidental release 166 MzVersion.parse_mz("v0.99.1"), # incompatible for upgrades 167 MzVersion.parse_mz("v0.113.1"), # incompatible for upgrades 168} 169 170_SKIP_IMAGE_CHECK_BELOW_THIS_VERSION = MzVersion.parse_mz("v0.77.0") 171 172 173def resolve_ancestor_image_tag(ancestor_overrides: dict[str, MzVersion]) -> str | None: 174 """ 175 Resolve the ancestor image tag. 176 :param ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS 177 :return: image of the ancestor 178 """ 179 180 manual_ancestor_override = os.getenv("COMMON_ANCESTOR_OVERRIDE") 181 if manual_ancestor_override is not None: 182 image_tag = _manual_ancestor_specification_to_image_tag( 183 manual_ancestor_override 184 ) 185 print( 186 f"Using specified {image_tag} as image tag for ancestor (context: specified in $COMMON_ANCESTOR_OVERRIDE)" 187 ) 188 return image_tag 189 190 ancestor_image_resolution = _create_ancestor_image_resolution(ancestor_overrides) 191 result = ancestor_image_resolution.resolve_image_tag() 192 if result is None: 193 return None 194 image_tag, context = result 195 print(f"Using {image_tag} as image tag for ancestor (context: {context})") 196 return image_tag 197 198 199def _create_ancestor_image_resolution( 200 ancestor_overrides: dict[str, MzVersion], 201) -> AncestorImageResolutionBase: 202 if buildkite.is_in_buildkite(): 203 return AncestorImageResolutionInBuildkite(ancestor_overrides) 204 else: 205 return AncestorImageResolutionLocal(ancestor_overrides) 206 207 208def _manual_ancestor_specification_to_image_tag(ancestor_spec: str) -> str: 209 if MzVersion.is_valid_version_string(ancestor_spec): 210 return release_version_to_image_tag(MzVersion.parse_mz(ancestor_spec)) 211 else: 212 return commit_to_image_tag(ancestor_spec) 213 214 215class AncestorImageResolutionBase: 216 def __init__(self, ancestor_overrides: dict[str, MzVersion]): 217 self.ancestor_overrides = ancestor_overrides 218 219 def resolve_image_tag(self) -> tuple[str, str] | None: 220 raise NotImplementedError 221 222 def _get_override_commit_instead_of_version( 223 self, 224 version: MzVersion, 225 ) -> str | None: 226 """ 227 If a commit specifies a mz version as prerequisite (to avoid regressions) that is newer than the provided 228 version (i.e., prerequisite not satisfied by the latest version), then return that commit's hash if the commit 229 contained in the current state. 230 Otherwise, return none. 231 """ 232 for ( 233 commit_hash, 234 min_required_mz_version, 235 ) in self.ancestor_overrides.items(): 236 if version >= min_required_mz_version: 237 continue 238 239 if git.contains_commit(commit_hash): 240 # commit would require at least min_required_mz_version 241 return commit_hash 242 243 return None 244 245 def _resolve_image_tag_of_previous_release( 246 self, context_prefix: str, previous_minor: bool 247 ) -> tuple[str, str] | None: 248 tagged_release_version = git.get_tagged_release_version(version_type=MzVersion) 249 assert tagged_release_version is not None 250 previous_release_version = get_previous_published_version( 251 tagged_release_version, previous_minor=previous_minor 252 ) 253 254 override_commit = self._get_override_commit_instead_of_version( 255 previous_release_version 256 ) 257 258 if override_commit is not None: 259 # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use 260 # use the commit instead of the previous release 261 # return ( 262 # commit_to_image_tag(override_commit), 263 # f"commit override instead of previous release ({previous_release_version})", 264 # ) 265 return None 266 267 return ( 268 release_version_to_image_tag(previous_release_version), 269 f"{context_prefix} {tagged_release_version}", 270 ) 271 272 def _resolve_image_tag_of_previous_release_from_current( 273 self, context: str 274 ) -> tuple[str, str] | None: 275 # Even though we are on main we might be in an older state, pick the 276 # latest release that was before our current version. 277 current_version = MzVersion.parse_cargo() 278 279 previous_published_version = get_previous_published_version( 280 current_version, previous_minor=True 281 ) 282 override_commit = self._get_override_commit_instead_of_version( 283 previous_published_version 284 ) 285 286 if override_commit is not None: 287 # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use 288 # use the commit instead of the latest release 289 # return ( 290 # commit_to_image_tag(override_commit), 291 # f"commit override instead of latest release ({previous_published_version})", 292 # ) 293 return None 294 295 return ( 296 release_version_to_image_tag(previous_published_version), 297 context, 298 ) 299 300 def _resolve_image_tag_of_merge_base( 301 self, 302 context_when_image_of_commit_exists: str, 303 ) -> tuple[str, str] | None: 304 # If the current PR has a known and accepted regression, don't compare 305 # against merge base of it 306 override_commit = self._get_override_commit_instead_of_version( 307 MzVersion.parse_cargo() 308 ) 309 common_ancestor_commit = buildkite.get_merge_base() 310 311 if override_commit is not None: 312 # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use 313 # return ( 314 # commit_to_image_tag(override_commit), 315 # f"commit override instead of merge base ({common_ancestor_commit})", 316 # ) 317 return None 318 319 ancestors = git.get_first_parent_commits(common_ancestor_commit, limit=20) 320 for ancestor in ancestors: 321 if image_of_commit_exists(ancestor): 322 context = ( 323 context_when_image_of_commit_exists 324 if ancestor == common_ancestor_commit 325 else f"ancestor of {context_when_image_of_commit_exists} (walked back from {common_ancestor_commit[:12]})" 326 ) 327 return ( 328 commit_to_image_tag(ancestor), 329 context, 330 ) 331 332 return None 333 334 335class AncestorImageResolutionLocal(AncestorImageResolutionBase): 336 def resolve_image_tag(self) -> tuple[str, str] | None: 337 if build_context.is_on_release_version(): 338 return self._resolve_image_tag_of_previous_release( 339 "previous minor release because on local release branch", 340 previous_minor=True, 341 ) 342 elif build_context.is_on_main_branch(): 343 return self._resolve_image_tag_of_previous_release_from_current( 344 "previous release from current because on local main branch" 345 ) 346 else: 347 return self._resolve_image_tag_of_merge_base( 348 "merge base of local non-main branch", 349 ) 350 351 352class AncestorImageResolutionInBuildkite(AncestorImageResolutionBase): 353 def resolve_image_tag(self) -> tuple[str, str] | None: 354 if buildkite.is_in_pull_request(): 355 return self._resolve_image_tag_of_merge_base( 356 "merge base of pull request", 357 ) 358 elif build_context.is_on_release_version(): 359 return self._resolve_image_tag_of_previous_release( 360 "previous minor release because on release branch", previous_minor=True 361 ) 362 else: 363 return self._resolve_image_tag_of_previous_release_from_current( 364 "previous release from current because not in a pull request and not on a release branch", 365 ) 366 367 368def get_latest_published_version() -> MzVersion: 369 """Get the latest mz version, older than current state, for which an image is published.""" 370 excluded_versions = set() 371 current_version = MzVersion.parse_cargo() 372 373 while True: 374 latest_published_version = git.get_latest_version( 375 version_type=MzVersion, 376 excluded_versions=excluded_versions, 377 current_version=current_version, 378 ) 379 380 if is_valid_release_image(latest_published_version): 381 return latest_published_version 382 else: 383 print( 384 f"Skipping version {latest_published_version} (image not found), trying earlier version" 385 ) 386 excluded_versions.add(latest_published_version) 387 388 389def get_previous_published_version( 390 release_version: MzVersion, previous_minor: bool 391) -> MzVersion: 392 """Get the highest preceding mz version to the specified version for which an image is published.""" 393 excluded_versions = set() 394 395 while True: 396 previous_published_version = get_previous_mz_version( 397 release_version, 398 previous_minor=previous_minor, 399 excluded_versions=excluded_versions, 400 ) 401 402 if is_valid_release_image(previous_published_version): 403 return previous_published_version 404 else: 405 print(f"Skipping version {previous_published_version} (image not found)") 406 excluded_versions.add(previous_published_version) 407 408 409def get_published_minor_mz_versions( 410 newest_first: bool = True, 411 limit: int | None = None, 412 include_filter: Callable[[MzVersion], bool] | None = None, 413 exclude_current_minor_version: bool = False, 414 max_version: MzVersion | None = None, 415 include_release_candidates: bool = False, 416) -> list[MzVersion]: 417 """ 418 Get the latest patch version for every minor version. 419 Use this version if it is NOT important whether a tag was introduced before or after creating this branch. 420 421 See also: #get_minor_mz_versions_listed_in_docs() 422 """ 423 424 # sorted in descending order 425 all_versions = get_all_mz_versions( 426 newest_first=True, include_release_candidates=include_release_candidates 427 ) 428 minor_versions: dict[str, MzVersion] = {} 429 430 version = MzVersion.parse_cargo() 431 current_version = f"{version.major}.{version.minor}" 432 433 # Note that this method must not apply limit_to_published_versions to a created list 434 # because in that case minor versions may get lost. 435 for version in all_versions: 436 if include_filter is not None and not include_filter(version): 437 # this version shall not be included 438 continue 439 440 if max_version is not None and version >= max_version: 441 continue 442 443 minor_version = f"{version.major}.{version.minor}" 444 445 if exclude_current_minor_version and minor_version == current_version: 446 continue 447 448 if minor_version in minor_versions.keys(): 449 # we already have a more recent version for this minor version 450 continue 451 452 if not is_valid_release_image(version): 453 # this version is not considered valid 454 continue 455 456 minor_versions[minor_version] = version 457 458 if limit is not None and len(minor_versions.keys()) == limit: 459 # collected enough versions 460 break 461 462 assert len(minor_versions) > 0 463 return sorted(minor_versions.values(), reverse=newest_first) 464 465 466def get_all_mz_versions( 467 newest_first: bool = True, 468 include_release_candidates: bool = False, 469) -> list[MzVersion]: 470 """ 471 Get all mz versions based on git tags. Versions known to be invalid are excluded. 472 473 See also: #get_all_mz_versions_listed_in_docs 474 """ 475 return [ 476 version 477 for version in get_version_tags( 478 version_type=MzVersion, newest_first=newest_first 479 ) 480 if version not in INVALID_VERSIONS 481 and (include_release_candidates or not version.prerelease) 482 ] 483 484 485def get_all_published_mz_versions( 486 newest_first: bool = True, limit: int | None = None 487) -> list[MzVersion]: 488 """Get all mz versions based on git tags. This method ensures that images of the versions exist.""" 489 all_versions = get_all_mz_versions(newest_first=newest_first) 490 print(f"all_versions: {all_versions}") 491 return limit_to_published_versions(all_versions, limit) 492 493 494def get_published_mz_versions_within_one_major_version( 495 newest_first: bool = True, 496) -> list[MzVersion]: 497 """Get all previous mz versions within one major version of the current version. Ensure that images of the versions exist.""" 498 current_version = MzVersion.parse_cargo() 499 all_versions = get_all_mz_versions(newest_first=newest_first) 500 versions_within_one_major_version = { 501 v 502 for v in all_versions 503 if abs(v.major - current_version.major) <= 1 and v <= current_version 504 } 505 506 return limit_to_published_versions(list(versions_within_one_major_version)) 507 508 509def limit_to_published_versions( 510 all_versions: list[MzVersion], limit: int | None = None 511) -> list[MzVersion]: 512 """Remove versions for which no image is published.""" 513 versions = [] 514 515 for v in all_versions: 516 if is_valid_release_image(v): 517 versions.append(v) 518 519 if limit is not None and len(versions) == limit: 520 break 521 522 return versions 523 524 525def get_previous_mz_version( 526 version: MzVersion, 527 previous_minor: bool, 528 excluded_versions: set[MzVersion] | None = None, 529) -> MzVersion: 530 """Get the predecessor of the specified version based on git tags.""" 531 if excluded_versions is None: 532 excluded_versions = set() 533 534 if previous_minor: 535 version = MzVersion.create(version.major, version.minor, 0) 536 537 if version.prerelease is not None and len(version.prerelease) > 0: 538 # simply drop the prerelease, do not try to find a decremented version 539 found_version = MzVersion.create(version.major, version.minor, version.patch) 540 541 if found_version not in excluded_versions: 542 return found_version 543 else: 544 # start searching with this version 545 version = found_version 546 547 all_versions: list[MzVersion] = get_version_tags(version_type=type(version)) 548 all_suitable_previous_versions = [ 549 v 550 for v in all_versions 551 if v < version 552 and (v.prerelease is None or len(v.prerelease) == 0) 553 and v not in INVALID_VERSIONS 554 and v not in excluded_versions 555 ] 556 return max(all_suitable_previous_versions) 557 558 559def is_valid_release_image(version: MzVersion) -> bool: 560 """ 561 Checks if a version is not known as an invalid version and has a published image. 562 Note that this method may take shortcuts on older versions. 563 """ 564 if version in INVALID_VERSIONS: 565 return False 566 567 if version < _SKIP_IMAGE_CHECK_BELOW_THIS_VERSION: 568 # optimization: assume that all versions older than this one are either valid or listed in INVALID_VERSIONS 569 return True 570 571 # This is a potentially expensive operation which pulls an image if it hasn't been pulled yet. 572 return docker.image_of_release_version_exists(version) 573 574 575def get_commits_of_accepted_regressions_between_versions( 576 ancestor_overrides: dict[str, MzVersion], 577 since_version_exclusive: MzVersion, 578 to_version_inclusive: MzVersion, 579) -> list[str]: 580 """ 581 Get commits of accepted regressions between both versions. 582 :param ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS 583 :return: commits 584 """ 585 586 assert since_version_exclusive <= to_version_inclusive 587 588 commits = [] 589 590 for ( 591 regression_introducing_commit, 592 first_version_with_regression, 593 ) in ancestor_overrides.items(): 594 if ( 595 since_version_exclusive 596 < first_version_with_regression 597 <= to_version_inclusive 598 ): 599 commits.append(regression_introducing_commit) 600 601 return commits 602 603 604class VersionsFromDocs: 605 """Materialize versions as listed in doc/user/content/releases 606 607 >>> len(VersionsFromDocs(respect_released_tag=True).all_versions()) > 0 608 True 609 610 >>> len(VersionsFromDocs(respect_released_tag=True).minor_versions()) > 0 611 True 612 613 >>> len(VersionsFromDocs(respect_released_tag=True).patch_versions(minor_version=MzVersion.parse_mz("v0.52.0"))) 614 4 615 616 >>> min(VersionsFromDocs(respect_released_tag=True).all_versions()) 617 MzVersion(major=0, minor=27, patch=0, prerelease=None, build=None) 618 """ 619 620 def __init__( 621 self, 622 respect_released_tag: bool, 623 respect_date: bool = False, 624 only_publish_helm_chart: bool = True, 625 skip_rc: bool = False, 626 ) -> None: 627 files = Path(MZ_ROOT / "doc" / "user" / "content" / "releases").glob("v*.md") 628 self.versions = [] 629 current_version = MzVersion.parse_cargo() 630 for f in files: 631 base = f.stem 632 metadata = frontmatter.load(f) 633 if respect_released_tag and not metadata.get("released", False): 634 continue 635 if only_publish_helm_chart and not metadata.get("publish_helm_chart", True): 636 continue 637 date: datetime.date = metadata["date"] 638 if respect_date and date > datetime.date.today(): 639 continue 640 641 current_patch = metadata.get("patch", 0) 642 current_rc = metadata.get("rc", 0) 643 644 if current_rc > 0: 645 if skip_rc: 646 continue 647 for rc in range(1, current_rc + 1): 648 version = MzVersion.parse_mz(f"{base}.{current_patch}-rc.{rc}") 649 if not respect_released_tag and version >= current_version: 650 continue 651 if version not in INVALID_VERSIONS: 652 self.versions.append(version) 653 else: 654 for patch in range(current_patch + 1): 655 version = MzVersion.parse_mz(f"{base}.{patch}") 656 if not respect_released_tag and version >= current_version: 657 continue 658 if version not in INVALID_VERSIONS: 659 self.versions.append(version) 660 661 assert len(self.versions) > 0 662 self.versions.sort() 663 664 def all_versions(self) -> list[MzVersion]: 665 return self.versions 666 667 def minor_versions(self) -> list[MzVersion]: 668 """Return the latest patch version for every minor version.""" 669 minor_versions = {} 670 for version in self.versions: 671 minor_versions[f"{version.major}.{version.minor}"] = version 672 673 assert len(minor_versions) > 0 674 return sorted(minor_versions.values()) 675 676 def patch_versions(self, minor_version: MzVersion) -> list[MzVersion]: 677 """Return all patch versions within the given minor version.""" 678 patch_versions = [] 679 for version in self.versions: 680 if ( 681 version.major == minor_version.major 682 and version.minor == minor_version.minor 683 ): 684 patch_versions.append(version) 685 686 assert len(patch_versions) > 0 687 return sorted(patch_versions)
MZ_ROOT =
PosixPath('/workspace')
@dataclass
class
SelfManagedVersion:
42def fetch_self_managed_versions() -> list[SelfManagedVersion]: 43 result: list[SelfManagedVersion] = [] 44 for entry in yaml.safe_load( 45 requests.get("https://materializeinc.github.io/materialize/index.yaml").text 46 )["entries"]["materialize-operator"]: 47 self_managed_version = SelfManagedVersion( 48 MzVersion.parse_mz(entry["version"]), 49 MzVersion.parse_mz(entry["appVersion"]), 50 ) 51 if ( 52 not self_managed_version.version.prerelease 53 and self_managed_version.version not in BAD_SELF_MANAGED_VERSIONS 54 ): 55 result.append(self_managed_version) 56 return result
def
get_all_self_managed_versions() -> list[materialize.mz_version.MzVersion]:
def
get_self_managed_versions( max_version: materialize.mz_version.MzVersion | None = None) -> list[materialize.mz_version.MzVersion]:
63def get_self_managed_versions( 64 max_version: MzVersion | None = None, 65) -> list[MzVersion]: 66 prefixes = set() 67 result = set() 68 self_managed_versions = fetch_self_managed_versions() 69 for version_info in self_managed_versions: 70 if max_version is not None and version_info.version >= max_version: 71 continue 72 prefix = (version_info.version.major, version_info.version.minor) 73 if ( 74 not version_info.version.prerelease 75 and prefix not in prefixes 76 and not version_info.helm_version.prerelease 77 ): 78 result.add(version_info.version) 79 prefixes.add(prefix) 80 return sorted(result)
def
get_compatible_upgrade_from_versions() -> list[materialize.mz_version.MzVersion]:
84def get_compatible_upgrade_from_versions() -> list[MzVersion]: 85 86 # Determine the current MzVersion from the environment, or from a version constant 87 current_version = MzVersion.parse_cargo() 88 89 published_versions_within_one_major_version = { 90 v 91 for v in get_published_mz_versions_within_one_major_version() 92 if abs(v.major - current_version.major) <= 1 and v <= current_version 93 } 94 95 if current_version.major <= 26: 96 # For versions <= 26, we can only upgrade from 25.2 self-managed versions 97 self_managed_25_2_versions = { 98 v.version 99 for v in fetch_self_managed_versions() 100 if v.helm_version.major == 25 and v.helm_version.minor == 2 101 } 102 103 return sorted( 104 self_managed_25_2_versions.union( 105 published_versions_within_one_major_version 106 ) 107 ) 108 else: 109 # For versions > 26, get all mz versions within 1 major version of current_version 110 return sorted(published_versions_within_one_major_version)
def
keep_latest_patch_per_minor( versions: list[materialize.mz_version.MzVersion]) -> list[materialize.mz_version.MzVersion]:
113def keep_latest_patch_per_minor(versions: list[MzVersion]) -> list[MzVersion]: 114 """Thin a version list down to the latest patch of each (major, minor). 115 116 A linear upgrade path that steps through every version grows unbounded as 117 releases accumulate, eventually exceeding the test's time budget. Keeping 118 only the latest patch of each minor preserves coverage of every minor 119 boundary, no minor is skipped, while dropping the redundant intra-minor 120 patch hops that dominate the runtime. The latest patch is the version a 121 user on that minor would actually upgrade from. 122 """ 123 latest_per_minor: dict[tuple[int, int], MzVersion] = {} 124 for version in versions: 125 key = (version.major, version.minor) 126 current = latest_per_minor.get(key) 127 if current is None or version > current: 128 latest_per_minor[key] = version 129 return sorted(latest_per_minor.values())
Thin a version list down to the latest patch of each (major, minor).
A linear upgrade path that steps through every version grows unbounded as releases accumulate, eventually exceeding the test's time budget. Keeping only the latest patch of each minor preserves coverage of every minor boundary, no minor is skipped, while dropping the redundant intra-minor patch hops that dominate the runtime. The latest patch is the version a user on that minor would actually upgrade from.
BAD_SELF_MANAGED_VERSIONS =
{MzVersion(major=0, minor=130, patch=4, prerelease=None, build=None), MzVersion(major=0, minor=130, patch=0, prerelease=None, build=None), MzVersion(major=0, minor=147, patch=14, prerelease=None, build=None), MzVersion(major=0, minor=130, patch=1, prerelease=None, build=None), MzVersion(major=0, minor=147, patch=7, prerelease=None, build=None), MzVersion(major=0, minor=157, patch=0, prerelease=None, build=None), MzVersion(major=0, minor=130, patch=2, prerelease=None, build=None), MzVersion(major=0, minor=130, patch=3, prerelease=None, build=None)}
INVALID_VERSIONS =
{MzVersion(major=0, minor=55, patch=5, prerelease=None, build=None), MzVersion(major=0, minor=93, patch=0, prerelease=None, build=None), MzVersion(major=0, minor=57, patch=5, prerelease=None, build=None), MzVersion(major=0, minor=89, patch=7, prerelease=None, build=None), MzVersion(major=0, minor=113, patch=1, prerelease=None, build=None), MzVersion(major=0, minor=55, patch=2, prerelease=None, build=None), MzVersion(major=0, minor=55, patch=4, prerelease=None, build=None), MzVersion(major=0, minor=56, patch=0, prerelease=None, build=None), MzVersion(major=0, minor=52, patch=1, prerelease=None, build=None), MzVersion(major=0, minor=92, patch=0, prerelease=None, build=None), MzVersion(major=0, minor=57, patch=2, prerelease=None, build=None), MzVersion(major=0, minor=55, patch=6, prerelease=None, build=None), MzVersion(major=0, minor=57, patch=6, prerelease=None, build=None), MzVersion(major=0, minor=81, patch=0, prerelease=None, build=None), MzVersion(major=0, minor=99, patch=1, prerelease=None, build=None), MzVersion(major=0, minor=81, patch=2, prerelease=None, build=None), MzVersion(major=0, minor=55, patch=1, prerelease=None, build=None), MzVersion(major=0, minor=55, patch=3, prerelease=None, build=None), MzVersion(major=0, minor=57, patch=1, prerelease=None, build=None), MzVersion(major=0, minor=81, patch=1, prerelease=None, build=None)}
def
resolve_ancestor_image_tag( ancestor_overrides: dict[str, materialize.mz_version.MzVersion]) -> str | None:
174def resolve_ancestor_image_tag(ancestor_overrides: dict[str, MzVersion]) -> str | None: 175 """ 176 Resolve the ancestor image tag. 177 :param ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS 178 :return: image of the ancestor 179 """ 180 181 manual_ancestor_override = os.getenv("COMMON_ANCESTOR_OVERRIDE") 182 if manual_ancestor_override is not None: 183 image_tag = _manual_ancestor_specification_to_image_tag( 184 manual_ancestor_override 185 ) 186 print( 187 f"Using specified {image_tag} as image tag for ancestor (context: specified in $COMMON_ANCESTOR_OVERRIDE)" 188 ) 189 return image_tag 190 191 ancestor_image_resolution = _create_ancestor_image_resolution(ancestor_overrides) 192 result = ancestor_image_resolution.resolve_image_tag() 193 if result is None: 194 return None 195 image_tag, context = result 196 print(f"Using {image_tag} as image tag for ancestor (context: {context})") 197 return image_tag
Resolve the ancestor image tag.
Parameters
- ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS
Returns
image of the ancestor
class
AncestorImageResolutionBase:
216class AncestorImageResolutionBase: 217 def __init__(self, ancestor_overrides: dict[str, MzVersion]): 218 self.ancestor_overrides = ancestor_overrides 219 220 def resolve_image_tag(self) -> tuple[str, str] | None: 221 raise NotImplementedError 222 223 def _get_override_commit_instead_of_version( 224 self, 225 version: MzVersion, 226 ) -> str | None: 227 """ 228 If a commit specifies a mz version as prerequisite (to avoid regressions) that is newer than the provided 229 version (i.e., prerequisite not satisfied by the latest version), then return that commit's hash if the commit 230 contained in the current state. 231 Otherwise, return none. 232 """ 233 for ( 234 commit_hash, 235 min_required_mz_version, 236 ) in self.ancestor_overrides.items(): 237 if version >= min_required_mz_version: 238 continue 239 240 if git.contains_commit(commit_hash): 241 # commit would require at least min_required_mz_version 242 return commit_hash 243 244 return None 245 246 def _resolve_image_tag_of_previous_release( 247 self, context_prefix: str, previous_minor: bool 248 ) -> tuple[str, str] | None: 249 tagged_release_version = git.get_tagged_release_version(version_type=MzVersion) 250 assert tagged_release_version is not None 251 previous_release_version = get_previous_published_version( 252 tagged_release_version, previous_minor=previous_minor 253 ) 254 255 override_commit = self._get_override_commit_instead_of_version( 256 previous_release_version 257 ) 258 259 if override_commit is not None: 260 # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use 261 # use the commit instead of the previous release 262 # return ( 263 # commit_to_image_tag(override_commit), 264 # f"commit override instead of previous release ({previous_release_version})", 265 # ) 266 return None 267 268 return ( 269 release_version_to_image_tag(previous_release_version), 270 f"{context_prefix} {tagged_release_version}", 271 ) 272 273 def _resolve_image_tag_of_previous_release_from_current( 274 self, context: str 275 ) -> tuple[str, str] | None: 276 # Even though we are on main we might be in an older state, pick the 277 # latest release that was before our current version. 278 current_version = MzVersion.parse_cargo() 279 280 previous_published_version = get_previous_published_version( 281 current_version, previous_minor=True 282 ) 283 override_commit = self._get_override_commit_instead_of_version( 284 previous_published_version 285 ) 286 287 if override_commit is not None: 288 # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use 289 # use the commit instead of the latest release 290 # return ( 291 # commit_to_image_tag(override_commit), 292 # f"commit override instead of latest release ({previous_published_version})", 293 # ) 294 return None 295 296 return ( 297 release_version_to_image_tag(previous_published_version), 298 context, 299 ) 300 301 def _resolve_image_tag_of_merge_base( 302 self, 303 context_when_image_of_commit_exists: str, 304 ) -> tuple[str, str] | None: 305 # If the current PR has a known and accepted regression, don't compare 306 # against merge base of it 307 override_commit = self._get_override_commit_instead_of_version( 308 MzVersion.parse_cargo() 309 ) 310 common_ancestor_commit = buildkite.get_merge_base() 311 312 if override_commit is not None: 313 # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use 314 # return ( 315 # commit_to_image_tag(override_commit), 316 # f"commit override instead of merge base ({common_ancestor_commit})", 317 # ) 318 return None 319 320 ancestors = git.get_first_parent_commits(common_ancestor_commit, limit=20) 321 for ancestor in ancestors: 322 if image_of_commit_exists(ancestor): 323 context = ( 324 context_when_image_of_commit_exists 325 if ancestor == common_ancestor_commit 326 else f"ancestor of {context_when_image_of_commit_exists} (walked back from {common_ancestor_commit[:12]})" 327 ) 328 return ( 329 commit_to_image_tag(ancestor), 330 context, 331 ) 332 333 return None
336class AncestorImageResolutionLocal(AncestorImageResolutionBase): 337 def resolve_image_tag(self) -> tuple[str, str] | None: 338 if build_context.is_on_release_version(): 339 return self._resolve_image_tag_of_previous_release( 340 "previous minor release because on local release branch", 341 previous_minor=True, 342 ) 343 elif build_context.is_on_main_branch(): 344 return self._resolve_image_tag_of_previous_release_from_current( 345 "previous release from current because on local main branch" 346 ) 347 else: 348 return self._resolve_image_tag_of_merge_base( 349 "merge base of local non-main branch", 350 )
def
resolve_image_tag(self) -> tuple[str, str] | None:
337 def resolve_image_tag(self) -> tuple[str, str] | None: 338 if build_context.is_on_release_version(): 339 return self._resolve_image_tag_of_previous_release( 340 "previous minor release because on local release branch", 341 previous_minor=True, 342 ) 343 elif build_context.is_on_main_branch(): 344 return self._resolve_image_tag_of_previous_release_from_current( 345 "previous release from current because on local main branch" 346 ) 347 else: 348 return self._resolve_image_tag_of_merge_base( 349 "merge base of local non-main branch", 350 )
Inherited Members
353class AncestorImageResolutionInBuildkite(AncestorImageResolutionBase): 354 def resolve_image_tag(self) -> tuple[str, str] | None: 355 if buildkite.is_in_pull_request(): 356 return self._resolve_image_tag_of_merge_base( 357 "merge base of pull request", 358 ) 359 elif build_context.is_on_release_version(): 360 return self._resolve_image_tag_of_previous_release( 361 "previous minor release because on release branch", previous_minor=True 362 ) 363 else: 364 return self._resolve_image_tag_of_previous_release_from_current( 365 "previous release from current because not in a pull request and not on a release branch", 366 )
def
resolve_image_tag(self) -> tuple[str, str] | None:
354 def resolve_image_tag(self) -> tuple[str, str] | None: 355 if buildkite.is_in_pull_request(): 356 return self._resolve_image_tag_of_merge_base( 357 "merge base of pull request", 358 ) 359 elif build_context.is_on_release_version(): 360 return self._resolve_image_tag_of_previous_release( 361 "previous minor release because on release branch", previous_minor=True 362 ) 363 else: 364 return self._resolve_image_tag_of_previous_release_from_current( 365 "previous release from current because not in a pull request and not on a release branch", 366 )
Inherited Members
def
get_latest_published_version() -> materialize.mz_version.MzVersion:
369def get_latest_published_version() -> MzVersion: 370 """Get the latest mz version, older than current state, for which an image is published.""" 371 excluded_versions = set() 372 current_version = MzVersion.parse_cargo() 373 374 while True: 375 latest_published_version = git.get_latest_version( 376 version_type=MzVersion, 377 excluded_versions=excluded_versions, 378 current_version=current_version, 379 ) 380 381 if is_valid_release_image(latest_published_version): 382 return latest_published_version 383 else: 384 print( 385 f"Skipping version {latest_published_version} (image not found), trying earlier version" 386 ) 387 excluded_versions.add(latest_published_version)
Get the latest mz version, older than current state, for which an image is published.
def
get_previous_published_version( release_version: materialize.mz_version.MzVersion, previous_minor: bool) -> materialize.mz_version.MzVersion:
390def get_previous_published_version( 391 release_version: MzVersion, previous_minor: bool 392) -> MzVersion: 393 """Get the highest preceding mz version to the specified version for which an image is published.""" 394 excluded_versions = set() 395 396 while True: 397 previous_published_version = get_previous_mz_version( 398 release_version, 399 previous_minor=previous_minor, 400 excluded_versions=excluded_versions, 401 ) 402 403 if is_valid_release_image(previous_published_version): 404 return previous_published_version 405 else: 406 print(f"Skipping version {previous_published_version} (image not found)") 407 excluded_versions.add(previous_published_version)
Get the highest preceding mz version to the specified version for which an image is published.
def
get_published_minor_mz_versions( newest_first: bool = True, limit: int | None = None, include_filter: Callable[[materialize.mz_version.MzVersion], bool] | None = None, exclude_current_minor_version: bool = False, max_version: materialize.mz_version.MzVersion | None = None, include_release_candidates: bool = False) -> list[materialize.mz_version.MzVersion]:
410def get_published_minor_mz_versions( 411 newest_first: bool = True, 412 limit: int | None = None, 413 include_filter: Callable[[MzVersion], bool] | None = None, 414 exclude_current_minor_version: bool = False, 415 max_version: MzVersion | None = None, 416 include_release_candidates: bool = False, 417) -> list[MzVersion]: 418 """ 419 Get the latest patch version for every minor version. 420 Use this version if it is NOT important whether a tag was introduced before or after creating this branch. 421 422 See also: #get_minor_mz_versions_listed_in_docs() 423 """ 424 425 # sorted in descending order 426 all_versions = get_all_mz_versions( 427 newest_first=True, include_release_candidates=include_release_candidates 428 ) 429 minor_versions: dict[str, MzVersion] = {} 430 431 version = MzVersion.parse_cargo() 432 current_version = f"{version.major}.{version.minor}" 433 434 # Note that this method must not apply limit_to_published_versions to a created list 435 # because in that case minor versions may get lost. 436 for version in all_versions: 437 if include_filter is not None and not include_filter(version): 438 # this version shall not be included 439 continue 440 441 if max_version is not None and version >= max_version: 442 continue 443 444 minor_version = f"{version.major}.{version.minor}" 445 446 if exclude_current_minor_version and minor_version == current_version: 447 continue 448 449 if minor_version in minor_versions.keys(): 450 # we already have a more recent version for this minor version 451 continue 452 453 if not is_valid_release_image(version): 454 # this version is not considered valid 455 continue 456 457 minor_versions[minor_version] = version 458 459 if limit is not None and len(minor_versions.keys()) == limit: 460 # collected enough versions 461 break 462 463 assert len(minor_versions) > 0 464 return sorted(minor_versions.values(), reverse=newest_first)
Get the latest patch version for every minor version. Use this version if it is NOT important whether a tag was introduced before or after creating this branch.
See also: #get_minor_mz_versions_listed_in_docs()
def
get_all_mz_versions( newest_first: bool = True, include_release_candidates: bool = False) -> list[materialize.mz_version.MzVersion]:
467def get_all_mz_versions( 468 newest_first: bool = True, 469 include_release_candidates: bool = False, 470) -> list[MzVersion]: 471 """ 472 Get all mz versions based on git tags. Versions known to be invalid are excluded. 473 474 See also: #get_all_mz_versions_listed_in_docs 475 """ 476 return [ 477 version 478 for version in get_version_tags( 479 version_type=MzVersion, newest_first=newest_first 480 ) 481 if version not in INVALID_VERSIONS 482 and (include_release_candidates or not version.prerelease) 483 ]
Get all mz versions based on git tags. Versions known to be invalid are excluded.
See also: #get_all_mz_versions_listed_in_docs
def
get_all_published_mz_versions( newest_first: bool = True, limit: int | None = None) -> list[materialize.mz_version.MzVersion]:
486def get_all_published_mz_versions( 487 newest_first: bool = True, limit: int | None = None 488) -> list[MzVersion]: 489 """Get all mz versions based on git tags. This method ensures that images of the versions exist.""" 490 all_versions = get_all_mz_versions(newest_first=newest_first) 491 print(f"all_versions: {all_versions}") 492 return limit_to_published_versions(all_versions, limit)
Get all mz versions based on git tags. This method ensures that images of the versions exist.
def
get_published_mz_versions_within_one_major_version(newest_first: bool = True) -> list[materialize.mz_version.MzVersion]:
495def get_published_mz_versions_within_one_major_version( 496 newest_first: bool = True, 497) -> list[MzVersion]: 498 """Get all previous mz versions within one major version of the current version. Ensure that images of the versions exist.""" 499 current_version = MzVersion.parse_cargo() 500 all_versions = get_all_mz_versions(newest_first=newest_first) 501 versions_within_one_major_version = { 502 v 503 for v in all_versions 504 if abs(v.major - current_version.major) <= 1 and v <= current_version 505 } 506 507 return limit_to_published_versions(list(versions_within_one_major_version))
Get all previous mz versions within one major version of the current version. Ensure that images of the versions exist.
def
limit_to_published_versions( all_versions: list[materialize.mz_version.MzVersion], limit: int | None = None) -> list[materialize.mz_version.MzVersion]:
510def limit_to_published_versions( 511 all_versions: list[MzVersion], limit: int | None = None 512) -> list[MzVersion]: 513 """Remove versions for which no image is published.""" 514 versions = [] 515 516 for v in all_versions: 517 if is_valid_release_image(v): 518 versions.append(v) 519 520 if limit is not None and len(versions) == limit: 521 break 522 523 return versions
Remove versions for which no image is published.
def
get_previous_mz_version( version: materialize.mz_version.MzVersion, previous_minor: bool, excluded_versions: set[materialize.mz_version.MzVersion] | None = None) -> materialize.mz_version.MzVersion:
526def get_previous_mz_version( 527 version: MzVersion, 528 previous_minor: bool, 529 excluded_versions: set[MzVersion] | None = None, 530) -> MzVersion: 531 """Get the predecessor of the specified version based on git tags.""" 532 if excluded_versions is None: 533 excluded_versions = set() 534 535 if previous_minor: 536 version = MzVersion.create(version.major, version.minor, 0) 537 538 if version.prerelease is not None and len(version.prerelease) > 0: 539 # simply drop the prerelease, do not try to find a decremented version 540 found_version = MzVersion.create(version.major, version.minor, version.patch) 541 542 if found_version not in excluded_versions: 543 return found_version 544 else: 545 # start searching with this version 546 version = found_version 547 548 all_versions: list[MzVersion] = get_version_tags(version_type=type(version)) 549 all_suitable_previous_versions = [ 550 v 551 for v in all_versions 552 if v < version 553 and (v.prerelease is None or len(v.prerelease) == 0) 554 and v not in INVALID_VERSIONS 555 and v not in excluded_versions 556 ] 557 return max(all_suitable_previous_versions)
Get the predecessor of the specified version based on git tags.
def
is_valid_release_image(version: materialize.mz_version.MzVersion) -> bool:
560def is_valid_release_image(version: MzVersion) -> bool: 561 """ 562 Checks if a version is not known as an invalid version and has a published image. 563 Note that this method may take shortcuts on older versions. 564 """ 565 if version in INVALID_VERSIONS: 566 return False 567 568 if version < _SKIP_IMAGE_CHECK_BELOW_THIS_VERSION: 569 # optimization: assume that all versions older than this one are either valid or listed in INVALID_VERSIONS 570 return True 571 572 # This is a potentially expensive operation which pulls an image if it hasn't been pulled yet. 573 return docker.image_of_release_version_exists(version)
Checks if a version is not known as an invalid version and has a published image. Note that this method may take shortcuts on older versions.
def
get_commits_of_accepted_regressions_between_versions( ancestor_overrides: dict[str, materialize.mz_version.MzVersion], since_version_exclusive: materialize.mz_version.MzVersion, to_version_inclusive: materialize.mz_version.MzVersion) -> list[str]:
576def get_commits_of_accepted_regressions_between_versions( 577 ancestor_overrides: dict[str, MzVersion], 578 since_version_exclusive: MzVersion, 579 to_version_inclusive: MzVersion, 580) -> list[str]: 581 """ 582 Get commits of accepted regressions between both versions. 583 :param ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS 584 :return: commits 585 """ 586 587 assert since_version_exclusive <= to_version_inclusive 588 589 commits = [] 590 591 for ( 592 regression_introducing_commit, 593 first_version_with_regression, 594 ) in ancestor_overrides.items(): 595 if ( 596 since_version_exclusive 597 < first_version_with_regression 598 <= to_version_inclusive 599 ): 600 commits.append(regression_introducing_commit) 601 602 return commits
Get commits of accepted regressions between both versions.
Parameters
- ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS
Returns
commits
class
VersionsFromDocs:
605class VersionsFromDocs: 606 """Materialize versions as listed in doc/user/content/releases 607 608 >>> len(VersionsFromDocs(respect_released_tag=True).all_versions()) > 0 609 True 610 611 >>> len(VersionsFromDocs(respect_released_tag=True).minor_versions()) > 0 612 True 613 614 >>> len(VersionsFromDocs(respect_released_tag=True).patch_versions(minor_version=MzVersion.parse_mz("v0.52.0"))) 615 4 616 617 >>> min(VersionsFromDocs(respect_released_tag=True).all_versions()) 618 MzVersion(major=0, minor=27, patch=0, prerelease=None, build=None) 619 """ 620 621 def __init__( 622 self, 623 respect_released_tag: bool, 624 respect_date: bool = False, 625 only_publish_helm_chart: bool = True, 626 skip_rc: bool = False, 627 ) -> None: 628 files = Path(MZ_ROOT / "doc" / "user" / "content" / "releases").glob("v*.md") 629 self.versions = [] 630 current_version = MzVersion.parse_cargo() 631 for f in files: 632 base = f.stem 633 metadata = frontmatter.load(f) 634 if respect_released_tag and not metadata.get("released", False): 635 continue 636 if only_publish_helm_chart and not metadata.get("publish_helm_chart", True): 637 continue 638 date: datetime.date = metadata["date"] 639 if respect_date and date > datetime.date.today(): 640 continue 641 642 current_patch = metadata.get("patch", 0) 643 current_rc = metadata.get("rc", 0) 644 645 if current_rc > 0: 646 if skip_rc: 647 continue 648 for rc in range(1, current_rc + 1): 649 version = MzVersion.parse_mz(f"{base}.{current_patch}-rc.{rc}") 650 if not respect_released_tag and version >= current_version: 651 continue 652 if version not in INVALID_VERSIONS: 653 self.versions.append(version) 654 else: 655 for patch in range(current_patch + 1): 656 version = MzVersion.parse_mz(f"{base}.{patch}") 657 if not respect_released_tag and version >= current_version: 658 continue 659 if version not in INVALID_VERSIONS: 660 self.versions.append(version) 661 662 assert len(self.versions) > 0 663 self.versions.sort() 664 665 def all_versions(self) -> list[MzVersion]: 666 return self.versions 667 668 def minor_versions(self) -> list[MzVersion]: 669 """Return the latest patch version for every minor version.""" 670 minor_versions = {} 671 for version in self.versions: 672 minor_versions[f"{version.major}.{version.minor}"] = version 673 674 assert len(minor_versions) > 0 675 return sorted(minor_versions.values()) 676 677 def patch_versions(self, minor_version: MzVersion) -> list[MzVersion]: 678 """Return all patch versions within the given minor version.""" 679 patch_versions = [] 680 for version in self.versions: 681 if ( 682 version.major == minor_version.major 683 and version.minor == minor_version.minor 684 ): 685 patch_versions.append(version) 686 687 assert len(patch_versions) > 0 688 return sorted(patch_versions)
Materialize versions as listed in doc/user/content/releases
>>> len(VersionsFromDocs(respect_released_tag=True).all_versions()) > 0
True
>>> len(VersionsFromDocs(respect_released_tag=True).minor_versions()) > 0
True
>>> len(VersionsFromDocs(respect_released_tag=True).patch_versions(minor_version=MzVersion.parse_mz("v0.52.0")))
4
>>> min(VersionsFromDocs(respect_released_tag=True).all_versions())
MzVersion(major=0, minor=27, patch=0, prerelease=None, build=None)
VersionsFromDocs( respect_released_tag: bool, respect_date: bool = False, only_publish_helm_chart: bool = True, skip_rc: bool = False)
621 def __init__( 622 self, 623 respect_released_tag: bool, 624 respect_date: bool = False, 625 only_publish_helm_chart: bool = True, 626 skip_rc: bool = False, 627 ) -> None: 628 files = Path(MZ_ROOT / "doc" / "user" / "content" / "releases").glob("v*.md") 629 self.versions = [] 630 current_version = MzVersion.parse_cargo() 631 for f in files: 632 base = f.stem 633 metadata = frontmatter.load(f) 634 if respect_released_tag and not metadata.get("released", False): 635 continue 636 if only_publish_helm_chart and not metadata.get("publish_helm_chart", True): 637 continue 638 date: datetime.date = metadata["date"] 639 if respect_date and date > datetime.date.today(): 640 continue 641 642 current_patch = metadata.get("patch", 0) 643 current_rc = metadata.get("rc", 0) 644 645 if current_rc > 0: 646 if skip_rc: 647 continue 648 for rc in range(1, current_rc + 1): 649 version = MzVersion.parse_mz(f"{base}.{current_patch}-rc.{rc}") 650 if not respect_released_tag and version >= current_version: 651 continue 652 if version not in INVALID_VERSIONS: 653 self.versions.append(version) 654 else: 655 for patch in range(current_patch + 1): 656 version = MzVersion.parse_mz(f"{base}.{patch}") 657 if not respect_released_tag and version >= current_version: 658 continue 659 if version not in INVALID_VERSIONS: 660 self.versions.append(version) 661 662 assert len(self.versions) > 0 663 self.versions.sort()
def
minor_versions(self) -> list[materialize.mz_version.MzVersion]:
668 def minor_versions(self) -> list[MzVersion]: 669 """Return the latest patch version for every minor version.""" 670 minor_versions = {} 671 for version in self.versions: 672 minor_versions[f"{version.major}.{version.minor}"] = version 673 674 assert len(minor_versions) > 0 675 return sorted(minor_versions.values())
Return the latest patch version for every minor version.
def
patch_versions( self, minor_version: materialize.mz_version.MzVersion) -> list[materialize.mz_version.MzVersion]:
677 def patch_versions(self, minor_version: MzVersion) -> list[MzVersion]: 678 """Return all patch versions within the given minor version.""" 679 patch_versions = [] 680 for version in self.versions: 681 if ( 682 version.major == minor_version.major 683 and version.minor == minor_version.minor 684 ): 685 patch_versions.append(version) 686 687 assert len(patch_versions) > 0 688 return sorted(patch_versions)
Return all patch versions within the given minor version.