misc.python.materialize.version_list
1# Copyright Materialize, Inc. and contributors. All rights reserved. 2# 3# Use of this software is governed by the Business Source License 4# included in the LICENSE file at the root of this repository. 5# 6# As of the Change Date specified in that file, in accordance with 7# the Business Source License, use of this software will be governed 8# by the Apache License, Version 2.0. 9 10 11from __future__ import annotations 12 13import datetime 14import os 15from collections.abc import Callable 16from dataclasses import dataclass 17from pathlib import Path 18 19import frontmatter 20import requests 21import yaml 22 23from materialize import build_context, buildkite, docker, git 24from materialize.docker import ( 25 commit_to_image_tag, 26 image_of_commit_exists, 27 release_version_to_image_tag, 28) 29from materialize.git import get_version_tags 30from materialize.mz_version import MzVersion 31 32MZ_ROOT = Path(os.environ["MZ_ROOT"]) 33 34 35@dataclass 36class SelfManagedVersion: 37 helm_version: MzVersion 38 version: MzVersion 39 40 41def fetch_self_managed_versions() -> list[SelfManagedVersion]: 42 result: list[SelfManagedVersion] = [] 43 for entry in yaml.safe_load( 44 requests.get("https://materializeinc.github.io/materialize/index.yaml").text 45 )["entries"]["materialize-operator"]: 46 self_managed_version = SelfManagedVersion( 47 MzVersion.parse_mz(entry["version"]), 48 MzVersion.parse_mz(entry["appVersion"]), 49 ) 50 if ( 51 not self_managed_version.version.prerelease 52 and self_managed_version.version not in BAD_SELF_MANAGED_VERSIONS 53 ): 54 result.append(self_managed_version) 55 return result 56 57 58def get_all_self_managed_versions() -> list[MzVersion]: 59 return sorted([version.version for version in fetch_self_managed_versions()]) 60 61 62def get_self_managed_versions() -> list[MzVersion]: 63 prefixes = set() 64 result = set() 65 self_managed_versions = fetch_self_managed_versions() 66 for version_info in self_managed_versions: 67 prefix = (version_info.version.major, version_info.version.minor) 68 if ( 69 not version_info.version.prerelease 70 and prefix not in prefixes 71 and not version_info.helm_version.prerelease 72 ): 73 result.add(version_info.version) 74 prefixes.add(prefix) 75 return sorted(result) 76 77 78# Gets the range of versions we can "upgrade from" to the current version, sorted in ascending order. 79def get_compatible_upgrade_from_versions() -> list[MzVersion]: 80 81 # Determine the current MzVersion from the environment, or from a version constant 82 current_version = MzVersion.parse_cargo() 83 84 published_versions_within_one_major_version = { 85 v 86 for v in get_published_mz_versions_within_one_major_version() 87 if abs(v.major - current_version.major) <= 1 and v <= current_version 88 } 89 90 if current_version.major <= 26: 91 # For versions <= 26, we can only upgrade from 25.2 self-managed versions 92 self_managed_25_2_versions = { 93 v.version 94 for v in fetch_self_managed_versions() 95 if v.helm_version.major == 25 and v.helm_version.minor == 2 96 } 97 98 return sorted( 99 self_managed_25_2_versions.union( 100 published_versions_within_one_major_version 101 ) 102 ) 103 else: 104 # For versions > 26, get all mz versions within 1 major version of current_version 105 return sorted(published_versions_within_one_major_version) 106 107 108BAD_SELF_MANAGED_VERSIONS = { 109 MzVersion.parse_mz("v0.130.0"), 110 MzVersion.parse_mz("v0.130.1"), 111 MzVersion.parse_mz("v0.130.2"), 112 MzVersion.parse_mz("v0.130.3"), 113 MzVersion.parse_mz("v0.130.4"), 114 MzVersion.parse_mz( 115 "v0.147.7" 116 ), # Incompatible for upgrades because it clears login attribute for roles due to catalog migration 117 MzVersion.parse_mz( 118 "v0.147.14" 119 ), # Incompatible for upgrades because it clears login attribute for roles due to catalog migration 120 MzVersion.parse_mz("v0.157.0"), 121} 122 123# not released on Docker 124INVALID_VERSIONS = { 125 MzVersion.parse_mz("v0.52.1"), 126 MzVersion.parse_mz("v0.55.1"), 127 MzVersion.parse_mz("v0.55.2"), 128 MzVersion.parse_mz("v0.55.3"), 129 MzVersion.parse_mz("v0.55.4"), 130 MzVersion.parse_mz("v0.55.5"), 131 MzVersion.parse_mz("v0.55.6"), 132 MzVersion.parse_mz("v0.56.0"), 133 MzVersion.parse_mz("v0.57.1"), 134 MzVersion.parse_mz("v0.57.2"), 135 MzVersion.parse_mz("v0.57.5"), 136 MzVersion.parse_mz("v0.57.6"), 137 MzVersion.parse_mz("v0.81.0"), # incompatible for upgrades 138 MzVersion.parse_mz("v0.81.1"), # incompatible for upgrades 139 MzVersion.parse_mz("v0.81.2"), # incompatible for upgrades 140 MzVersion.parse_mz("v0.89.7"), 141 MzVersion.parse_mz("v0.92.0"), # incompatible for upgrades 142 MzVersion.parse_mz("v0.93.0"), # accidental release 143 MzVersion.parse_mz("v0.99.1"), # incompatible for upgrades 144 MzVersion.parse_mz("v0.113.1"), # incompatible for upgrades 145} 146 147_SKIP_IMAGE_CHECK_BELOW_THIS_VERSION = MzVersion.parse_mz("v0.77.0") 148 149 150def resolve_ancestor_image_tag(ancestor_overrides: dict[str, MzVersion]) -> str | None: 151 """ 152 Resolve the ancestor image tag. 153 :param ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS 154 :return: image of the ancestor 155 """ 156 157 manual_ancestor_override = os.getenv("COMMON_ANCESTOR_OVERRIDE") 158 if manual_ancestor_override is not None: 159 image_tag = _manual_ancestor_specification_to_image_tag( 160 manual_ancestor_override 161 ) 162 print( 163 f"Using specified {image_tag} as image tag for ancestor (context: specified in $COMMON_ANCESTOR_OVERRIDE)" 164 ) 165 return image_tag 166 167 ancestor_image_resolution = _create_ancestor_image_resolution(ancestor_overrides) 168 result = ancestor_image_resolution.resolve_image_tag() 169 if result is None: 170 return None 171 image_tag, context = result 172 print(f"Using {image_tag} as image tag for ancestor (context: {context})") 173 return image_tag 174 175 176def _create_ancestor_image_resolution( 177 ancestor_overrides: dict[str, MzVersion], 178) -> AncestorImageResolutionBase: 179 if buildkite.is_in_buildkite(): 180 return AncestorImageResolutionInBuildkite(ancestor_overrides) 181 else: 182 return AncestorImageResolutionLocal(ancestor_overrides) 183 184 185def _manual_ancestor_specification_to_image_tag(ancestor_spec: str) -> str: 186 if MzVersion.is_valid_version_string(ancestor_spec): 187 return release_version_to_image_tag(MzVersion.parse_mz(ancestor_spec)) 188 else: 189 return commit_to_image_tag(ancestor_spec) 190 191 192class AncestorImageResolutionBase: 193 def __init__(self, ancestor_overrides: dict[str, MzVersion]): 194 self.ancestor_overrides = ancestor_overrides 195 196 def resolve_image_tag(self) -> tuple[str, str]: 197 raise NotImplementedError 198 199 def _get_override_commit_instead_of_version( 200 self, 201 version: MzVersion, 202 ) -> str | None: 203 """ 204 If a commit specifies a mz version as prerequisite (to avoid regressions) that is newer than the provided 205 version (i.e., prerequisite not satisfied by the latest version), then return that commit's hash if the commit 206 contained in the current state. 207 Otherwise, return none. 208 """ 209 for ( 210 commit_hash, 211 min_required_mz_version, 212 ) in self.ancestor_overrides.items(): 213 if version >= min_required_mz_version: 214 continue 215 216 if git.contains_commit(commit_hash): 217 # commit would require at least min_required_mz_version 218 return commit_hash 219 220 return None 221 222 def _resolve_image_tag_of_previous_release( 223 self, context_prefix: str, previous_minor: bool 224 ) -> tuple[str, str] | None: 225 tagged_release_version = git.get_tagged_release_version(version_type=MzVersion) 226 assert tagged_release_version is not None 227 previous_release_version = get_previous_published_version( 228 tagged_release_version, previous_minor=previous_minor 229 ) 230 231 override_commit = self._get_override_commit_instead_of_version( 232 previous_release_version 233 ) 234 235 if override_commit is not None: 236 # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use 237 # use the commit instead of the previous release 238 # return ( 239 # commit_to_image_tag(override_commit), 240 # f"commit override instead of previous release ({previous_release_version})", 241 # ) 242 return None 243 244 return ( 245 release_version_to_image_tag(previous_release_version), 246 f"{context_prefix} {tagged_release_version}", 247 ) 248 249 def _resolve_image_tag_of_previous_release_from_current( 250 self, context: str 251 ) -> tuple[str, str] | None: 252 # Even though we are on main we might be in an older state, pick the 253 # latest release that was before our current version. 254 current_version = MzVersion.parse_cargo() 255 256 previous_published_version = get_previous_published_version( 257 current_version, previous_minor=True 258 ) 259 override_commit = self._get_override_commit_instead_of_version( 260 previous_published_version 261 ) 262 263 if override_commit is not None: 264 # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use 265 # use the commit instead of the latest release 266 # return ( 267 # commit_to_image_tag(override_commit), 268 # f"commit override instead of latest release ({previous_published_version})", 269 # ) 270 return None 271 272 return ( 273 release_version_to_image_tag(previous_published_version), 274 context, 275 ) 276 277 def _resolve_image_tag_of_merge_base( 278 self, 279 context_when_image_of_commit_exists: str, 280 context_when_falling_back_to_latest: str, 281 ) -> tuple[str, str] | None: 282 # If the current PR has a known and accepted regression, don't compare 283 # against merge base of it 284 override_commit = self._get_override_commit_instead_of_version( 285 MzVersion.parse_cargo() 286 ) 287 common_ancestor_commit = buildkite.get_merge_base() 288 289 if override_commit is not None: 290 # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use 291 # return ( 292 # commit_to_image_tag(override_commit), 293 # f"commit override instead of merge base ({common_ancestor_commit})", 294 # ) 295 return None 296 297 if image_of_commit_exists(common_ancestor_commit): 298 return ( 299 commit_to_image_tag(common_ancestor_commit), 300 context_when_image_of_commit_exists, 301 ) 302 else: 303 return ( 304 release_version_to_image_tag(get_latest_published_version()), 305 context_when_falling_back_to_latest, 306 ) 307 308 309class AncestorImageResolutionLocal(AncestorImageResolutionBase): 310 def resolve_image_tag(self) -> tuple[str, str] | None: 311 if build_context.is_on_release_version(): 312 return self._resolve_image_tag_of_previous_release( 313 "previous minor release because on local release branch", 314 previous_minor=True, 315 ) 316 elif build_context.is_on_main_branch(): 317 return self._resolve_image_tag_of_previous_release_from_current( 318 "previous release from current because on local main branch" 319 ) 320 else: 321 return self._resolve_image_tag_of_merge_base( 322 "merge base of local non-main branch", 323 "latest release because image of merge base of local non-main branch not available", 324 ) 325 326 327class AncestorImageResolutionInBuildkite(AncestorImageResolutionBase): 328 def resolve_image_tag(self) -> tuple[str, str] | None: 329 if buildkite.is_in_pull_request(): 330 return self._resolve_image_tag_of_merge_base( 331 "merge base of pull request", 332 "latest release because image of merge base of pull request not available", 333 ) 334 elif build_context.is_on_release_version(): 335 return self._resolve_image_tag_of_previous_release( 336 "previous minor release because on release branch", previous_minor=True 337 ) 338 else: 339 return self._resolve_image_tag_of_previous_release_from_current( 340 "previous release from current because not in a pull request and not on a release branch", 341 ) 342 343 344def get_latest_published_version() -> MzVersion: 345 """Get the latest mz version, older than current state, for which an image is published.""" 346 excluded_versions = set() 347 current_version = MzVersion.parse_cargo() 348 349 while True: 350 latest_published_version = git.get_latest_version( 351 version_type=MzVersion, 352 excluded_versions=excluded_versions, 353 current_version=current_version, 354 ) 355 356 if is_valid_release_image(latest_published_version): 357 return latest_published_version 358 else: 359 print( 360 f"Skipping version {latest_published_version} (image not found), trying earlier version" 361 ) 362 excluded_versions.add(latest_published_version) 363 364 365def get_previous_published_version( 366 release_version: MzVersion, previous_minor: bool 367) -> MzVersion: 368 """Get the highest preceding mz version to the specified version for which an image is published.""" 369 excluded_versions = set() 370 371 while True: 372 previous_published_version = get_previous_mz_version( 373 release_version, 374 previous_minor=previous_minor, 375 excluded_versions=excluded_versions, 376 ) 377 378 if is_valid_release_image(previous_published_version): 379 return previous_published_version 380 else: 381 print(f"Skipping version {previous_published_version} (image not found)") 382 excluded_versions.add(previous_published_version) 383 384 385def get_published_minor_mz_versions( 386 newest_first: bool = True, 387 limit: int | None = None, 388 include_filter: Callable[[MzVersion], bool] | None = None, 389 exclude_current_minor_version: bool = False, 390) -> list[MzVersion]: 391 """ 392 Get the latest patch version for every minor version. 393 Use this version if it is NOT important whether a tag was introduced before or after creating this branch. 394 395 See also: #get_minor_mz_versions_listed_in_docs() 396 """ 397 398 # sorted in descending order 399 all_versions = get_all_mz_versions(newest_first=True) 400 minor_versions: dict[str, MzVersion] = {} 401 402 version = MzVersion.parse_cargo() 403 current_version = f"{version.major}.{version.minor}" 404 405 # Note that this method must not apply limit_to_published_versions to a created list 406 # because in that case minor versions may get lost. 407 for version in all_versions: 408 if include_filter is not None and not include_filter(version): 409 # this version shall not be included 410 continue 411 412 minor_version = f"{version.major}.{version.minor}" 413 414 if exclude_current_minor_version and minor_version == current_version: 415 continue 416 417 if minor_version in minor_versions.keys(): 418 # we already have a more recent version for this minor version 419 continue 420 421 if not is_valid_release_image(version): 422 # this version is not considered valid 423 continue 424 425 minor_versions[minor_version] = version 426 427 if limit is not None and len(minor_versions.keys()) == limit: 428 # collected enough versions 429 break 430 431 assert len(minor_versions) > 0 432 return sorted(minor_versions.values(), reverse=newest_first) 433 434 435def get_minor_mz_versions_listed_in_docs(respect_released_tag: bool) -> list[MzVersion]: 436 """ 437 Get the latest patch version for every minor version in ascending order. 438 Use this version if it is important whether a tag was introduced before or after creating this branch. 439 440 See also: #get_published_minor_mz_versions() 441 """ 442 return VersionsFromDocs(respect_released_tag).minor_versions() 443 444 445def get_all_mz_versions( 446 newest_first: bool = True, 447) -> list[MzVersion]: 448 """ 449 Get all mz versions based on git tags. Versions known to be invalid are excluded. 450 451 See also: #get_all_mz_versions_listed_in_docs 452 """ 453 return [ 454 version 455 for version in get_version_tags( 456 version_type=MzVersion, newest_first=newest_first 457 ) 458 if version not in INVALID_VERSIONS 459 # Exclude release candidates 460 and not version.prerelease 461 ] 462 463 464def get_all_mz_versions_listed_in_docs( 465 respect_released_tag: bool, 466) -> list[MzVersion]: 467 """ 468 Get all mz versions based on docs. Versions known to be invalid are excluded. 469 470 See also: #get_all_mz_versions() 471 """ 472 return VersionsFromDocs(respect_released_tag).all_versions() 473 474 475def get_all_published_mz_versions( 476 newest_first: bool = True, limit: int | None = None 477) -> list[MzVersion]: 478 """Get all mz versions based on git tags. This method ensures that images of the versions exist.""" 479 all_versions = get_all_mz_versions(newest_first=newest_first) 480 print(f"all_versions: {all_versions}") 481 return limit_to_published_versions(all_versions, limit) 482 483 484def get_published_mz_versions_within_one_major_version( 485 newest_first: bool = True, 486) -> list[MzVersion]: 487 """Get all previous mz versions within one major version of the current version. Ensure that images of the versions exist.""" 488 current_version = MzVersion.parse_cargo() 489 all_versions = get_all_mz_versions(newest_first=newest_first) 490 versions_within_one_major_version = { 491 v 492 for v in all_versions 493 if abs(v.major - current_version.major) <= 1 and v <= current_version 494 } 495 496 return limit_to_published_versions(list(versions_within_one_major_version)) 497 498 499def limit_to_published_versions( 500 all_versions: list[MzVersion], limit: int | None = None 501) -> list[MzVersion]: 502 """Remove versions for which no image is published.""" 503 versions = [] 504 505 for v in all_versions: 506 if is_valid_release_image(v): 507 versions.append(v) 508 509 if limit is not None and len(versions) == limit: 510 break 511 512 return versions 513 514 515def get_previous_mz_version( 516 version: MzVersion, 517 previous_minor: bool, 518 excluded_versions: set[MzVersion] | None = None, 519) -> MzVersion: 520 """Get the predecessor of the specified version based on git tags.""" 521 if excluded_versions is None: 522 excluded_versions = set() 523 524 if previous_minor: 525 version = MzVersion.create(version.major, version.minor, 0) 526 527 if version.prerelease is not None and len(version.prerelease) > 0: 528 # simply drop the prerelease, do not try to find a decremented version 529 found_version = MzVersion.create(version.major, version.minor, version.patch) 530 531 if found_version not in excluded_versions: 532 return found_version 533 else: 534 # start searching with this version 535 version = found_version 536 537 all_versions: list[MzVersion] = get_version_tags(version_type=type(version)) 538 all_suitable_previous_versions = [ 539 v 540 for v in all_versions 541 if v < version 542 and (v.prerelease is None or len(v.prerelease) == 0) 543 and v not in INVALID_VERSIONS 544 and v not in excluded_versions 545 ] 546 return max(all_suitable_previous_versions) 547 548 549def is_valid_release_image(version: MzVersion) -> bool: 550 """ 551 Checks if a version is not known as an invalid version and has a published image. 552 Note that this method may take shortcuts on older versions. 553 """ 554 if version in INVALID_VERSIONS: 555 return False 556 557 if version < _SKIP_IMAGE_CHECK_BELOW_THIS_VERSION: 558 # optimization: assume that all versions older than this one are either valid or listed in INVALID_VERSIONS 559 return True 560 561 # This is a potentially expensive operation which pulls an image if it hasn't been pulled yet. 562 return docker.image_of_release_version_exists(version) 563 564 565def get_commits_of_accepted_regressions_between_versions( 566 ancestor_overrides: dict[str, MzVersion], 567 since_version_exclusive: MzVersion, 568 to_version_inclusive: MzVersion, 569) -> list[str]: 570 """ 571 Get commits of accepted regressions between both versions. 572 :param ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS 573 :return: commits 574 """ 575 576 assert since_version_exclusive <= to_version_inclusive 577 578 commits = [] 579 580 for ( 581 regression_introducing_commit, 582 first_version_with_regression, 583 ) in ancestor_overrides.items(): 584 if ( 585 since_version_exclusive 586 < first_version_with_regression 587 <= to_version_inclusive 588 ): 589 commits.append(regression_introducing_commit) 590 591 return commits 592 593 594class VersionsFromDocs: 595 """Materialize versions as listed in doc/user/content/releases 596 597 >>> len(VersionsFromDocs(respect_released_tag=True).all_versions()) > 0 598 True 599 600 >>> len(VersionsFromDocs(respect_released_tag=True).minor_versions()) > 0 601 True 602 603 >>> len(VersionsFromDocs(respect_released_tag=True).patch_versions(minor_version=MzVersion.parse_mz("v0.52.0"))) 604 4 605 606 >>> min(VersionsFromDocs(respect_released_tag=True).all_versions()) 607 MzVersion(major=0, minor=27, patch=0, prerelease=None, build=None) 608 """ 609 610 def __init__( 611 self, 612 respect_released_tag: bool, 613 respect_date: bool = False, 614 only_publish_helm_chart: bool = True, 615 skip_rc: bool = False, 616 ) -> None: 617 files = Path(MZ_ROOT / "doc" / "user" / "content" / "releases").glob("v*.md") 618 self.versions = [] 619 current_version = MzVersion.parse_cargo() 620 for f in files: 621 base = f.stem 622 metadata = frontmatter.load(f) 623 if respect_released_tag and not metadata.get("released", False): 624 continue 625 if only_publish_helm_chart and not metadata.get("publish_helm_chart", True): 626 continue 627 date: datetime.date = metadata["date"] 628 if respect_date and date > datetime.date.today(): 629 continue 630 631 current_patch = metadata.get("patch", 0) 632 current_rc = metadata.get("rc", 0) 633 634 if current_rc > 0: 635 if skip_rc: 636 continue 637 for rc in range(1, current_rc + 1): 638 version = MzVersion.parse_mz(f"{base}.{current_patch}-rc.{rc}") 639 if not respect_released_tag and version >= current_version: 640 continue 641 if version not in INVALID_VERSIONS: 642 self.versions.append(version) 643 else: 644 for patch in range(current_patch + 1): 645 version = MzVersion.parse_mz(f"{base}.{patch}") 646 if not respect_released_tag and version >= current_version: 647 continue 648 if version not in INVALID_VERSIONS: 649 self.versions.append(version) 650 651 assert len(self.versions) > 0 652 self.versions.sort() 653 654 def all_versions(self) -> list[MzVersion]: 655 return self.versions 656 657 def minor_versions(self) -> list[MzVersion]: 658 """Return the latest patch version for every minor version.""" 659 minor_versions = {} 660 for version in self.versions: 661 minor_versions[f"{version.major}.{version.minor}"] = version 662 663 assert len(minor_versions) > 0 664 return sorted(minor_versions.values()) 665 666 def patch_versions(self, minor_version: MzVersion) -> list[MzVersion]: 667 """Return all patch versions within the given minor version.""" 668 patch_versions = [] 669 for version in self.versions: 670 if ( 671 version.major == minor_version.major 672 and version.minor == minor_version.minor 673 ): 674 patch_versions.append(version) 675 676 assert len(patch_versions) > 0 677 return sorted(patch_versions)
MZ_ROOT =
PosixPath('/var/lib/buildkite-agent/builds/buildkite-15f2293-i-005d40206ba5565b4-1/materialize/deploy')
@dataclass
class
SelfManagedVersion:
42def fetch_self_managed_versions() -> list[SelfManagedVersion]: 43 result: list[SelfManagedVersion] = [] 44 for entry in yaml.safe_load( 45 requests.get("https://materializeinc.github.io/materialize/index.yaml").text 46 )["entries"]["materialize-operator"]: 47 self_managed_version = SelfManagedVersion( 48 MzVersion.parse_mz(entry["version"]), 49 MzVersion.parse_mz(entry["appVersion"]), 50 ) 51 if ( 52 not self_managed_version.version.prerelease 53 and self_managed_version.version not in BAD_SELF_MANAGED_VERSIONS 54 ): 55 result.append(self_managed_version) 56 return result
def
get_all_self_managed_versions() -> list[materialize.mz_version.MzVersion]:
def
get_self_managed_versions() -> list[materialize.mz_version.MzVersion]:
63def get_self_managed_versions() -> list[MzVersion]: 64 prefixes = set() 65 result = set() 66 self_managed_versions = fetch_self_managed_versions() 67 for version_info in self_managed_versions: 68 prefix = (version_info.version.major, version_info.version.minor) 69 if ( 70 not version_info.version.prerelease 71 and prefix not in prefixes 72 and not version_info.helm_version.prerelease 73 ): 74 result.add(version_info.version) 75 prefixes.add(prefix) 76 return sorted(result)
def
get_compatible_upgrade_from_versions() -> list[materialize.mz_version.MzVersion]:
80def get_compatible_upgrade_from_versions() -> list[MzVersion]: 81 82 # Determine the current MzVersion from the environment, or from a version constant 83 current_version = MzVersion.parse_cargo() 84 85 published_versions_within_one_major_version = { 86 v 87 for v in get_published_mz_versions_within_one_major_version() 88 if abs(v.major - current_version.major) <= 1 and v <= current_version 89 } 90 91 if current_version.major <= 26: 92 # For versions <= 26, we can only upgrade from 25.2 self-managed versions 93 self_managed_25_2_versions = { 94 v.version 95 for v in fetch_self_managed_versions() 96 if v.helm_version.major == 25 and v.helm_version.minor == 2 97 } 98 99 return sorted( 100 self_managed_25_2_versions.union( 101 published_versions_within_one_major_version 102 ) 103 ) 104 else: 105 # For versions > 26, get all mz versions within 1 major version of current_version 106 return sorted(published_versions_within_one_major_version)
BAD_SELF_MANAGED_VERSIONS =
{MzVersion(major=0, minor=130, patch=4, prerelease=None, build=None), MzVersion(major=0, minor=130, patch=0, prerelease=None, build=None), MzVersion(major=0, minor=147, patch=14, prerelease=None, build=None), MzVersion(major=0, minor=130, patch=1, prerelease=None, build=None), MzVersion(major=0, minor=147, patch=7, prerelease=None, build=None), MzVersion(major=0, minor=157, patch=0, prerelease=None, build=None), MzVersion(major=0, minor=130, patch=2, prerelease=None, build=None), MzVersion(major=0, minor=130, patch=3, prerelease=None, build=None)}
INVALID_VERSIONS =
{MzVersion(major=0, minor=55, patch=5, prerelease=None, build=None), MzVersion(major=0, minor=93, patch=0, prerelease=None, build=None), MzVersion(major=0, minor=57, patch=5, prerelease=None, build=None), MzVersion(major=0, minor=89, patch=7, prerelease=None, build=None), MzVersion(major=0, minor=113, patch=1, prerelease=None, build=None), MzVersion(major=0, minor=55, patch=2, prerelease=None, build=None), MzVersion(major=0, minor=55, patch=4, prerelease=None, build=None), MzVersion(major=0, minor=56, patch=0, prerelease=None, build=None), MzVersion(major=0, minor=52, patch=1, prerelease=None, build=None), MzVersion(major=0, minor=92, patch=0, prerelease=None, build=None), MzVersion(major=0, minor=57, patch=2, prerelease=None, build=None), MzVersion(major=0, minor=55, patch=6, prerelease=None, build=None), MzVersion(major=0, minor=57, patch=6, prerelease=None, build=None), MzVersion(major=0, minor=81, patch=0, prerelease=None, build=None), MzVersion(major=0, minor=99, patch=1, prerelease=None, build=None), MzVersion(major=0, minor=81, patch=2, prerelease=None, build=None), MzVersion(major=0, minor=55, patch=1, prerelease=None, build=None), MzVersion(major=0, minor=55, patch=3, prerelease=None, build=None), MzVersion(major=0, minor=57, patch=1, prerelease=None, build=None), MzVersion(major=0, minor=81, patch=1, prerelease=None, build=None)}
def
resolve_ancestor_image_tag( ancestor_overrides: dict[str, materialize.mz_version.MzVersion]) -> str | None:
151def resolve_ancestor_image_tag(ancestor_overrides: dict[str, MzVersion]) -> str | None: 152 """ 153 Resolve the ancestor image tag. 154 :param ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS 155 :return: image of the ancestor 156 """ 157 158 manual_ancestor_override = os.getenv("COMMON_ANCESTOR_OVERRIDE") 159 if manual_ancestor_override is not None: 160 image_tag = _manual_ancestor_specification_to_image_tag( 161 manual_ancestor_override 162 ) 163 print( 164 f"Using specified {image_tag} as image tag for ancestor (context: specified in $COMMON_ANCESTOR_OVERRIDE)" 165 ) 166 return image_tag 167 168 ancestor_image_resolution = _create_ancestor_image_resolution(ancestor_overrides) 169 result = ancestor_image_resolution.resolve_image_tag() 170 if result is None: 171 return None 172 image_tag, context = result 173 print(f"Using {image_tag} as image tag for ancestor (context: {context})") 174 return image_tag
Resolve the ancestor image tag.
Parameters
- ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS
Returns
image of the ancestor
class
AncestorImageResolutionBase:
193class AncestorImageResolutionBase: 194 def __init__(self, ancestor_overrides: dict[str, MzVersion]): 195 self.ancestor_overrides = ancestor_overrides 196 197 def resolve_image_tag(self) -> tuple[str, str]: 198 raise NotImplementedError 199 200 def _get_override_commit_instead_of_version( 201 self, 202 version: MzVersion, 203 ) -> str | None: 204 """ 205 If a commit specifies a mz version as prerequisite (to avoid regressions) that is newer than the provided 206 version (i.e., prerequisite not satisfied by the latest version), then return that commit's hash if the commit 207 contained in the current state. 208 Otherwise, return none. 209 """ 210 for ( 211 commit_hash, 212 min_required_mz_version, 213 ) in self.ancestor_overrides.items(): 214 if version >= min_required_mz_version: 215 continue 216 217 if git.contains_commit(commit_hash): 218 # commit would require at least min_required_mz_version 219 return commit_hash 220 221 return None 222 223 def _resolve_image_tag_of_previous_release( 224 self, context_prefix: str, previous_minor: bool 225 ) -> tuple[str, str] | None: 226 tagged_release_version = git.get_tagged_release_version(version_type=MzVersion) 227 assert tagged_release_version is not None 228 previous_release_version = get_previous_published_version( 229 tagged_release_version, previous_minor=previous_minor 230 ) 231 232 override_commit = self._get_override_commit_instead_of_version( 233 previous_release_version 234 ) 235 236 if override_commit is not None: 237 # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use 238 # use the commit instead of the previous release 239 # return ( 240 # commit_to_image_tag(override_commit), 241 # f"commit override instead of previous release ({previous_release_version})", 242 # ) 243 return None 244 245 return ( 246 release_version_to_image_tag(previous_release_version), 247 f"{context_prefix} {tagged_release_version}", 248 ) 249 250 def _resolve_image_tag_of_previous_release_from_current( 251 self, context: str 252 ) -> tuple[str, str] | None: 253 # Even though we are on main we might be in an older state, pick the 254 # latest release that was before our current version. 255 current_version = MzVersion.parse_cargo() 256 257 previous_published_version = get_previous_published_version( 258 current_version, previous_minor=True 259 ) 260 override_commit = self._get_override_commit_instead_of_version( 261 previous_published_version 262 ) 263 264 if override_commit is not None: 265 # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use 266 # use the commit instead of the latest release 267 # return ( 268 # commit_to_image_tag(override_commit), 269 # f"commit override instead of latest release ({previous_published_version})", 270 # ) 271 return None 272 273 return ( 274 release_version_to_image_tag(previous_published_version), 275 context, 276 ) 277 278 def _resolve_image_tag_of_merge_base( 279 self, 280 context_when_image_of_commit_exists: str, 281 context_when_falling_back_to_latest: str, 282 ) -> tuple[str, str] | None: 283 # If the current PR has a known and accepted regression, don't compare 284 # against merge base of it 285 override_commit = self._get_override_commit_instead_of_version( 286 MzVersion.parse_cargo() 287 ) 288 common_ancestor_commit = buildkite.get_merge_base() 289 290 if override_commit is not None: 291 # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use 292 # return ( 293 # commit_to_image_tag(override_commit), 294 # f"commit override instead of merge base ({common_ancestor_commit})", 295 # ) 296 return None 297 298 if image_of_commit_exists(common_ancestor_commit): 299 return ( 300 commit_to_image_tag(common_ancestor_commit), 301 context_when_image_of_commit_exists, 302 ) 303 else: 304 return ( 305 release_version_to_image_tag(get_latest_published_version()), 306 context_when_falling_back_to_latest, 307 )
310class AncestorImageResolutionLocal(AncestorImageResolutionBase): 311 def resolve_image_tag(self) -> tuple[str, str] | None: 312 if build_context.is_on_release_version(): 313 return self._resolve_image_tag_of_previous_release( 314 "previous minor release because on local release branch", 315 previous_minor=True, 316 ) 317 elif build_context.is_on_main_branch(): 318 return self._resolve_image_tag_of_previous_release_from_current( 319 "previous release from current because on local main branch" 320 ) 321 else: 322 return self._resolve_image_tag_of_merge_base( 323 "merge base of local non-main branch", 324 "latest release because image of merge base of local non-main branch not available", 325 )
def
resolve_image_tag(self) -> tuple[str, str] | None:
311 def resolve_image_tag(self) -> tuple[str, str] | None: 312 if build_context.is_on_release_version(): 313 return self._resolve_image_tag_of_previous_release( 314 "previous minor release because on local release branch", 315 previous_minor=True, 316 ) 317 elif build_context.is_on_main_branch(): 318 return self._resolve_image_tag_of_previous_release_from_current( 319 "previous release from current because on local main branch" 320 ) 321 else: 322 return self._resolve_image_tag_of_merge_base( 323 "merge base of local non-main branch", 324 "latest release because image of merge base of local non-main branch not available", 325 )
Inherited Members
328class AncestorImageResolutionInBuildkite(AncestorImageResolutionBase): 329 def resolve_image_tag(self) -> tuple[str, str] | None: 330 if buildkite.is_in_pull_request(): 331 return self._resolve_image_tag_of_merge_base( 332 "merge base of pull request", 333 "latest release because image of merge base of pull request not available", 334 ) 335 elif build_context.is_on_release_version(): 336 return self._resolve_image_tag_of_previous_release( 337 "previous minor release because on release branch", previous_minor=True 338 ) 339 else: 340 return self._resolve_image_tag_of_previous_release_from_current( 341 "previous release from current because not in a pull request and not on a release branch", 342 )
def
resolve_image_tag(self) -> tuple[str, str] | None:
329 def resolve_image_tag(self) -> tuple[str, str] | None: 330 if buildkite.is_in_pull_request(): 331 return self._resolve_image_tag_of_merge_base( 332 "merge base of pull request", 333 "latest release because image of merge base of pull request not available", 334 ) 335 elif build_context.is_on_release_version(): 336 return self._resolve_image_tag_of_previous_release( 337 "previous minor release because on release branch", previous_minor=True 338 ) 339 else: 340 return self._resolve_image_tag_of_previous_release_from_current( 341 "previous release from current because not in a pull request and not on a release branch", 342 )
Inherited Members
def
get_latest_published_version() -> materialize.mz_version.MzVersion:
345def get_latest_published_version() -> MzVersion: 346 """Get the latest mz version, older than current state, for which an image is published.""" 347 excluded_versions = set() 348 current_version = MzVersion.parse_cargo() 349 350 while True: 351 latest_published_version = git.get_latest_version( 352 version_type=MzVersion, 353 excluded_versions=excluded_versions, 354 current_version=current_version, 355 ) 356 357 if is_valid_release_image(latest_published_version): 358 return latest_published_version 359 else: 360 print( 361 f"Skipping version {latest_published_version} (image not found), trying earlier version" 362 ) 363 excluded_versions.add(latest_published_version)
Get the latest mz version, older than current state, for which an image is published.
def
get_previous_published_version( release_version: materialize.mz_version.MzVersion, previous_minor: bool) -> materialize.mz_version.MzVersion:
366def get_previous_published_version( 367 release_version: MzVersion, previous_minor: bool 368) -> MzVersion: 369 """Get the highest preceding mz version to the specified version for which an image is published.""" 370 excluded_versions = set() 371 372 while True: 373 previous_published_version = get_previous_mz_version( 374 release_version, 375 previous_minor=previous_minor, 376 excluded_versions=excluded_versions, 377 ) 378 379 if is_valid_release_image(previous_published_version): 380 return previous_published_version 381 else: 382 print(f"Skipping version {previous_published_version} (image not found)") 383 excluded_versions.add(previous_published_version)
Get the highest preceding mz version to the specified version for which an image is published.
def
get_published_minor_mz_versions( newest_first: bool = True, limit: int | None = None, include_filter: Callable[[materialize.mz_version.MzVersion], bool] | None = None, exclude_current_minor_version: bool = False) -> list[materialize.mz_version.MzVersion]:
386def get_published_minor_mz_versions( 387 newest_first: bool = True, 388 limit: int | None = None, 389 include_filter: Callable[[MzVersion], bool] | None = None, 390 exclude_current_minor_version: bool = False, 391) -> list[MzVersion]: 392 """ 393 Get the latest patch version for every minor version. 394 Use this version if it is NOT important whether a tag was introduced before or after creating this branch. 395 396 See also: #get_minor_mz_versions_listed_in_docs() 397 """ 398 399 # sorted in descending order 400 all_versions = get_all_mz_versions(newest_first=True) 401 minor_versions: dict[str, MzVersion] = {} 402 403 version = MzVersion.parse_cargo() 404 current_version = f"{version.major}.{version.minor}" 405 406 # Note that this method must not apply limit_to_published_versions to a created list 407 # because in that case minor versions may get lost. 408 for version in all_versions: 409 if include_filter is not None and not include_filter(version): 410 # this version shall not be included 411 continue 412 413 minor_version = f"{version.major}.{version.minor}" 414 415 if exclude_current_minor_version and minor_version == current_version: 416 continue 417 418 if minor_version in minor_versions.keys(): 419 # we already have a more recent version for this minor version 420 continue 421 422 if not is_valid_release_image(version): 423 # this version is not considered valid 424 continue 425 426 minor_versions[minor_version] = version 427 428 if limit is not None and len(minor_versions.keys()) == limit: 429 # collected enough versions 430 break 431 432 assert len(minor_versions) > 0 433 return sorted(minor_versions.values(), reverse=newest_first)
Get the latest patch version for every minor version. Use this version if it is NOT important whether a tag was introduced before or after creating this branch.
See also: #get_minor_mz_versions_listed_in_docs()
def
get_minor_mz_versions_listed_in_docs(respect_released_tag: bool) -> list[materialize.mz_version.MzVersion]:
436def get_minor_mz_versions_listed_in_docs(respect_released_tag: bool) -> list[MzVersion]: 437 """ 438 Get the latest patch version for every minor version in ascending order. 439 Use this version if it is important whether a tag was introduced before or after creating this branch. 440 441 See also: #get_published_minor_mz_versions() 442 """ 443 return VersionsFromDocs(respect_released_tag).minor_versions()
Get the latest patch version for every minor version in ascending order. Use this version if it is important whether a tag was introduced before or after creating this branch.
See also: #get_published_minor_mz_versions()
def
get_all_mz_versions(newest_first: bool = True) -> list[materialize.mz_version.MzVersion]:
446def get_all_mz_versions( 447 newest_first: bool = True, 448) -> list[MzVersion]: 449 """ 450 Get all mz versions based on git tags. Versions known to be invalid are excluded. 451 452 See also: #get_all_mz_versions_listed_in_docs 453 """ 454 return [ 455 version 456 for version in get_version_tags( 457 version_type=MzVersion, newest_first=newest_first 458 ) 459 if version not in INVALID_VERSIONS 460 # Exclude release candidates 461 and not version.prerelease 462 ]
Get all mz versions based on git tags. Versions known to be invalid are excluded.
See also: #get_all_mz_versions_listed_in_docs
def
get_all_mz_versions_listed_in_docs(respect_released_tag: bool) -> list[materialize.mz_version.MzVersion]:
465def get_all_mz_versions_listed_in_docs( 466 respect_released_tag: bool, 467) -> list[MzVersion]: 468 """ 469 Get all mz versions based on docs. Versions known to be invalid are excluded. 470 471 See also: #get_all_mz_versions() 472 """ 473 return VersionsFromDocs(respect_released_tag).all_versions()
Get all mz versions based on docs. Versions known to be invalid are excluded.
See also: #get_all_mz_versions()
def
get_all_published_mz_versions( newest_first: bool = True, limit: int | None = None) -> list[materialize.mz_version.MzVersion]:
476def get_all_published_mz_versions( 477 newest_first: bool = True, limit: int | None = None 478) -> list[MzVersion]: 479 """Get all mz versions based on git tags. This method ensures that images of the versions exist.""" 480 all_versions = get_all_mz_versions(newest_first=newest_first) 481 print(f"all_versions: {all_versions}") 482 return limit_to_published_versions(all_versions, limit)
Get all mz versions based on git tags. This method ensures that images of the versions exist.
def
get_published_mz_versions_within_one_major_version(newest_first: bool = True) -> list[materialize.mz_version.MzVersion]:
485def get_published_mz_versions_within_one_major_version( 486 newest_first: bool = True, 487) -> list[MzVersion]: 488 """Get all previous mz versions within one major version of the current version. Ensure that images of the versions exist.""" 489 current_version = MzVersion.parse_cargo() 490 all_versions = get_all_mz_versions(newest_first=newest_first) 491 versions_within_one_major_version = { 492 v 493 for v in all_versions 494 if abs(v.major - current_version.major) <= 1 and v <= current_version 495 } 496 497 return limit_to_published_versions(list(versions_within_one_major_version))
Get all previous mz versions within one major version of the current version. Ensure that images of the versions exist.
def
limit_to_published_versions( all_versions: list[materialize.mz_version.MzVersion], limit: int | None = None) -> list[materialize.mz_version.MzVersion]:
500def limit_to_published_versions( 501 all_versions: list[MzVersion], limit: int | None = None 502) -> list[MzVersion]: 503 """Remove versions for which no image is published.""" 504 versions = [] 505 506 for v in all_versions: 507 if is_valid_release_image(v): 508 versions.append(v) 509 510 if limit is not None and len(versions) == limit: 511 break 512 513 return versions
Remove versions for which no image is published.
def
get_previous_mz_version( version: materialize.mz_version.MzVersion, previous_minor: bool, excluded_versions: set[materialize.mz_version.MzVersion] | None = None) -> materialize.mz_version.MzVersion:
516def get_previous_mz_version( 517 version: MzVersion, 518 previous_minor: bool, 519 excluded_versions: set[MzVersion] | None = None, 520) -> MzVersion: 521 """Get the predecessor of the specified version based on git tags.""" 522 if excluded_versions is None: 523 excluded_versions = set() 524 525 if previous_minor: 526 version = MzVersion.create(version.major, version.minor, 0) 527 528 if version.prerelease is not None and len(version.prerelease) > 0: 529 # simply drop the prerelease, do not try to find a decremented version 530 found_version = MzVersion.create(version.major, version.minor, version.patch) 531 532 if found_version not in excluded_versions: 533 return found_version 534 else: 535 # start searching with this version 536 version = found_version 537 538 all_versions: list[MzVersion] = get_version_tags(version_type=type(version)) 539 all_suitable_previous_versions = [ 540 v 541 for v in all_versions 542 if v < version 543 and (v.prerelease is None or len(v.prerelease) == 0) 544 and v not in INVALID_VERSIONS 545 and v not in excluded_versions 546 ] 547 return max(all_suitable_previous_versions)
Get the predecessor of the specified version based on git tags.
def
is_valid_release_image(version: materialize.mz_version.MzVersion) -> bool:
550def is_valid_release_image(version: MzVersion) -> bool: 551 """ 552 Checks if a version is not known as an invalid version and has a published image. 553 Note that this method may take shortcuts on older versions. 554 """ 555 if version in INVALID_VERSIONS: 556 return False 557 558 if version < _SKIP_IMAGE_CHECK_BELOW_THIS_VERSION: 559 # optimization: assume that all versions older than this one are either valid or listed in INVALID_VERSIONS 560 return True 561 562 # This is a potentially expensive operation which pulls an image if it hasn't been pulled yet. 563 return docker.image_of_release_version_exists(version)
Checks if a version is not known as an invalid version and has a published image. Note that this method may take shortcuts on older versions.
def
get_commits_of_accepted_regressions_between_versions( ancestor_overrides: dict[str, materialize.mz_version.MzVersion], since_version_exclusive: materialize.mz_version.MzVersion, to_version_inclusive: materialize.mz_version.MzVersion) -> list[str]:
566def get_commits_of_accepted_regressions_between_versions( 567 ancestor_overrides: dict[str, MzVersion], 568 since_version_exclusive: MzVersion, 569 to_version_inclusive: MzVersion, 570) -> list[str]: 571 """ 572 Get commits of accepted regressions between both versions. 573 :param ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS 574 :return: commits 575 """ 576 577 assert since_version_exclusive <= to_version_inclusive 578 579 commits = [] 580 581 for ( 582 regression_introducing_commit, 583 first_version_with_regression, 584 ) in ancestor_overrides.items(): 585 if ( 586 since_version_exclusive 587 < first_version_with_regression 588 <= to_version_inclusive 589 ): 590 commits.append(regression_introducing_commit) 591 592 return commits
Get commits of accepted regressions between both versions.
Parameters
- ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS
Returns
commits
class
VersionsFromDocs:
595class VersionsFromDocs: 596 """Materialize versions as listed in doc/user/content/releases 597 598 >>> len(VersionsFromDocs(respect_released_tag=True).all_versions()) > 0 599 True 600 601 >>> len(VersionsFromDocs(respect_released_tag=True).minor_versions()) > 0 602 True 603 604 >>> len(VersionsFromDocs(respect_released_tag=True).patch_versions(minor_version=MzVersion.parse_mz("v0.52.0"))) 605 4 606 607 >>> min(VersionsFromDocs(respect_released_tag=True).all_versions()) 608 MzVersion(major=0, minor=27, patch=0, prerelease=None, build=None) 609 """ 610 611 def __init__( 612 self, 613 respect_released_tag: bool, 614 respect_date: bool = False, 615 only_publish_helm_chart: bool = True, 616 skip_rc: bool = False, 617 ) -> None: 618 files = Path(MZ_ROOT / "doc" / "user" / "content" / "releases").glob("v*.md") 619 self.versions = [] 620 current_version = MzVersion.parse_cargo() 621 for f in files: 622 base = f.stem 623 metadata = frontmatter.load(f) 624 if respect_released_tag and not metadata.get("released", False): 625 continue 626 if only_publish_helm_chart and not metadata.get("publish_helm_chart", True): 627 continue 628 date: datetime.date = metadata["date"] 629 if respect_date and date > datetime.date.today(): 630 continue 631 632 current_patch = metadata.get("patch", 0) 633 current_rc = metadata.get("rc", 0) 634 635 if current_rc > 0: 636 if skip_rc: 637 continue 638 for rc in range(1, current_rc + 1): 639 version = MzVersion.parse_mz(f"{base}.{current_patch}-rc.{rc}") 640 if not respect_released_tag and version >= current_version: 641 continue 642 if version not in INVALID_VERSIONS: 643 self.versions.append(version) 644 else: 645 for patch in range(current_patch + 1): 646 version = MzVersion.parse_mz(f"{base}.{patch}") 647 if not respect_released_tag and version >= current_version: 648 continue 649 if version not in INVALID_VERSIONS: 650 self.versions.append(version) 651 652 assert len(self.versions) > 0 653 self.versions.sort() 654 655 def all_versions(self) -> list[MzVersion]: 656 return self.versions 657 658 def minor_versions(self) -> list[MzVersion]: 659 """Return the latest patch version for every minor version.""" 660 minor_versions = {} 661 for version in self.versions: 662 minor_versions[f"{version.major}.{version.minor}"] = version 663 664 assert len(minor_versions) > 0 665 return sorted(minor_versions.values()) 666 667 def patch_versions(self, minor_version: MzVersion) -> list[MzVersion]: 668 """Return all patch versions within the given minor version.""" 669 patch_versions = [] 670 for version in self.versions: 671 if ( 672 version.major == minor_version.major 673 and version.minor == minor_version.minor 674 ): 675 patch_versions.append(version) 676 677 assert len(patch_versions) > 0 678 return sorted(patch_versions)
Materialize versions as listed in doc/user/content/releases
>>> len(VersionsFromDocs(respect_released_tag=True).all_versions()) > 0
True
>>> len(VersionsFromDocs(respect_released_tag=True).minor_versions()) > 0
True
>>> len(VersionsFromDocs(respect_released_tag=True).patch_versions(minor_version=MzVersion.parse_mz("v0.52.0")))
4
>>> min(VersionsFromDocs(respect_released_tag=True).all_versions())
MzVersion(major=0, minor=27, patch=0, prerelease=None, build=None)
VersionsFromDocs( respect_released_tag: bool, respect_date: bool = False, only_publish_helm_chart: bool = True, skip_rc: bool = False)
611 def __init__( 612 self, 613 respect_released_tag: bool, 614 respect_date: bool = False, 615 only_publish_helm_chart: bool = True, 616 skip_rc: bool = False, 617 ) -> None: 618 files = Path(MZ_ROOT / "doc" / "user" / "content" / "releases").glob("v*.md") 619 self.versions = [] 620 current_version = MzVersion.parse_cargo() 621 for f in files: 622 base = f.stem 623 metadata = frontmatter.load(f) 624 if respect_released_tag and not metadata.get("released", False): 625 continue 626 if only_publish_helm_chart and not metadata.get("publish_helm_chart", True): 627 continue 628 date: datetime.date = metadata["date"] 629 if respect_date and date > datetime.date.today(): 630 continue 631 632 current_patch = metadata.get("patch", 0) 633 current_rc = metadata.get("rc", 0) 634 635 if current_rc > 0: 636 if skip_rc: 637 continue 638 for rc in range(1, current_rc + 1): 639 version = MzVersion.parse_mz(f"{base}.{current_patch}-rc.{rc}") 640 if not respect_released_tag and version >= current_version: 641 continue 642 if version not in INVALID_VERSIONS: 643 self.versions.append(version) 644 else: 645 for patch in range(current_patch + 1): 646 version = MzVersion.parse_mz(f"{base}.{patch}") 647 if not respect_released_tag and version >= current_version: 648 continue 649 if version not in INVALID_VERSIONS: 650 self.versions.append(version) 651 652 assert len(self.versions) > 0 653 self.versions.sort()
def
minor_versions(self) -> list[materialize.mz_version.MzVersion]:
658 def minor_versions(self) -> list[MzVersion]: 659 """Return the latest patch version for every minor version.""" 660 minor_versions = {} 661 for version in self.versions: 662 minor_versions[f"{version.major}.{version.minor}"] = version 663 664 assert len(minor_versions) > 0 665 return sorted(minor_versions.values())
Return the latest patch version for every minor version.
def
patch_versions( self, minor_version: materialize.mz_version.MzVersion) -> list[materialize.mz_version.MzVersion]:
667 def patch_versions(self, minor_version: MzVersion) -> list[MzVersion]: 668 """Return all patch versions within the given minor version.""" 669 patch_versions = [] 670 for version in self.versions: 671 if ( 672 version.major == minor_version.major 673 and version.minor == minor_version.minor 674 ): 675 patch_versions.append(version) 676 677 assert len(patch_versions) > 0 678 return sorted(patch_versions)
Return all patch versions within the given minor version.