misc.python.materialize.version_list
1# Copyright Materialize, Inc. and contributors. All rights reserved. 2# 3# Use of this software is governed by the Business Source License 4# included in the LICENSE file at the root of this repository. 5# 6# As of the Change Date specified in that file, in accordance with 7# the Business Source License, use of this software will be governed 8# by the Apache License, Version 2.0. 9 10 11from __future__ import annotations 12 13import datetime 14import os 15from collections.abc import Callable 16from dataclasses import dataclass 17from pathlib import Path 18 19import frontmatter 20import requests 21import yaml 22 23from materialize import build_context, buildkite, docker, git 24from materialize.docker import ( 25 commit_to_image_tag, 26 image_of_commit_exists, 27 release_version_to_image_tag, 28) 29from materialize.git import get_version_tags 30from materialize.mz_version import MzVersion 31 32MZ_ROOT = Path(os.environ["MZ_ROOT"]) 33 34 35@dataclass 36class SelfManagedVersion: 37 helm_version: MzVersion 38 version: MzVersion 39 40 41def fetch_self_managed_versions() -> list[SelfManagedVersion]: 42 result: list[SelfManagedVersion] = [] 43 for entry in yaml.safe_load( 44 requests.get("https://materializeinc.github.io/materialize/index.yaml").text 45 )["entries"]["materialize-operator"]: 46 self_managed_version = SelfManagedVersion( 47 MzVersion.parse_mz(entry["version"]), 48 MzVersion.parse_mz(entry["appVersion"]), 49 ) 50 if ( 51 not self_managed_version.version.prerelease 52 and self_managed_version.version not in BAD_SELF_MANAGED_VERSIONS 53 ): 54 result.append(self_managed_version) 55 return result 56 57 58def get_all_self_managed_versions() -> list[MzVersion]: 59 return sorted([version.version for version in fetch_self_managed_versions()]) 60 61 62def get_self_managed_versions( 63 max_version: MzVersion | None = None, 64) -> list[MzVersion]: 65 prefixes = set() 66 result = set() 67 self_managed_versions = fetch_self_managed_versions() 68 for version_info in self_managed_versions: 69 if max_version is not None and version_info.version >= max_version: 70 continue 71 prefix = (version_info.version.major, version_info.version.minor) 72 if ( 73 not version_info.version.prerelease 74 and prefix not in prefixes 75 and not version_info.helm_version.prerelease 76 ): 77 result.add(version_info.version) 78 prefixes.add(prefix) 79 return sorted(result) 80 81 82# Gets the range of versions we can "upgrade from" to the current version, sorted in ascending order. 83def get_compatible_upgrade_from_versions() -> list[MzVersion]: 84 85 # Determine the current MzVersion from the environment, or from a version constant 86 current_version = MzVersion.parse_cargo() 87 88 published_versions_within_one_major_version = { 89 v 90 for v in get_published_mz_versions_within_one_major_version() 91 if abs(v.major - current_version.major) <= 1 and v <= current_version 92 } 93 94 if current_version.major <= 26: 95 # For versions <= 26, we can only upgrade from 25.2 self-managed versions 96 self_managed_25_2_versions = { 97 v.version 98 for v in fetch_self_managed_versions() 99 if v.helm_version.major == 25 and v.helm_version.minor == 2 100 } 101 102 return sorted( 103 self_managed_25_2_versions.union( 104 published_versions_within_one_major_version 105 ) 106 ) 107 else: 108 # For versions > 26, get all mz versions within 1 major version of current_version 109 return sorted(published_versions_within_one_major_version) 110 111 112BAD_SELF_MANAGED_VERSIONS = { 113 MzVersion.parse_mz("v0.130.0"), 114 MzVersion.parse_mz("v0.130.1"), 115 MzVersion.parse_mz("v0.130.2"), 116 MzVersion.parse_mz("v0.130.3"), 117 MzVersion.parse_mz("v0.130.4"), 118 MzVersion.parse_mz( 119 "v0.147.7" 120 ), # Incompatible for upgrades because it clears login attribute for roles due to catalog migration 121 MzVersion.parse_mz( 122 "v0.147.14" 123 ), # Incompatible for upgrades because it clears login attribute for roles due to catalog migration 124 MzVersion.parse_mz("v0.157.0"), 125} 126 127# not released on Docker 128INVALID_VERSIONS = { 129 MzVersion.parse_mz("v0.52.1"), 130 MzVersion.parse_mz("v0.55.1"), 131 MzVersion.parse_mz("v0.55.2"), 132 MzVersion.parse_mz("v0.55.3"), 133 MzVersion.parse_mz("v0.55.4"), 134 MzVersion.parse_mz("v0.55.5"), 135 MzVersion.parse_mz("v0.55.6"), 136 MzVersion.parse_mz("v0.56.0"), 137 MzVersion.parse_mz("v0.57.1"), 138 MzVersion.parse_mz("v0.57.2"), 139 MzVersion.parse_mz("v0.57.5"), 140 MzVersion.parse_mz("v0.57.6"), 141 MzVersion.parse_mz("v0.81.0"), # incompatible for upgrades 142 MzVersion.parse_mz("v0.81.1"), # incompatible for upgrades 143 MzVersion.parse_mz("v0.81.2"), # incompatible for upgrades 144 MzVersion.parse_mz("v0.89.7"), 145 MzVersion.parse_mz("v0.92.0"), # incompatible for upgrades 146 MzVersion.parse_mz("v0.93.0"), # accidental release 147 MzVersion.parse_mz("v0.99.1"), # incompatible for upgrades 148 MzVersion.parse_mz("v0.113.1"), # incompatible for upgrades 149} 150 151_SKIP_IMAGE_CHECK_BELOW_THIS_VERSION = MzVersion.parse_mz("v0.77.0") 152 153 154def resolve_ancestor_image_tag(ancestor_overrides: dict[str, MzVersion]) -> str | None: 155 """ 156 Resolve the ancestor image tag. 157 :param ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS 158 :return: image of the ancestor 159 """ 160 161 manual_ancestor_override = os.getenv("COMMON_ANCESTOR_OVERRIDE") 162 if manual_ancestor_override is not None: 163 image_tag = _manual_ancestor_specification_to_image_tag( 164 manual_ancestor_override 165 ) 166 print( 167 f"Using specified {image_tag} as image tag for ancestor (context: specified in $COMMON_ANCESTOR_OVERRIDE)" 168 ) 169 return image_tag 170 171 ancestor_image_resolution = _create_ancestor_image_resolution(ancestor_overrides) 172 result = ancestor_image_resolution.resolve_image_tag() 173 if result is None: 174 return None 175 image_tag, context = result 176 print(f"Using {image_tag} as image tag for ancestor (context: {context})") 177 return image_tag 178 179 180def _create_ancestor_image_resolution( 181 ancestor_overrides: dict[str, MzVersion], 182) -> AncestorImageResolutionBase: 183 if buildkite.is_in_buildkite(): 184 return AncestorImageResolutionInBuildkite(ancestor_overrides) 185 else: 186 return AncestorImageResolutionLocal(ancestor_overrides) 187 188 189def _manual_ancestor_specification_to_image_tag(ancestor_spec: str) -> str: 190 if MzVersion.is_valid_version_string(ancestor_spec): 191 return release_version_to_image_tag(MzVersion.parse_mz(ancestor_spec)) 192 else: 193 return commit_to_image_tag(ancestor_spec) 194 195 196class AncestorImageResolutionBase: 197 def __init__(self, ancestor_overrides: dict[str, MzVersion]): 198 self.ancestor_overrides = ancestor_overrides 199 200 def resolve_image_tag(self) -> tuple[str, str] | None: 201 raise NotImplementedError 202 203 def _get_override_commit_instead_of_version( 204 self, 205 version: MzVersion, 206 ) -> str | None: 207 """ 208 If a commit specifies a mz version as prerequisite (to avoid regressions) that is newer than the provided 209 version (i.e., prerequisite not satisfied by the latest version), then return that commit's hash if the commit 210 contained in the current state. 211 Otherwise, return none. 212 """ 213 for ( 214 commit_hash, 215 min_required_mz_version, 216 ) in self.ancestor_overrides.items(): 217 if version >= min_required_mz_version: 218 continue 219 220 if git.contains_commit(commit_hash): 221 # commit would require at least min_required_mz_version 222 return commit_hash 223 224 return None 225 226 def _resolve_image_tag_of_previous_release( 227 self, context_prefix: str, previous_minor: bool 228 ) -> tuple[str, str] | None: 229 tagged_release_version = git.get_tagged_release_version(version_type=MzVersion) 230 assert tagged_release_version is not None 231 previous_release_version = get_previous_published_version( 232 tagged_release_version, previous_minor=previous_minor 233 ) 234 235 override_commit = self._get_override_commit_instead_of_version( 236 previous_release_version 237 ) 238 239 if override_commit is not None: 240 # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use 241 # use the commit instead of the previous release 242 # return ( 243 # commit_to_image_tag(override_commit), 244 # f"commit override instead of previous release ({previous_release_version})", 245 # ) 246 return None 247 248 return ( 249 release_version_to_image_tag(previous_release_version), 250 f"{context_prefix} {tagged_release_version}", 251 ) 252 253 def _resolve_image_tag_of_previous_release_from_current( 254 self, context: str 255 ) -> tuple[str, str] | None: 256 # Even though we are on main we might be in an older state, pick the 257 # latest release that was before our current version. 258 current_version = MzVersion.parse_cargo() 259 260 previous_published_version = get_previous_published_version( 261 current_version, previous_minor=True 262 ) 263 override_commit = self._get_override_commit_instead_of_version( 264 previous_published_version 265 ) 266 267 if override_commit is not None: 268 # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use 269 # use the commit instead of the latest release 270 # return ( 271 # commit_to_image_tag(override_commit), 272 # f"commit override instead of latest release ({previous_published_version})", 273 # ) 274 return None 275 276 return ( 277 release_version_to_image_tag(previous_published_version), 278 context, 279 ) 280 281 def _resolve_image_tag_of_merge_base( 282 self, 283 context_when_image_of_commit_exists: str, 284 ) -> tuple[str, str] | None: 285 # If the current PR has a known and accepted regression, don't compare 286 # against merge base of it 287 override_commit = self._get_override_commit_instead_of_version( 288 MzVersion.parse_cargo() 289 ) 290 common_ancestor_commit = buildkite.get_merge_base() 291 292 if override_commit is not None: 293 # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use 294 # return ( 295 # commit_to_image_tag(override_commit), 296 # f"commit override instead of merge base ({common_ancestor_commit})", 297 # ) 298 return None 299 300 ancestors = git.get_first_parent_commits(common_ancestor_commit, limit=20) 301 for ancestor in ancestors: 302 if image_of_commit_exists(ancestor): 303 context = ( 304 context_when_image_of_commit_exists 305 if ancestor == common_ancestor_commit 306 else f"ancestor of {context_when_image_of_commit_exists} (walked back from {common_ancestor_commit[:12]})" 307 ) 308 return ( 309 commit_to_image_tag(ancestor), 310 context, 311 ) 312 313 return None 314 315 316class AncestorImageResolutionLocal(AncestorImageResolutionBase): 317 def resolve_image_tag(self) -> tuple[str, str] | None: 318 if build_context.is_on_release_version(): 319 return self._resolve_image_tag_of_previous_release( 320 "previous minor release because on local release branch", 321 previous_minor=True, 322 ) 323 elif build_context.is_on_main_branch(): 324 return self._resolve_image_tag_of_previous_release_from_current( 325 "previous release from current because on local main branch" 326 ) 327 else: 328 return self._resolve_image_tag_of_merge_base( 329 "merge base of local non-main branch", 330 ) 331 332 333class AncestorImageResolutionInBuildkite(AncestorImageResolutionBase): 334 def resolve_image_tag(self) -> tuple[str, str] | None: 335 if buildkite.is_in_pull_request(): 336 return self._resolve_image_tag_of_merge_base( 337 "merge base of pull request", 338 ) 339 elif build_context.is_on_release_version(): 340 return self._resolve_image_tag_of_previous_release( 341 "previous minor release because on release branch", previous_minor=True 342 ) 343 else: 344 return self._resolve_image_tag_of_previous_release_from_current( 345 "previous release from current because not in a pull request and not on a release branch", 346 ) 347 348 349def get_latest_published_version() -> MzVersion: 350 """Get the latest mz version, older than current state, for which an image is published.""" 351 excluded_versions = set() 352 current_version = MzVersion.parse_cargo() 353 354 while True: 355 latest_published_version = git.get_latest_version( 356 version_type=MzVersion, 357 excluded_versions=excluded_versions, 358 current_version=current_version, 359 ) 360 361 if is_valid_release_image(latest_published_version): 362 return latest_published_version 363 else: 364 print( 365 f"Skipping version {latest_published_version} (image not found), trying earlier version" 366 ) 367 excluded_versions.add(latest_published_version) 368 369 370def get_previous_published_version( 371 release_version: MzVersion, previous_minor: bool 372) -> MzVersion: 373 """Get the highest preceding mz version to the specified version for which an image is published.""" 374 excluded_versions = set() 375 376 while True: 377 previous_published_version = get_previous_mz_version( 378 release_version, 379 previous_minor=previous_minor, 380 excluded_versions=excluded_versions, 381 ) 382 383 if is_valid_release_image(previous_published_version): 384 return previous_published_version 385 else: 386 print(f"Skipping version {previous_published_version} (image not found)") 387 excluded_versions.add(previous_published_version) 388 389 390def get_published_minor_mz_versions( 391 newest_first: bool = True, 392 limit: int | None = None, 393 include_filter: Callable[[MzVersion], bool] | None = None, 394 exclude_current_minor_version: bool = False, 395 max_version: MzVersion | None = None, 396) -> list[MzVersion]: 397 """ 398 Get the latest patch version for every minor version. 399 Use this version if it is NOT important whether a tag was introduced before or after creating this branch. 400 401 See also: #get_minor_mz_versions_listed_in_docs() 402 """ 403 404 # sorted in descending order 405 all_versions = get_all_mz_versions(newest_first=True) 406 minor_versions: dict[str, MzVersion] = {} 407 408 version = MzVersion.parse_cargo() 409 current_version = f"{version.major}.{version.minor}" 410 411 # Note that this method must not apply limit_to_published_versions to a created list 412 # because in that case minor versions may get lost. 413 for version in all_versions: 414 if include_filter is not None and not include_filter(version): 415 # this version shall not be included 416 continue 417 418 if max_version is not None and version >= max_version: 419 continue 420 421 minor_version = f"{version.major}.{version.minor}" 422 423 if exclude_current_minor_version and minor_version == current_version: 424 continue 425 426 if minor_version in minor_versions.keys(): 427 # we already have a more recent version for this minor version 428 continue 429 430 if not is_valid_release_image(version): 431 # this version is not considered valid 432 continue 433 434 minor_versions[minor_version] = version 435 436 if limit is not None and len(minor_versions.keys()) == limit: 437 # collected enough versions 438 break 439 440 assert len(minor_versions) > 0 441 return sorted(minor_versions.values(), reverse=newest_first) 442 443 444def get_all_mz_versions( 445 newest_first: bool = True, 446) -> list[MzVersion]: 447 """ 448 Get all mz versions based on git tags. Versions known to be invalid are excluded. 449 450 See also: #get_all_mz_versions_listed_in_docs 451 """ 452 return [ 453 version 454 for version in get_version_tags( 455 version_type=MzVersion, newest_first=newest_first 456 ) 457 if version not in INVALID_VERSIONS 458 # Exclude release candidates 459 and not version.prerelease 460 ] 461 462 463def get_all_published_mz_versions( 464 newest_first: bool = True, limit: int | None = None 465) -> list[MzVersion]: 466 """Get all mz versions based on git tags. This method ensures that images of the versions exist.""" 467 all_versions = get_all_mz_versions(newest_first=newest_first) 468 print(f"all_versions: {all_versions}") 469 return limit_to_published_versions(all_versions, limit) 470 471 472def get_published_mz_versions_within_one_major_version( 473 newest_first: bool = True, 474) -> list[MzVersion]: 475 """Get all previous mz versions within one major version of the current version. Ensure that images of the versions exist.""" 476 current_version = MzVersion.parse_cargo() 477 all_versions = get_all_mz_versions(newest_first=newest_first) 478 versions_within_one_major_version = { 479 v 480 for v in all_versions 481 if abs(v.major - current_version.major) <= 1 and v <= current_version 482 } 483 484 return limit_to_published_versions(list(versions_within_one_major_version)) 485 486 487def limit_to_published_versions( 488 all_versions: list[MzVersion], limit: int | None = None 489) -> list[MzVersion]: 490 """Remove versions for which no image is published.""" 491 versions = [] 492 493 for v in all_versions: 494 if is_valid_release_image(v): 495 versions.append(v) 496 497 if limit is not None and len(versions) == limit: 498 break 499 500 return versions 501 502 503def get_previous_mz_version( 504 version: MzVersion, 505 previous_minor: bool, 506 excluded_versions: set[MzVersion] | None = None, 507) -> MzVersion: 508 """Get the predecessor of the specified version based on git tags.""" 509 if excluded_versions is None: 510 excluded_versions = set() 511 512 if previous_minor: 513 version = MzVersion.create(version.major, version.minor, 0) 514 515 if version.prerelease is not None and len(version.prerelease) > 0: 516 # simply drop the prerelease, do not try to find a decremented version 517 found_version = MzVersion.create(version.major, version.minor, version.patch) 518 519 if found_version not in excluded_versions: 520 return found_version 521 else: 522 # start searching with this version 523 version = found_version 524 525 all_versions: list[MzVersion] = get_version_tags(version_type=type(version)) 526 all_suitable_previous_versions = [ 527 v 528 for v in all_versions 529 if v < version 530 and (v.prerelease is None or len(v.prerelease) == 0) 531 and v not in INVALID_VERSIONS 532 and v not in excluded_versions 533 ] 534 return max(all_suitable_previous_versions) 535 536 537def is_valid_release_image(version: MzVersion) -> bool: 538 """ 539 Checks if a version is not known as an invalid version and has a published image. 540 Note that this method may take shortcuts on older versions. 541 """ 542 if version in INVALID_VERSIONS: 543 return False 544 545 if version < _SKIP_IMAGE_CHECK_BELOW_THIS_VERSION: 546 # optimization: assume that all versions older than this one are either valid or listed in INVALID_VERSIONS 547 return True 548 549 # This is a potentially expensive operation which pulls an image if it hasn't been pulled yet. 550 return docker.image_of_release_version_exists(version) 551 552 553def get_commits_of_accepted_regressions_between_versions( 554 ancestor_overrides: dict[str, MzVersion], 555 since_version_exclusive: MzVersion, 556 to_version_inclusive: MzVersion, 557) -> list[str]: 558 """ 559 Get commits of accepted regressions between both versions. 560 :param ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS 561 :return: commits 562 """ 563 564 assert since_version_exclusive <= to_version_inclusive 565 566 commits = [] 567 568 for ( 569 regression_introducing_commit, 570 first_version_with_regression, 571 ) in ancestor_overrides.items(): 572 if ( 573 since_version_exclusive 574 < first_version_with_regression 575 <= to_version_inclusive 576 ): 577 commits.append(regression_introducing_commit) 578 579 return commits 580 581 582class VersionsFromDocs: 583 """Materialize versions as listed in doc/user/content/releases 584 585 >>> len(VersionsFromDocs(respect_released_tag=True).all_versions()) > 0 586 True 587 588 >>> len(VersionsFromDocs(respect_released_tag=True).minor_versions()) > 0 589 True 590 591 >>> len(VersionsFromDocs(respect_released_tag=True).patch_versions(minor_version=MzVersion.parse_mz("v0.52.0"))) 592 4 593 594 >>> min(VersionsFromDocs(respect_released_tag=True).all_versions()) 595 MzVersion(major=0, minor=27, patch=0, prerelease=None, build=None) 596 """ 597 598 def __init__( 599 self, 600 respect_released_tag: bool, 601 respect_date: bool = False, 602 only_publish_helm_chart: bool = True, 603 skip_rc: bool = False, 604 ) -> None: 605 files = Path(MZ_ROOT / "doc" / "user" / "content" / "releases").glob("v*.md") 606 self.versions = [] 607 current_version = MzVersion.parse_cargo() 608 for f in files: 609 base = f.stem 610 metadata = frontmatter.load(f) 611 if respect_released_tag and not metadata.get("released", False): 612 continue 613 if only_publish_helm_chart and not metadata.get("publish_helm_chart", True): 614 continue 615 date: datetime.date = metadata["date"] 616 if respect_date and date > datetime.date.today(): 617 continue 618 619 current_patch = metadata.get("patch", 0) 620 current_rc = metadata.get("rc", 0) 621 622 if current_rc > 0: 623 if skip_rc: 624 continue 625 for rc in range(1, current_rc + 1): 626 version = MzVersion.parse_mz(f"{base}.{current_patch}-rc.{rc}") 627 if not respect_released_tag and version >= current_version: 628 continue 629 if version not in INVALID_VERSIONS: 630 self.versions.append(version) 631 else: 632 for patch in range(current_patch + 1): 633 version = MzVersion.parse_mz(f"{base}.{patch}") 634 if not respect_released_tag and version >= current_version: 635 continue 636 if version not in INVALID_VERSIONS: 637 self.versions.append(version) 638 639 assert len(self.versions) > 0 640 self.versions.sort() 641 642 def all_versions(self) -> list[MzVersion]: 643 return self.versions 644 645 def minor_versions(self) -> list[MzVersion]: 646 """Return the latest patch version for every minor version.""" 647 minor_versions = {} 648 for version in self.versions: 649 minor_versions[f"{version.major}.{version.minor}"] = version 650 651 assert len(minor_versions) > 0 652 return sorted(minor_versions.values()) 653 654 def patch_versions(self, minor_version: MzVersion) -> list[MzVersion]: 655 """Return all patch versions within the given minor version.""" 656 patch_versions = [] 657 for version in self.versions: 658 if ( 659 version.major == minor_version.major 660 and version.minor == minor_version.minor 661 ): 662 patch_versions.append(version) 663 664 assert len(patch_versions) > 0 665 return sorted(patch_versions)
MZ_ROOT =
PosixPath('/var/lib/buildkite-agent/builds/buildkite-15f2293-i-04151a3597e2b37a7-1/materialize/deploy')
@dataclass
class
SelfManagedVersion:
42def fetch_self_managed_versions() -> list[SelfManagedVersion]: 43 result: list[SelfManagedVersion] = [] 44 for entry in yaml.safe_load( 45 requests.get("https://materializeinc.github.io/materialize/index.yaml").text 46 )["entries"]["materialize-operator"]: 47 self_managed_version = SelfManagedVersion( 48 MzVersion.parse_mz(entry["version"]), 49 MzVersion.parse_mz(entry["appVersion"]), 50 ) 51 if ( 52 not self_managed_version.version.prerelease 53 and self_managed_version.version not in BAD_SELF_MANAGED_VERSIONS 54 ): 55 result.append(self_managed_version) 56 return result
def
get_all_self_managed_versions() -> list[materialize.mz_version.MzVersion]:
def
get_self_managed_versions( max_version: materialize.mz_version.MzVersion | None = None) -> list[materialize.mz_version.MzVersion]:
63def get_self_managed_versions( 64 max_version: MzVersion | None = None, 65) -> list[MzVersion]: 66 prefixes = set() 67 result = set() 68 self_managed_versions = fetch_self_managed_versions() 69 for version_info in self_managed_versions: 70 if max_version is not None and version_info.version >= max_version: 71 continue 72 prefix = (version_info.version.major, version_info.version.minor) 73 if ( 74 not version_info.version.prerelease 75 and prefix not in prefixes 76 and not version_info.helm_version.prerelease 77 ): 78 result.add(version_info.version) 79 prefixes.add(prefix) 80 return sorted(result)
def
get_compatible_upgrade_from_versions() -> list[materialize.mz_version.MzVersion]:
84def get_compatible_upgrade_from_versions() -> list[MzVersion]: 85 86 # Determine the current MzVersion from the environment, or from a version constant 87 current_version = MzVersion.parse_cargo() 88 89 published_versions_within_one_major_version = { 90 v 91 for v in get_published_mz_versions_within_one_major_version() 92 if abs(v.major - current_version.major) <= 1 and v <= current_version 93 } 94 95 if current_version.major <= 26: 96 # For versions <= 26, we can only upgrade from 25.2 self-managed versions 97 self_managed_25_2_versions = { 98 v.version 99 for v in fetch_self_managed_versions() 100 if v.helm_version.major == 25 and v.helm_version.minor == 2 101 } 102 103 return sorted( 104 self_managed_25_2_versions.union( 105 published_versions_within_one_major_version 106 ) 107 ) 108 else: 109 # For versions > 26, get all mz versions within 1 major version of current_version 110 return sorted(published_versions_within_one_major_version)
BAD_SELF_MANAGED_VERSIONS =
{MzVersion(major=0, minor=130, patch=4, prerelease=None, build=None), MzVersion(major=0, minor=130, patch=0, prerelease=None, build=None), MzVersion(major=0, minor=147, patch=14, prerelease=None, build=None), MzVersion(major=0, minor=130, patch=1, prerelease=None, build=None), MzVersion(major=0, minor=147, patch=7, prerelease=None, build=None), MzVersion(major=0, minor=157, patch=0, prerelease=None, build=None), MzVersion(major=0, minor=130, patch=2, prerelease=None, build=None), MzVersion(major=0, minor=130, patch=3, prerelease=None, build=None)}
INVALID_VERSIONS =
{MzVersion(major=0, minor=55, patch=5, prerelease=None, build=None), MzVersion(major=0, minor=93, patch=0, prerelease=None, build=None), MzVersion(major=0, minor=57, patch=5, prerelease=None, build=None), MzVersion(major=0, minor=89, patch=7, prerelease=None, build=None), MzVersion(major=0, minor=113, patch=1, prerelease=None, build=None), MzVersion(major=0, minor=55, patch=2, prerelease=None, build=None), MzVersion(major=0, minor=55, patch=4, prerelease=None, build=None), MzVersion(major=0, minor=56, patch=0, prerelease=None, build=None), MzVersion(major=0, minor=52, patch=1, prerelease=None, build=None), MzVersion(major=0, minor=92, patch=0, prerelease=None, build=None), MzVersion(major=0, minor=57, patch=2, prerelease=None, build=None), MzVersion(major=0, minor=55, patch=6, prerelease=None, build=None), MzVersion(major=0, minor=57, patch=6, prerelease=None, build=None), MzVersion(major=0, minor=81, patch=0, prerelease=None, build=None), MzVersion(major=0, minor=99, patch=1, prerelease=None, build=None), MzVersion(major=0, minor=81, patch=2, prerelease=None, build=None), MzVersion(major=0, minor=55, patch=1, prerelease=None, build=None), MzVersion(major=0, minor=55, patch=3, prerelease=None, build=None), MzVersion(major=0, minor=57, patch=1, prerelease=None, build=None), MzVersion(major=0, minor=81, patch=1, prerelease=None, build=None)}
def
resolve_ancestor_image_tag( ancestor_overrides: dict[str, materialize.mz_version.MzVersion]) -> str | None:
155def resolve_ancestor_image_tag(ancestor_overrides: dict[str, MzVersion]) -> str | None: 156 """ 157 Resolve the ancestor image tag. 158 :param ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS 159 :return: image of the ancestor 160 """ 161 162 manual_ancestor_override = os.getenv("COMMON_ANCESTOR_OVERRIDE") 163 if manual_ancestor_override is not None: 164 image_tag = _manual_ancestor_specification_to_image_tag( 165 manual_ancestor_override 166 ) 167 print( 168 f"Using specified {image_tag} as image tag for ancestor (context: specified in $COMMON_ANCESTOR_OVERRIDE)" 169 ) 170 return image_tag 171 172 ancestor_image_resolution = _create_ancestor_image_resolution(ancestor_overrides) 173 result = ancestor_image_resolution.resolve_image_tag() 174 if result is None: 175 return None 176 image_tag, context = result 177 print(f"Using {image_tag} as image tag for ancestor (context: {context})") 178 return image_tag
Resolve the ancestor image tag.
Parameters
- ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS
Returns
image of the ancestor
class
AncestorImageResolutionBase:
197class AncestorImageResolutionBase: 198 def __init__(self, ancestor_overrides: dict[str, MzVersion]): 199 self.ancestor_overrides = ancestor_overrides 200 201 def resolve_image_tag(self) -> tuple[str, str] | None: 202 raise NotImplementedError 203 204 def _get_override_commit_instead_of_version( 205 self, 206 version: MzVersion, 207 ) -> str | None: 208 """ 209 If a commit specifies a mz version as prerequisite (to avoid regressions) that is newer than the provided 210 version (i.e., prerequisite not satisfied by the latest version), then return that commit's hash if the commit 211 contained in the current state. 212 Otherwise, return none. 213 """ 214 for ( 215 commit_hash, 216 min_required_mz_version, 217 ) in self.ancestor_overrides.items(): 218 if version >= min_required_mz_version: 219 continue 220 221 if git.contains_commit(commit_hash): 222 # commit would require at least min_required_mz_version 223 return commit_hash 224 225 return None 226 227 def _resolve_image_tag_of_previous_release( 228 self, context_prefix: str, previous_minor: bool 229 ) -> tuple[str, str] | None: 230 tagged_release_version = git.get_tagged_release_version(version_type=MzVersion) 231 assert tagged_release_version is not None 232 previous_release_version = get_previous_published_version( 233 tagged_release_version, previous_minor=previous_minor 234 ) 235 236 override_commit = self._get_override_commit_instead_of_version( 237 previous_release_version 238 ) 239 240 if override_commit is not None: 241 # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use 242 # use the commit instead of the previous release 243 # return ( 244 # commit_to_image_tag(override_commit), 245 # f"commit override instead of previous release ({previous_release_version})", 246 # ) 247 return None 248 249 return ( 250 release_version_to_image_tag(previous_release_version), 251 f"{context_prefix} {tagged_release_version}", 252 ) 253 254 def _resolve_image_tag_of_previous_release_from_current( 255 self, context: str 256 ) -> tuple[str, str] | None: 257 # Even though we are on main we might be in an older state, pick the 258 # latest release that was before our current version. 259 current_version = MzVersion.parse_cargo() 260 261 previous_published_version = get_previous_published_version( 262 current_version, previous_minor=True 263 ) 264 override_commit = self._get_override_commit_instead_of_version( 265 previous_published_version 266 ) 267 268 if override_commit is not None: 269 # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use 270 # use the commit instead of the latest release 271 # return ( 272 # commit_to_image_tag(override_commit), 273 # f"commit override instead of latest release ({previous_published_version})", 274 # ) 275 return None 276 277 return ( 278 release_version_to_image_tag(previous_published_version), 279 context, 280 ) 281 282 def _resolve_image_tag_of_merge_base( 283 self, 284 context_when_image_of_commit_exists: str, 285 ) -> tuple[str, str] | None: 286 # If the current PR has a known and accepted regression, don't compare 287 # against merge base of it 288 override_commit = self._get_override_commit_instead_of_version( 289 MzVersion.parse_cargo() 290 ) 291 common_ancestor_commit = buildkite.get_merge_base() 292 293 if override_commit is not None: 294 # TODO(def-): This currently doesn't work because we only tag the Optimized builds with tags like v0.164.0-dev.0--main.gc28d0061a6c9e63ee50a5f555c5d90373d006686, but not the Release builds we should use 295 # return ( 296 # commit_to_image_tag(override_commit), 297 # f"commit override instead of merge base ({common_ancestor_commit})", 298 # ) 299 return None 300 301 ancestors = git.get_first_parent_commits(common_ancestor_commit, limit=20) 302 for ancestor in ancestors: 303 if image_of_commit_exists(ancestor): 304 context = ( 305 context_when_image_of_commit_exists 306 if ancestor == common_ancestor_commit 307 else f"ancestor of {context_when_image_of_commit_exists} (walked back from {common_ancestor_commit[:12]})" 308 ) 309 return ( 310 commit_to_image_tag(ancestor), 311 context, 312 ) 313 314 return None
317class AncestorImageResolutionLocal(AncestorImageResolutionBase): 318 def resolve_image_tag(self) -> tuple[str, str] | None: 319 if build_context.is_on_release_version(): 320 return self._resolve_image_tag_of_previous_release( 321 "previous minor release because on local release branch", 322 previous_minor=True, 323 ) 324 elif build_context.is_on_main_branch(): 325 return self._resolve_image_tag_of_previous_release_from_current( 326 "previous release from current because on local main branch" 327 ) 328 else: 329 return self._resolve_image_tag_of_merge_base( 330 "merge base of local non-main branch", 331 )
def
resolve_image_tag(self) -> tuple[str, str] | None:
318 def resolve_image_tag(self) -> tuple[str, str] | None: 319 if build_context.is_on_release_version(): 320 return self._resolve_image_tag_of_previous_release( 321 "previous minor release because on local release branch", 322 previous_minor=True, 323 ) 324 elif build_context.is_on_main_branch(): 325 return self._resolve_image_tag_of_previous_release_from_current( 326 "previous release from current because on local main branch" 327 ) 328 else: 329 return self._resolve_image_tag_of_merge_base( 330 "merge base of local non-main branch", 331 )
Inherited Members
334class AncestorImageResolutionInBuildkite(AncestorImageResolutionBase): 335 def resolve_image_tag(self) -> tuple[str, str] | None: 336 if buildkite.is_in_pull_request(): 337 return self._resolve_image_tag_of_merge_base( 338 "merge base of pull request", 339 ) 340 elif build_context.is_on_release_version(): 341 return self._resolve_image_tag_of_previous_release( 342 "previous minor release because on release branch", previous_minor=True 343 ) 344 else: 345 return self._resolve_image_tag_of_previous_release_from_current( 346 "previous release from current because not in a pull request and not on a release branch", 347 )
def
resolve_image_tag(self) -> tuple[str, str] | None:
335 def resolve_image_tag(self) -> tuple[str, str] | None: 336 if buildkite.is_in_pull_request(): 337 return self._resolve_image_tag_of_merge_base( 338 "merge base of pull request", 339 ) 340 elif build_context.is_on_release_version(): 341 return self._resolve_image_tag_of_previous_release( 342 "previous minor release because on release branch", previous_minor=True 343 ) 344 else: 345 return self._resolve_image_tag_of_previous_release_from_current( 346 "previous release from current because not in a pull request and not on a release branch", 347 )
Inherited Members
def
get_latest_published_version() -> materialize.mz_version.MzVersion:
350def get_latest_published_version() -> MzVersion: 351 """Get the latest mz version, older than current state, for which an image is published.""" 352 excluded_versions = set() 353 current_version = MzVersion.parse_cargo() 354 355 while True: 356 latest_published_version = git.get_latest_version( 357 version_type=MzVersion, 358 excluded_versions=excluded_versions, 359 current_version=current_version, 360 ) 361 362 if is_valid_release_image(latest_published_version): 363 return latest_published_version 364 else: 365 print( 366 f"Skipping version {latest_published_version} (image not found), trying earlier version" 367 ) 368 excluded_versions.add(latest_published_version)
Get the latest mz version, older than current state, for which an image is published.
def
get_previous_published_version( release_version: materialize.mz_version.MzVersion, previous_minor: bool) -> materialize.mz_version.MzVersion:
371def get_previous_published_version( 372 release_version: MzVersion, previous_minor: bool 373) -> MzVersion: 374 """Get the highest preceding mz version to the specified version for which an image is published.""" 375 excluded_versions = set() 376 377 while True: 378 previous_published_version = get_previous_mz_version( 379 release_version, 380 previous_minor=previous_minor, 381 excluded_versions=excluded_versions, 382 ) 383 384 if is_valid_release_image(previous_published_version): 385 return previous_published_version 386 else: 387 print(f"Skipping version {previous_published_version} (image not found)") 388 excluded_versions.add(previous_published_version)
Get the highest preceding mz version to the specified version for which an image is published.
def
get_published_minor_mz_versions( newest_first: bool = True, limit: int | None = None, include_filter: Callable[[materialize.mz_version.MzVersion], bool] | None = None, exclude_current_minor_version: bool = False, max_version: materialize.mz_version.MzVersion | None = None) -> list[materialize.mz_version.MzVersion]:
391def get_published_minor_mz_versions( 392 newest_first: bool = True, 393 limit: int | None = None, 394 include_filter: Callable[[MzVersion], bool] | None = None, 395 exclude_current_minor_version: bool = False, 396 max_version: MzVersion | None = None, 397) -> list[MzVersion]: 398 """ 399 Get the latest patch version for every minor version. 400 Use this version if it is NOT important whether a tag was introduced before or after creating this branch. 401 402 See also: #get_minor_mz_versions_listed_in_docs() 403 """ 404 405 # sorted in descending order 406 all_versions = get_all_mz_versions(newest_first=True) 407 minor_versions: dict[str, MzVersion] = {} 408 409 version = MzVersion.parse_cargo() 410 current_version = f"{version.major}.{version.minor}" 411 412 # Note that this method must not apply limit_to_published_versions to a created list 413 # because in that case minor versions may get lost. 414 for version in all_versions: 415 if include_filter is not None and not include_filter(version): 416 # this version shall not be included 417 continue 418 419 if max_version is not None and version >= max_version: 420 continue 421 422 minor_version = f"{version.major}.{version.minor}" 423 424 if exclude_current_minor_version and minor_version == current_version: 425 continue 426 427 if minor_version in minor_versions.keys(): 428 # we already have a more recent version for this minor version 429 continue 430 431 if not is_valid_release_image(version): 432 # this version is not considered valid 433 continue 434 435 minor_versions[minor_version] = version 436 437 if limit is not None and len(minor_versions.keys()) == limit: 438 # collected enough versions 439 break 440 441 assert len(minor_versions) > 0 442 return sorted(minor_versions.values(), reverse=newest_first)
Get the latest patch version for every minor version. Use this version if it is NOT important whether a tag was introduced before or after creating this branch.
See also: #get_minor_mz_versions_listed_in_docs()
def
get_all_mz_versions(newest_first: bool = True) -> list[materialize.mz_version.MzVersion]:
445def get_all_mz_versions( 446 newest_first: bool = True, 447) -> list[MzVersion]: 448 """ 449 Get all mz versions based on git tags. Versions known to be invalid are excluded. 450 451 See also: #get_all_mz_versions_listed_in_docs 452 """ 453 return [ 454 version 455 for version in get_version_tags( 456 version_type=MzVersion, newest_first=newest_first 457 ) 458 if version not in INVALID_VERSIONS 459 # Exclude release candidates 460 and not version.prerelease 461 ]
Get all mz versions based on git tags. Versions known to be invalid are excluded.
See also: #get_all_mz_versions_listed_in_docs
def
get_all_published_mz_versions( newest_first: bool = True, limit: int | None = None) -> list[materialize.mz_version.MzVersion]:
464def get_all_published_mz_versions( 465 newest_first: bool = True, limit: int | None = None 466) -> list[MzVersion]: 467 """Get all mz versions based on git tags. This method ensures that images of the versions exist.""" 468 all_versions = get_all_mz_versions(newest_first=newest_first) 469 print(f"all_versions: {all_versions}") 470 return limit_to_published_versions(all_versions, limit)
Get all mz versions based on git tags. This method ensures that images of the versions exist.
def
get_published_mz_versions_within_one_major_version(newest_first: bool = True) -> list[materialize.mz_version.MzVersion]:
473def get_published_mz_versions_within_one_major_version( 474 newest_first: bool = True, 475) -> list[MzVersion]: 476 """Get all previous mz versions within one major version of the current version. Ensure that images of the versions exist.""" 477 current_version = MzVersion.parse_cargo() 478 all_versions = get_all_mz_versions(newest_first=newest_first) 479 versions_within_one_major_version = { 480 v 481 for v in all_versions 482 if abs(v.major - current_version.major) <= 1 and v <= current_version 483 } 484 485 return limit_to_published_versions(list(versions_within_one_major_version))
Get all previous mz versions within one major version of the current version. Ensure that images of the versions exist.
def
limit_to_published_versions( all_versions: list[materialize.mz_version.MzVersion], limit: int | None = None) -> list[materialize.mz_version.MzVersion]:
488def limit_to_published_versions( 489 all_versions: list[MzVersion], limit: int | None = None 490) -> list[MzVersion]: 491 """Remove versions for which no image is published.""" 492 versions = [] 493 494 for v in all_versions: 495 if is_valid_release_image(v): 496 versions.append(v) 497 498 if limit is not None and len(versions) == limit: 499 break 500 501 return versions
Remove versions for which no image is published.
def
get_previous_mz_version( version: materialize.mz_version.MzVersion, previous_minor: bool, excluded_versions: set[materialize.mz_version.MzVersion] | None = None) -> materialize.mz_version.MzVersion:
504def get_previous_mz_version( 505 version: MzVersion, 506 previous_minor: bool, 507 excluded_versions: set[MzVersion] | None = None, 508) -> MzVersion: 509 """Get the predecessor of the specified version based on git tags.""" 510 if excluded_versions is None: 511 excluded_versions = set() 512 513 if previous_minor: 514 version = MzVersion.create(version.major, version.minor, 0) 515 516 if version.prerelease is not None and len(version.prerelease) > 0: 517 # simply drop the prerelease, do not try to find a decremented version 518 found_version = MzVersion.create(version.major, version.minor, version.patch) 519 520 if found_version not in excluded_versions: 521 return found_version 522 else: 523 # start searching with this version 524 version = found_version 525 526 all_versions: list[MzVersion] = get_version_tags(version_type=type(version)) 527 all_suitable_previous_versions = [ 528 v 529 for v in all_versions 530 if v < version 531 and (v.prerelease is None or len(v.prerelease) == 0) 532 and v not in INVALID_VERSIONS 533 and v not in excluded_versions 534 ] 535 return max(all_suitable_previous_versions)
Get the predecessor of the specified version based on git tags.
def
is_valid_release_image(version: materialize.mz_version.MzVersion) -> bool:
538def is_valid_release_image(version: MzVersion) -> bool: 539 """ 540 Checks if a version is not known as an invalid version and has a published image. 541 Note that this method may take shortcuts on older versions. 542 """ 543 if version in INVALID_VERSIONS: 544 return False 545 546 if version < _SKIP_IMAGE_CHECK_BELOW_THIS_VERSION: 547 # optimization: assume that all versions older than this one are either valid or listed in INVALID_VERSIONS 548 return True 549 550 # This is a potentially expensive operation which pulls an image if it hasn't been pulled yet. 551 return docker.image_of_release_version_exists(version)
Checks if a version is not known as an invalid version and has a published image. Note that this method may take shortcuts on older versions.
def
get_commits_of_accepted_regressions_between_versions( ancestor_overrides: dict[str, materialize.mz_version.MzVersion], since_version_exclusive: materialize.mz_version.MzVersion, to_version_inclusive: materialize.mz_version.MzVersion) -> list[str]:
554def get_commits_of_accepted_regressions_between_versions( 555 ancestor_overrides: dict[str, MzVersion], 556 since_version_exclusive: MzVersion, 557 to_version_inclusive: MzVersion, 558) -> list[str]: 559 """ 560 Get commits of accepted regressions between both versions. 561 :param ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS 562 :return: commits 563 """ 564 565 assert since_version_exclusive <= to_version_inclusive 566 567 commits = [] 568 569 for ( 570 regression_introducing_commit, 571 first_version_with_regression, 572 ) in ancestor_overrides.items(): 573 if ( 574 since_version_exclusive 575 < first_version_with_regression 576 <= to_version_inclusive 577 ): 578 commits.append(regression_introducing_commit) 579 580 return commits
Get commits of accepted regressions between both versions.
Parameters
- ancestor_overrides: one of #ANCESTOR_OVERRIDES_FOR_PERFORMANCE_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_SCALABILITY_REGRESSIONS, #ANCESTOR_OVERRIDES_FOR_CORRECTNESS_REGRESSIONS
Returns
commits
class
VersionsFromDocs:
583class VersionsFromDocs: 584 """Materialize versions as listed in doc/user/content/releases 585 586 >>> len(VersionsFromDocs(respect_released_tag=True).all_versions()) > 0 587 True 588 589 >>> len(VersionsFromDocs(respect_released_tag=True).minor_versions()) > 0 590 True 591 592 >>> len(VersionsFromDocs(respect_released_tag=True).patch_versions(minor_version=MzVersion.parse_mz("v0.52.0"))) 593 4 594 595 >>> min(VersionsFromDocs(respect_released_tag=True).all_versions()) 596 MzVersion(major=0, minor=27, patch=0, prerelease=None, build=None) 597 """ 598 599 def __init__( 600 self, 601 respect_released_tag: bool, 602 respect_date: bool = False, 603 only_publish_helm_chart: bool = True, 604 skip_rc: bool = False, 605 ) -> None: 606 files = Path(MZ_ROOT / "doc" / "user" / "content" / "releases").glob("v*.md") 607 self.versions = [] 608 current_version = MzVersion.parse_cargo() 609 for f in files: 610 base = f.stem 611 metadata = frontmatter.load(f) 612 if respect_released_tag and not metadata.get("released", False): 613 continue 614 if only_publish_helm_chart and not metadata.get("publish_helm_chart", True): 615 continue 616 date: datetime.date = metadata["date"] 617 if respect_date and date > datetime.date.today(): 618 continue 619 620 current_patch = metadata.get("patch", 0) 621 current_rc = metadata.get("rc", 0) 622 623 if current_rc > 0: 624 if skip_rc: 625 continue 626 for rc in range(1, current_rc + 1): 627 version = MzVersion.parse_mz(f"{base}.{current_patch}-rc.{rc}") 628 if not respect_released_tag and version >= current_version: 629 continue 630 if version not in INVALID_VERSIONS: 631 self.versions.append(version) 632 else: 633 for patch in range(current_patch + 1): 634 version = MzVersion.parse_mz(f"{base}.{patch}") 635 if not respect_released_tag and version >= current_version: 636 continue 637 if version not in INVALID_VERSIONS: 638 self.versions.append(version) 639 640 assert len(self.versions) > 0 641 self.versions.sort() 642 643 def all_versions(self) -> list[MzVersion]: 644 return self.versions 645 646 def minor_versions(self) -> list[MzVersion]: 647 """Return the latest patch version for every minor version.""" 648 minor_versions = {} 649 for version in self.versions: 650 minor_versions[f"{version.major}.{version.minor}"] = version 651 652 assert len(minor_versions) > 0 653 return sorted(minor_versions.values()) 654 655 def patch_versions(self, minor_version: MzVersion) -> list[MzVersion]: 656 """Return all patch versions within the given minor version.""" 657 patch_versions = [] 658 for version in self.versions: 659 if ( 660 version.major == minor_version.major 661 and version.minor == minor_version.minor 662 ): 663 patch_versions.append(version) 664 665 assert len(patch_versions) > 0 666 return sorted(patch_versions)
Materialize versions as listed in doc/user/content/releases
>>> len(VersionsFromDocs(respect_released_tag=True).all_versions()) > 0
True
>>> len(VersionsFromDocs(respect_released_tag=True).minor_versions()) > 0
True
>>> len(VersionsFromDocs(respect_released_tag=True).patch_versions(minor_version=MzVersion.parse_mz("v0.52.0")))
4
>>> min(VersionsFromDocs(respect_released_tag=True).all_versions())
MzVersion(major=0, minor=27, patch=0, prerelease=None, build=None)
VersionsFromDocs( respect_released_tag: bool, respect_date: bool = False, only_publish_helm_chart: bool = True, skip_rc: bool = False)
599 def __init__( 600 self, 601 respect_released_tag: bool, 602 respect_date: bool = False, 603 only_publish_helm_chart: bool = True, 604 skip_rc: bool = False, 605 ) -> None: 606 files = Path(MZ_ROOT / "doc" / "user" / "content" / "releases").glob("v*.md") 607 self.versions = [] 608 current_version = MzVersion.parse_cargo() 609 for f in files: 610 base = f.stem 611 metadata = frontmatter.load(f) 612 if respect_released_tag and not metadata.get("released", False): 613 continue 614 if only_publish_helm_chart and not metadata.get("publish_helm_chart", True): 615 continue 616 date: datetime.date = metadata["date"] 617 if respect_date and date > datetime.date.today(): 618 continue 619 620 current_patch = metadata.get("patch", 0) 621 current_rc = metadata.get("rc", 0) 622 623 if current_rc > 0: 624 if skip_rc: 625 continue 626 for rc in range(1, current_rc + 1): 627 version = MzVersion.parse_mz(f"{base}.{current_patch}-rc.{rc}") 628 if not respect_released_tag and version >= current_version: 629 continue 630 if version not in INVALID_VERSIONS: 631 self.versions.append(version) 632 else: 633 for patch in range(current_patch + 1): 634 version = MzVersion.parse_mz(f"{base}.{patch}") 635 if not respect_released_tag and version >= current_version: 636 continue 637 if version not in INVALID_VERSIONS: 638 self.versions.append(version) 639 640 assert len(self.versions) > 0 641 self.versions.sort()
def
minor_versions(self) -> list[materialize.mz_version.MzVersion]:
646 def minor_versions(self) -> list[MzVersion]: 647 """Return the latest patch version for every minor version.""" 648 minor_versions = {} 649 for version in self.versions: 650 minor_versions[f"{version.major}.{version.minor}"] = version 651 652 assert len(minor_versions) > 0 653 return sorted(minor_versions.values())
Return the latest patch version for every minor version.
def
patch_versions( self, minor_version: materialize.mz_version.MzVersion) -> list[materialize.mz_version.MzVersion]:
655 def patch_versions(self, minor_version: MzVersion) -> list[MzVersion]: 656 """Return all patch versions within the given minor version.""" 657 patch_versions = [] 658 for version in self.versions: 659 if ( 660 version.major == minor_version.major 661 and version.minor == minor_version.minor 662 ): 663 patch_versions.append(version) 664 665 assert len(patch_versions) > 0 666 return sorted(patch_versions)
Return all patch versions within the given minor version.