Module materialize.feature_benchmark.comparator
Expand source code Browse git
# Copyright Materialize, Inc. and contributors. All rights reserved.
#
# Use of this software is governed by the Business Source License
# included in the LICENSE file at the root of this repository.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0.
from typing import Generic, Protocol, TypeVar
from materialize.feature_benchmark.measurement import MeasurementType
T = TypeVar("T")
class Comparator(Generic[T]):
def __init__(self, type: MeasurementType, name: str, threshold: float) -> None:
self.name = name
self.type = type
self.threshold = threshold
self._points: list[T] = []
def append(self, point: T) -> None:
self._points.append(point)
def this(self) -> T:
return self._points[0]
def this_as_str(self) -> str:
if self.this() is None:
return " None"
else:
return f"{self.this():>11.3f}"
def other(self) -> T:
return self._points[1]
def other_as_str(self) -> str:
if self.other() is None:
return " None"
else:
return f"{self.other():>11.3f}"
def is_regression(self) -> bool:
assert False
def ratio(self) -> float | None:
assert False
def human_readable(self) -> str:
return str(self)
class SuccessComparator(Comparator[float]):
def is_regression(self) -> bool:
return False
class RelativeThresholdComparator(Comparator[float | None]):
def ratio(self) -> float | None:
if self._points[0] is None or self._points[1] is None:
return None
else:
return self._points[0] / self._points[1]
def is_regression(self) -> bool:
ratio = self.ratio()
if ratio is None:
return False
if ratio > 1:
return ratio - 1 > self.threshold
else:
return False
def human_readable(self) -> str:
ratio = self.ratio()
if ratio is None:
return "N/A"
if ratio >= 2:
return f"{ratio:3.1f} TIMES more/slower"
elif ratio > 1:
return f"{-(1-ratio)*100:3.1f} pct more/slower"
elif ratio == 1:
return " same"
elif ratio > 0.5:
return f"{(1-ratio)*100:3.1f} pct less/faster"
else:
return f"{(1/ratio):3.1f} times less/faster"
class Overlappable(Protocol):
def overlap(self, other: "Overlappable") -> float:
...
class OverlapComparator(Comparator[Overlappable]):
def ratio(self) -> float:
return self._points[0].overlap(other=self._points[1])
def is_regression(self) -> bool:
return self.ratio() < 1 - self.threshold
Classes
class Comparator (type: MeasurementType, name: str, threshold: float)
-
Abstract base class for generic types.
A generic type is typically declared by inheriting from this class parameterized with one or more type variables. For example, a generic mapping type might be defined as::
class Mapping(Generic[KT, VT]): def getitem(self, key: KT) -> VT: … # Etc.
This class can then be used as follows::
def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: try: return mapping[key] except KeyError: return default
Expand source code Browse git
class Comparator(Generic[T]): def __init__(self, type: MeasurementType, name: str, threshold: float) -> None: self.name = name self.type = type self.threshold = threshold self._points: list[T] = [] def append(self, point: T) -> None: self._points.append(point) def this(self) -> T: return self._points[0] def this_as_str(self) -> str: if self.this() is None: return " None" else: return f"{self.this():>11.3f}" def other(self) -> T: return self._points[1] def other_as_str(self) -> str: if self.other() is None: return " None" else: return f"{self.other():>11.3f}" def is_regression(self) -> bool: assert False def ratio(self) -> float | None: assert False def human_readable(self) -> str: return str(self)
Ancestors
- typing.Generic
Subclasses
Methods
def append(self, point: ~T) ‑> None
-
Expand source code Browse git
def append(self, point: T) -> None: self._points.append(point)
def human_readable(self) ‑> str
-
Expand source code Browse git
def human_readable(self) -> str: return str(self)
def is_regression(self) ‑> bool
-
Expand source code Browse git
def is_regression(self) -> bool: assert False
def other(self) ‑> ~T
-
Expand source code Browse git
def other(self) -> T: return self._points[1]
def other_as_str(self) ‑> str
-
Expand source code Browse git
def other_as_str(self) -> str: if self.other() is None: return " None" else: return f"{self.other():>11.3f}"
def ratio(self) ‑> float | None
-
Expand source code Browse git
def ratio(self) -> float | None: assert False
def this(self) ‑> ~T
-
Expand source code Browse git
def this(self) -> T: return self._points[0]
def this_as_str(self) ‑> str
-
Expand source code Browse git
def this_as_str(self) -> str: if self.this() is None: return " None" else: return f"{self.this():>11.3f}"
class OverlapComparator (type: MeasurementType, name: str, threshold: float)
-
Abstract base class for generic types.
A generic type is typically declared by inheriting from this class parameterized with one or more type variables. For example, a generic mapping type might be defined as::
class Mapping(Generic[KT, VT]): def getitem(self, key: KT) -> VT: … # Etc.
This class can then be used as follows::
def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: try: return mapping[key] except KeyError: return default
Expand source code Browse git
class OverlapComparator(Comparator[Overlappable]): def ratio(self) -> float: return self._points[0].overlap(other=self._points[1]) def is_regression(self) -> bool: return self.ratio() < 1 - self.threshold
Ancestors
- Comparator
- typing.Generic
Methods
def is_regression(self) ‑> bool
-
Expand source code Browse git
def is_regression(self) -> bool: return self.ratio() < 1 - self.threshold
def ratio(self) ‑> float
-
Expand source code Browse git
def ratio(self) -> float: return self._points[0].overlap(other=self._points[1])
class Overlappable (*args, **kwargs)
-
Base class for protocol classes.
Protocol classes are defined as::
class Proto(Protocol): def meth(self) -> int: ...
Such classes are primarily used with static type checkers that recognize structural subtyping (static duck-typing), for example::
class C: def meth(self) -> int: return 0 def func(x: Proto) -> int: return x.meth() func(C()) # Passes static type check
See PEP 544 for details. Protocol classes decorated with @typing.runtime_checkable act as simple-minded runtime protocols that check only the presence of given attributes, ignoring their type signatures. Protocol classes can be generic, they are defined as::
class GenProto(Protocol[T]): def meth(self) -> T: ...
Expand source code Browse git
class Overlappable(Protocol): def overlap(self, other: "Overlappable") -> float: ...
Ancestors
- typing.Protocol
- typing.Generic
Methods
def overlap(self, other: Overlappable) ‑> float
-
Expand source code Browse git
def overlap(self, other: "Overlappable") -> float: ...
class RelativeThresholdComparator (type: MeasurementType, name: str, threshold: float)
-
Abstract base class for generic types.
A generic type is typically declared by inheriting from this class parameterized with one or more type variables. For example, a generic mapping type might be defined as::
class Mapping(Generic[KT, VT]): def getitem(self, key: KT) -> VT: … # Etc.
This class can then be used as follows::
def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: try: return mapping[key] except KeyError: return default
Expand source code Browse git
class RelativeThresholdComparator(Comparator[float | None]): def ratio(self) -> float | None: if self._points[0] is None or self._points[1] is None: return None else: return self._points[0] / self._points[1] def is_regression(self) -> bool: ratio = self.ratio() if ratio is None: return False if ratio > 1: return ratio - 1 > self.threshold else: return False def human_readable(self) -> str: ratio = self.ratio() if ratio is None: return "N/A" if ratio >= 2: return f"{ratio:3.1f} TIMES more/slower" elif ratio > 1: return f"{-(1-ratio)*100:3.1f} pct more/slower" elif ratio == 1: return " same" elif ratio > 0.5: return f"{(1-ratio)*100:3.1f} pct less/faster" else: return f"{(1/ratio):3.1f} times less/faster"
Ancestors
- Comparator
- typing.Generic
Methods
def human_readable(self) ‑> str
-
Expand source code Browse git
def human_readable(self) -> str: ratio = self.ratio() if ratio is None: return "N/A" if ratio >= 2: return f"{ratio:3.1f} TIMES more/slower" elif ratio > 1: return f"{-(1-ratio)*100:3.1f} pct more/slower" elif ratio == 1: return " same" elif ratio > 0.5: return f"{(1-ratio)*100:3.1f} pct less/faster" else: return f"{(1/ratio):3.1f} times less/faster"
def is_regression(self) ‑> bool
-
Expand source code Browse git
def is_regression(self) -> bool: ratio = self.ratio() if ratio is None: return False if ratio > 1: return ratio - 1 > self.threshold else: return False
def ratio(self) ‑> float | None
-
Expand source code Browse git
def ratio(self) -> float | None: if self._points[0] is None or self._points[1] is None: return None else: return self._points[0] / self._points[1]
class SuccessComparator (type: MeasurementType, name: str, threshold: float)
-
Abstract base class for generic types.
A generic type is typically declared by inheriting from this class parameterized with one or more type variables. For example, a generic mapping type might be defined as::
class Mapping(Generic[KT, VT]): def getitem(self, key: KT) -> VT: … # Etc.
This class can then be used as follows::
def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: try: return mapping[key] except KeyError: return default
Expand source code Browse git
class SuccessComparator(Comparator[float]): def is_regression(self) -> bool: return False
Ancestors
- Comparator
- typing.Generic
Methods
def is_regression(self) ‑> bool
-
Expand source code Browse git
def is_regression(self) -> bool: return False