Module materialize.feature_benchmark.comparator

Expand source code Browse git
# Copyright Materialize, Inc. and contributors. All rights reserved.
#
# Use of this software is governed by the Business Source License
# included in the LICENSE file at the root of this repository.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0.

from typing import Generic, Protocol, TypeVar

from materialize.feature_benchmark.measurement import MeasurementType

T = TypeVar("T")


class Comparator(Generic[T]):
    def __init__(self, type: MeasurementType, name: str, threshold: float) -> None:
        self.name = name
        self.type = type
        self.threshold = threshold
        self._points: list[T] = []

    def append(self, point: T) -> None:
        self._points.append(point)

    def this(self) -> T:
        return self._points[0]

    def this_as_str(self) -> str:
        if self.this() is None:
            return "           None"
        else:
            return f"{self.this():>11.3f}"

    def other(self) -> T:
        return self._points[1]

    def other_as_str(self) -> str:
        if self.other() is None:
            return "           None"
        else:
            return f"{self.other():>11.3f}"

    def is_regression(self) -> bool:
        assert False

    def ratio(self) -> float | None:
        assert False

    def human_readable(self) -> str:
        return str(self)


class SuccessComparator(Comparator[float]):
    def is_regression(self) -> bool:
        return False


class RelativeThresholdComparator(Comparator[float | None]):
    def ratio(self) -> float | None:
        if self._points[0] is None or self._points[1] is None:
            return None
        else:
            return self._points[0] / self._points[1]

    def is_regression(self) -> bool:
        ratio = self.ratio()

        if ratio is None:
            return False
        if ratio > 1:
            return ratio - 1 > self.threshold
        else:
            return False

    def human_readable(self) -> str:
        ratio = self.ratio()
        if ratio is None:
            return "N/A"
        if ratio >= 2:
            return f"{ratio:3.1f} TIMES more/slower"
        elif ratio > 1:
            return f"{-(1-ratio)*100:3.1f} pct   more/slower"
        elif ratio == 1:
            return "          same"
        elif ratio > 0.5:
            return f"{(1-ratio)*100:3.1f} pct   less/faster"
        else:
            return f"{(1/ratio):3.1f} times less/faster"


class Overlappable(Protocol):
    def overlap(self, other: "Overlappable") -> float:
        ...


class OverlapComparator(Comparator[Overlappable]):
    def ratio(self) -> float:
        return self._points[0].overlap(other=self._points[1])

    def is_regression(self) -> bool:
        return self.ratio() < 1 - self.threshold

Classes

class Comparator (type: MeasurementType, name: str, threshold: float)

Abstract base class for generic types.

A generic type is typically declared by inheriting from this class parameterized with one or more type variables. For example, a generic mapping type might be defined as::

class Mapping(Generic[KT, VT]): def getitem(self, key: KT) -> VT: … # Etc.

This class can then be used as follows::

def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: try: return mapping[key] except KeyError: return default

Expand source code Browse git
class Comparator(Generic[T]):
    def __init__(self, type: MeasurementType, name: str, threshold: float) -> None:
        self.name = name
        self.type = type
        self.threshold = threshold
        self._points: list[T] = []

    def append(self, point: T) -> None:
        self._points.append(point)

    def this(self) -> T:
        return self._points[0]

    def this_as_str(self) -> str:
        if self.this() is None:
            return "           None"
        else:
            return f"{self.this():>11.3f}"

    def other(self) -> T:
        return self._points[1]

    def other_as_str(self) -> str:
        if self.other() is None:
            return "           None"
        else:
            return f"{self.other():>11.3f}"

    def is_regression(self) -> bool:
        assert False

    def ratio(self) -> float | None:
        assert False

    def human_readable(self) -> str:
        return str(self)

Ancestors

  • typing.Generic

Subclasses

Methods

def append(self, point: ~T) ‑> None
Expand source code Browse git
def append(self, point: T) -> None:
    self._points.append(point)
def human_readable(self) ‑> str
Expand source code Browse git
def human_readable(self) -> str:
    return str(self)
def is_regression(self) ‑> bool
Expand source code Browse git
def is_regression(self) -> bool:
    assert False
def other(self) ‑> ~T
Expand source code Browse git
def other(self) -> T:
    return self._points[1]
def other_as_str(self) ‑> str
Expand source code Browse git
def other_as_str(self) -> str:
    if self.other() is None:
        return "           None"
    else:
        return f"{self.other():>11.3f}"
def ratio(self) ‑> float | None
Expand source code Browse git
def ratio(self) -> float | None:
    assert False
def this(self) ‑> ~T
Expand source code Browse git
def this(self) -> T:
    return self._points[0]
def this_as_str(self) ‑> str
Expand source code Browse git
def this_as_str(self) -> str:
    if self.this() is None:
        return "           None"
    else:
        return f"{self.this():>11.3f}"
class OverlapComparator (type: MeasurementType, name: str, threshold: float)

Abstract base class for generic types.

A generic type is typically declared by inheriting from this class parameterized with one or more type variables. For example, a generic mapping type might be defined as::

class Mapping(Generic[KT, VT]): def getitem(self, key: KT) -> VT: … # Etc.

This class can then be used as follows::

def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: try: return mapping[key] except KeyError: return default

Expand source code Browse git
class OverlapComparator(Comparator[Overlappable]):
    def ratio(self) -> float:
        return self._points[0].overlap(other=self._points[1])

    def is_regression(self) -> bool:
        return self.ratio() < 1 - self.threshold

Ancestors

Methods

def is_regression(self) ‑> bool
Expand source code Browse git
def is_regression(self) -> bool:
    return self.ratio() < 1 - self.threshold
def ratio(self) ‑> float
Expand source code Browse git
def ratio(self) -> float:
    return self._points[0].overlap(other=self._points[1])
class Overlappable (*args, **kwargs)

Base class for protocol classes.

Protocol classes are defined as::

class Proto(Protocol):
    def meth(self) -> int:
        ...

Such classes are primarily used with static type checkers that recognize structural subtyping (static duck-typing), for example::

class C:
    def meth(self) -> int:
        return 0

def func(x: Proto) -> int:
    return x.meth()

func(C())  # Passes static type check

See PEP 544 for details. Protocol classes decorated with @typing.runtime_checkable act as simple-minded runtime protocols that check only the presence of given attributes, ignoring their type signatures. Protocol classes can be generic, they are defined as::

class GenProto(Protocol[T]):
    def meth(self) -> T:
        ...
Expand source code Browse git
class Overlappable(Protocol):
    def overlap(self, other: "Overlappable") -> float:
        ...

Ancestors

  • typing.Protocol
  • typing.Generic

Methods

def overlap(self, other: Overlappable) ‑> float
Expand source code Browse git
def overlap(self, other: "Overlappable") -> float:
    ...
class RelativeThresholdComparator (type: MeasurementType, name: str, threshold: float)

Abstract base class for generic types.

A generic type is typically declared by inheriting from this class parameterized with one or more type variables. For example, a generic mapping type might be defined as::

class Mapping(Generic[KT, VT]): def getitem(self, key: KT) -> VT: … # Etc.

This class can then be used as follows::

def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: try: return mapping[key] except KeyError: return default

Expand source code Browse git
class RelativeThresholdComparator(Comparator[float | None]):
    def ratio(self) -> float | None:
        if self._points[0] is None or self._points[1] is None:
            return None
        else:
            return self._points[0] / self._points[1]

    def is_regression(self) -> bool:
        ratio = self.ratio()

        if ratio is None:
            return False
        if ratio > 1:
            return ratio - 1 > self.threshold
        else:
            return False

    def human_readable(self) -> str:
        ratio = self.ratio()
        if ratio is None:
            return "N/A"
        if ratio >= 2:
            return f"{ratio:3.1f} TIMES more/slower"
        elif ratio > 1:
            return f"{-(1-ratio)*100:3.1f} pct   more/slower"
        elif ratio == 1:
            return "          same"
        elif ratio > 0.5:
            return f"{(1-ratio)*100:3.1f} pct   less/faster"
        else:
            return f"{(1/ratio):3.1f} times less/faster"

Ancestors

Methods

def human_readable(self) ‑> str
Expand source code Browse git
def human_readable(self) -> str:
    ratio = self.ratio()
    if ratio is None:
        return "N/A"
    if ratio >= 2:
        return f"{ratio:3.1f} TIMES more/slower"
    elif ratio > 1:
        return f"{-(1-ratio)*100:3.1f} pct   more/slower"
    elif ratio == 1:
        return "          same"
    elif ratio > 0.5:
        return f"{(1-ratio)*100:3.1f} pct   less/faster"
    else:
        return f"{(1/ratio):3.1f} times less/faster"
def is_regression(self) ‑> bool
Expand source code Browse git
def is_regression(self) -> bool:
    ratio = self.ratio()

    if ratio is None:
        return False
    if ratio > 1:
        return ratio - 1 > self.threshold
    else:
        return False
def ratio(self) ‑> float | None
Expand source code Browse git
def ratio(self) -> float | None:
    if self._points[0] is None or self._points[1] is None:
        return None
    else:
        return self._points[0] / self._points[1]
class SuccessComparator (type: MeasurementType, name: str, threshold: float)

Abstract base class for generic types.

A generic type is typically declared by inheriting from this class parameterized with one or more type variables. For example, a generic mapping type might be defined as::

class Mapping(Generic[KT, VT]): def getitem(self, key: KT) -> VT: … # Etc.

This class can then be used as follows::

def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: try: return mapping[key] except KeyError: return default

Expand source code Browse git
class SuccessComparator(Comparator[float]):
    def is_regression(self) -> bool:
        return False

Ancestors

Methods

def is_regression(self) ‑> bool
Expand source code Browse git
def is_regression(self) -> bool:
    return False