Module materialize.xcompile

Support for cross-compiling to Linux.

Expand source code Browse git
# Copyright Materialize, Inc. and contributors. All rights reserved.
#
# Use of this software is governed by the Business Source License
# included in the LICENSE file at the root of this repository.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0.

"""Support for cross-compiling to Linux."""

import os
import platform
import sys
from enum import Enum
from typing import Dict, List, Optional

from materialize import ROOT, spawn


class Arch(Enum):
    """A CPU architecture."""

    X86_64 = "x86_64"
    """The 64-bit x86 architecture."""

    AARCH64 = "aarch64"
    """The 64-bit ARM architecture."""

    def __str__(self) -> str:
        return self.value

    def go_str(self) -> str:
        """Return the architecture name in Go nomenclature: amd64 or arm64."""
        if self == Arch.X86_64:
            return "amd64"
        elif self == Arch.AARCH64:
            return "arm64"
        else:
            raise RuntimeError("unreachable")

    @staticmethod
    def host() -> "Arch":
        if platform.machine() == "x86_64":
            return Arch.X86_64
        elif platform.machine() in ["aarch64", "arm64"]:
            return Arch.AARCH64
        else:
            raise RuntimeError(f"unknown host architecture {platform.machine()}")


def target(arch: Arch) -> str:
    """Construct a Linux target triple for the specified architecture."""
    return f"{arch}-unknown-linux-gnu"


def cargo(
    arch: Arch, subcommand: str, rustflags: List[str], channel: Optional[str] = None
) -> List[str]:
    """Construct a Cargo invocation for cross compiling.

    Args:
        arch: The CPU architecture to build for.
        subcommand: The Cargo subcommand to invoke.
        rustflags: Override the flags passed to the Rust compiler. If the list
            is empty, the default flags are used.
        channel: The Rust toolchain channel to use. Either None/"stable" or "nightly".

    Returns:
        A list of arguments specifying the beginning of the command to invoke.
    """
    _target = target(arch)
    _target_env = _target.upper().replace("-", "_")

    rustflags += ["-Clink-arg=-Wl,--compress-debug-sections=zlib"]

    if sys.platform == "darwin":
        _bootstrap_darwin(arch)
        sysroot = spawn.capture([f"{_target}-cc", "-print-sysroot"]).strip()
        rustflags += [f"-L{sysroot}/lib"]
        extra_env = {
            f"CMAKE_SYSTEM_NAME": "Linux",
            f"CARGO_TARGET_{_target_env}_LINKER": f"{_target}-cc",
            f"CARGO_TARGET_DIR": str(ROOT / "target-xcompile"),
            f"TARGET_AR": f"{_target}-ar",
            f"TARGET_CPP": f"{_target}-cpp",
            f"TARGET_CC": f"{_target}-cc",
            f"TARGET_CXX": f"{_target}-c++",
            f"TARGET_CXXSTDLIB": "static=stdc++",
            f"TARGET_LD": f"{_target}-ld",
            f"TARGET_RANLIB": f"{_target}-ranlib",
        }
    else:
        # NOTE(benesch): The required Rust flags have to be duplicated with
        # their definitions in ci/builder/Dockerfile because `rustc` has no way
        # to merge together Rust flags from different sources.
        rustflags += [
            "-Clink-arg=-fuse-ld=lld",
            f"-L/opt/x-tools/{_target}/{_target}/sysroot/lib",
        ]
        extra_env: Dict[str, str] = {}

    env = {
        **extra_env,
        "RUSTFLAGS": " ".join(rustflags),
    }

    return [
        *_enter_builder(arch, channel),
        "env",
        *(f"{k}={v}" for k, v in env.items()),
        "cargo",
        subcommand,
        "--target",
        _target,
    ]


def tool(arch: Arch, name: str, channel: Optional[str] = None) -> List[str]:
    """Constructs a cross-compiling binutils tool invocation.

    Args:
        arch: The CPU architecture to build for.
        name: The name of the binutils tool to invoke.
        channel: The Rust toolchain channel to use. Either None/"stable" or "nightly".

    Returns:
        A list of arguments specifying the beginning of the command to invoke.
    """
    if sys.platform == "darwin":
        _bootstrap_darwin(arch)
    return [
        *_enter_builder(arch, channel),
        f"{target(arch)}-{name}",
    ]


def _enter_builder(arch: Arch, channel: Optional[str] = None) -> List[str]:
    if "MZ_DEV_CI_BUILDER" in os.environ or sys.platform == "darwin":
        return []
    else:
        return [
            "env",
            f"MZ_DEV_CI_BUILDER_ARCH={arch}",
            "bin/ci-builder",
            "run",
            channel if channel else "stable",
        ]


def _bootstrap_darwin(arch: Arch) -> None:
    # Building in Docker for Mac is painfully slow, so we install a
    # cross-compiling toolchain on the host and use that instead.

    BOOTSTRAP_VERSION = "4"
    BOOTSTRAP_FILE = ROOT / "target-xcompile" / target(arch) / ".xcompile-bootstrap"
    try:
        contents = BOOTSTRAP_FILE.read_text()
    except FileNotFoundError:
        contents = ""
    if contents == BOOTSTRAP_VERSION:
        return

    spawn.runv(["brew", "install", f"materializeinc/crosstools/{target(arch)}"])
    spawn.runv(["rustup", "target", "add", target(arch)])

    BOOTSTRAP_FILE.parent.mkdir(parents=True, exist_ok=True)
    BOOTSTRAP_FILE.write_text(BOOTSTRAP_VERSION)

Functions

def cargo(arch: Arch, subcommand: str, rustflags: List[str], channel: Optional[str] = None) ‑> List[str]

Construct a Cargo invocation for cross compiling.

Args

arch
The CPU architecture to build for.
subcommand
The Cargo subcommand to invoke.
rustflags
Override the flags passed to the Rust compiler. If the list is empty, the default flags are used.
channel
The Rust toolchain channel to use. Either None/"stable" or "nightly".

Returns

A list of arguments specifying the beginning of the command to invoke.

Expand source code Browse git
def cargo(
    arch: Arch, subcommand: str, rustflags: List[str], channel: Optional[str] = None
) -> List[str]:
    """Construct a Cargo invocation for cross compiling.

    Args:
        arch: The CPU architecture to build for.
        subcommand: The Cargo subcommand to invoke.
        rustflags: Override the flags passed to the Rust compiler. If the list
            is empty, the default flags are used.
        channel: The Rust toolchain channel to use. Either None/"stable" or "nightly".

    Returns:
        A list of arguments specifying the beginning of the command to invoke.
    """
    _target = target(arch)
    _target_env = _target.upper().replace("-", "_")

    rustflags += ["-Clink-arg=-Wl,--compress-debug-sections=zlib"]

    if sys.platform == "darwin":
        _bootstrap_darwin(arch)
        sysroot = spawn.capture([f"{_target}-cc", "-print-sysroot"]).strip()
        rustflags += [f"-L{sysroot}/lib"]
        extra_env = {
            f"CMAKE_SYSTEM_NAME": "Linux",
            f"CARGO_TARGET_{_target_env}_LINKER": f"{_target}-cc",
            f"CARGO_TARGET_DIR": str(ROOT / "target-xcompile"),
            f"TARGET_AR": f"{_target}-ar",
            f"TARGET_CPP": f"{_target}-cpp",
            f"TARGET_CC": f"{_target}-cc",
            f"TARGET_CXX": f"{_target}-c++",
            f"TARGET_CXXSTDLIB": "static=stdc++",
            f"TARGET_LD": f"{_target}-ld",
            f"TARGET_RANLIB": f"{_target}-ranlib",
        }
    else:
        # NOTE(benesch): The required Rust flags have to be duplicated with
        # their definitions in ci/builder/Dockerfile because `rustc` has no way
        # to merge together Rust flags from different sources.
        rustflags += [
            "-Clink-arg=-fuse-ld=lld",
            f"-L/opt/x-tools/{_target}/{_target}/sysroot/lib",
        ]
        extra_env: Dict[str, str] = {}

    env = {
        **extra_env,
        "RUSTFLAGS": " ".join(rustflags),
    }

    return [
        *_enter_builder(arch, channel),
        "env",
        *(f"{k}={v}" for k, v in env.items()),
        "cargo",
        subcommand,
        "--target",
        _target,
    ]
def target(arch: Arch) ‑> str

Construct a Linux target triple for the specified architecture.

Expand source code Browse git
def target(arch: Arch) -> str:
    """Construct a Linux target triple for the specified architecture."""
    return f"{arch}-unknown-linux-gnu"
def tool(arch: Arch, name: str, channel: Optional[str] = None) ‑> List[str]

Constructs a cross-compiling binutils tool invocation.

Args

arch
The CPU architecture to build for.
name
The name of the binutils tool to invoke.
channel
The Rust toolchain channel to use. Either None/"stable" or "nightly".

Returns

A list of arguments specifying the beginning of the command to invoke.

Expand source code Browse git
def tool(arch: Arch, name: str, channel: Optional[str] = None) -> List[str]:
    """Constructs a cross-compiling binutils tool invocation.

    Args:
        arch: The CPU architecture to build for.
        name: The name of the binutils tool to invoke.
        channel: The Rust toolchain channel to use. Either None/"stable" or "nightly".

    Returns:
        A list of arguments specifying the beginning of the command to invoke.
    """
    if sys.platform == "darwin":
        _bootstrap_darwin(arch)
    return [
        *_enter_builder(arch, channel),
        f"{target(arch)}-{name}",
    ]

Classes

class Arch (value, names=None, *, module=None, qualname=None, type=None, start=1)

A CPU architecture.

Expand source code Browse git
class Arch(Enum):
    """A CPU architecture."""

    X86_64 = "x86_64"
    """The 64-bit x86 architecture."""

    AARCH64 = "aarch64"
    """The 64-bit ARM architecture."""

    def __str__(self) -> str:
        return self.value

    def go_str(self) -> str:
        """Return the architecture name in Go nomenclature: amd64 or arm64."""
        if self == Arch.X86_64:
            return "amd64"
        elif self == Arch.AARCH64:
            return "arm64"
        else:
            raise RuntimeError("unreachable")

    @staticmethod
    def host() -> "Arch":
        if platform.machine() == "x86_64":
            return Arch.X86_64
        elif platform.machine() in ["aarch64", "arm64"]:
            return Arch.AARCH64
        else:
            raise RuntimeError(f"unknown host architecture {platform.machine()}")

Ancestors

  • enum.Enum

Class variables

var AARCH64

The 64-bit ARM architecture.

var X86_64

The 64-bit x86 architecture.

Static methods

def host() ‑> Arch
Expand source code Browse git
@staticmethod
def host() -> "Arch":
    if platform.machine() == "x86_64":
        return Arch.X86_64
    elif platform.machine() in ["aarch64", "arm64"]:
        return Arch.AARCH64
    else:
        raise RuntimeError(f"unknown host architecture {platform.machine()}")

Methods

def go_str(self) ‑> str

Return the architecture name in Go nomenclature: amd64 or arm64.

Expand source code Browse git
def go_str(self) -> str:
    """Return the architecture name in Go nomenclature: amd64 or arm64."""
    if self == Arch.X86_64:
        return "amd64"
    elif self == Arch.AARCH64:
        return "arm64"
    else:
        raise RuntimeError("unreachable")