misc.python.materialize.cargo

  1# Copyright Materialize, Inc. and contributors. All rights reserved.
  2#
  3# Use of this software is governed by the Business Source License
  4# included in the LICENSE file at the root of this repository.
  5#
  6# As of the Change Date specified in that file, in accordance with
  7# the Business Source License, use of this software will be governed
  8# by the Apache License, Version 2.0.
  9
 10"""A pure Python metadata parser for Cargo, Rust's package manager.
 11
 12See the [Cargo][] documentation for details. Only the features that are presently
 13necessary to support this repository are implemented.
 14
 15[Cargo]: https://doc.rust-lang.org/cargo/
 16"""
 17
 18from functools import cache
 19from pathlib import Path
 20
 21import toml
 22
 23from materialize import git
 24
 25
 26class Crate:
 27    """A Cargo crate.
 28
 29    A crate directory must contain a `Cargo.toml` file with `package.name` and
 30    `package.version` keys.
 31
 32    Args:
 33        root: The path to the root of the workspace.
 34        path: The path to the crate directory.
 35
 36    Attributes:
 37        name: The name of the crate.
 38        version: The version of the crate.
 39        features: The features of the crate.
 40        path: The path to the crate.
 41        path_build_dependencies: The build dependencies which are declared
 42            using paths.
 43        path_dev_dependencies: The dev dependencies which are declared using
 44            paths.
 45        path_dependencies: The dependencies which are declared using paths.
 46        non_workspace_deps: The dependencies which are declared neither using
 47            paths nor with `workspace = true`, including target-specific ones,
 48            mapped to the sections that declare them.
 49        rust_version: The minimum Rust version declared in the crate, if any.
 50        bins: The names of all binaries in the crate.
 51        examples: The names of all examples in the crate.
 52    """
 53
 54    _inputs_cache: set[str] | None
 55
 56    def __init__(self, root: Path, path: Path):
 57        self.root = root
 58        self._inputs_cache = None
 59        with open(path / "Cargo.toml") as f:
 60            config = toml.load(f)
 61        self.name = config["package"]["name"]
 62        self.version_string = config["package"]["version"]
 63        self.features = config.get("features", {})
 64        self.path = path
 65        self.path_build_dependencies: set[str] = set()
 66        self.path_dev_dependencies: set[str] = set()
 67        self.path_dependencies: set[str] = set()
 68        self.non_workspace_deps: dict[str, list[str]] = {}
 69        for dep_type, field in [
 70            ("build-dependencies", self.path_build_dependencies),
 71            ("dev-dependencies", self.path_dev_dependencies),
 72            ("dependencies", self.path_dependencies),
 73        ]:
 74            if dep_type in config:
 75                for name, c in config[dep_type].items():
 76                    if isinstance(c, dict) and "path" in c:
 77                        field.add(c.get("package", name))
 78                    elif isinstance(c, dict) and c.get("workspace"):
 79                        pass
 80                    else:
 81                        self.non_workspace_deps.setdefault(name, []).append(dep_type)
 82        for target, target_config in config.get("target", {}).items():
 83            for dep_type in ("build-dependencies", "dev-dependencies", "dependencies"):
 84                for name, c in target_config.get(dep_type, {}).items():
 85                    if isinstance(c, dict) and ("path" in c or c.get("workspace")):
 86                        continue
 87                    self.non_workspace_deps.setdefault(name, []).append(
 88                        f"target.'{target}'.{dep_type}"
 89                    )
 90        self.rust_version: str | None = None
 91        try:
 92            self.rust_version = str(config["package"]["rust-version"])
 93        except KeyError:
 94            pass
 95        self.bins = []
 96        if "bin" in config:
 97            for bin in config["bin"]:
 98                self.bins.append(bin["name"])
 99        if config["package"].get("autobins", True):
100            if (path / "src" / "main.rs").exists():
101                self.bins.append(self.name)
102            for p in (path / "src" / "bin").glob("*.rs"):
103                self.bins.append(p.stem)
104            for p in (path / "src" / "bin").glob("*/main.rs"):
105                self.bins.append(p.parent.stem)
106        self.examples = []
107        if "example" in config:
108            for example in config["example"]:
109                self.examples.append(example["name"])
110        if config["package"].get("autoexamples", True):
111            for p in (path / "examples").glob("*.rs"):
112                self.examples.append(p.stem)
113            for p in (path / "examples").glob("*/main.rs"):
114                self.examples.append(p.parent.stem)
115
116    def inputs(self) -> set[str]:
117        """Compute the files that can impact the compilation of this crate.
118
119        Note that the returned list may have false positives (i.e., include
120        files that do not in fact impact the compilation of this crate), but it
121        is not believed to have false negatives.
122
123        Returns:
124            inputs: A list of input files, relative to the root of the
125                Cargo workspace.
126        """
127        # NOTE(benesch): it would be nice to have fine-grained tracking of only
128        # exactly the files that go into a Rust crate, but doing this properly
129        # requires parsing Rust code, and we don't want to force a dependency on
130        # a Rust toolchain for users running demos. Instead, we assume that all†
131        # files in a crate's directory are inputs to that crate.
132        #
133        # † As a development convenience, we omit mzcompose configuration files
134        # within a crate. This is technically incorrect if someone writes
135        # `include!("mzcompose.py")`, but that seems like a crazy thing to do.
136        if self._inputs_cache is not None:
137            return self._inputs_cache
138        return git.expand_globs(
139            self.root,
140            f"{self.path}/**",
141            f":(exclude){self.path}/mzcompose",
142            f":(exclude){self.path}/mzcompose.py",
143        )
144
145
146class Workspace:
147    """A Cargo workspace.
148
149    A workspace directory must contain a `Cargo.toml` file with a
150    `workspace.members` key.
151
152    Args:
153        root: The path to the root of the workspace.
154
155    Attributes:
156        root: The path to the root of the workspace.
157        crates: A mapping from name to crate definition.
158    """
159
160    def __init__(self, root: Path):
161        self.root = root
162        with open(root / "Cargo.toml") as f:
163            config = toml.load(f)
164
165        workspace_config = config["workspace"]
166
167        self.crates: dict[str, Crate] = {}
168        for path in workspace_config["members"]:
169            crate = Crate(root, root / path)
170            self.crates[crate.name] = crate
171        self.exclude: dict[str, Crate] = {}
172        for path in workspace_config.get("exclude", []):
173            if path.endswith("*") and (root / path.rstrip("*")).exists():
174                for item in (root / path.rstrip("*")).iterdir():
175                    if item.is_dir() and (item / "Cargo.toml").exists():
176                        crate = Crate(root, root / item)
177                        self.exclude[crate.name] = crate
178        self.all_crates = self.crates | self.exclude
179
180        self.default_members: list[str] = workspace_config.get("default-members", [])
181        self.workspace_dependencies: dict[str, object] = workspace_config.get(
182            "dependencies", {}
183        )
184
185        self.rust_version: str | None = None
186        try:
187            self.rust_version = workspace_config["package"].get("rust-version")
188        except KeyError:
189            pass
190
191    def crate_for_bin(self, bin: str) -> Crate:
192        """Find the crate containing the named binary.
193
194        Args:
195            bin: The name of the binary to find.
196
197        Raises:
198            ValueError: The named binary did not exist in exactly one crate in
199                the Cargo workspace.
200        """
201        out = None
202        for crate in self.crates.values():
203            for b in crate.bins:
204                if b == bin:
205                    if out is not None:
206                        raise ValueError(
207                            f"bin {bin} appears more than once in cargo workspace"
208                        )
209                    out = crate
210        if out is None:
211            raise ValueError(f"bin {bin} does not exist in cargo workspace")
212        return out
213
214    def crate_for_example(self, example: str) -> Crate:
215        """Find the crate containing the named example.
216
217        Args:
218            example: The name of the example to find.
219
220        Raises:
221            ValueError: The named example did not exist in exactly one crate in
222                the Cargo workspace.
223        """
224        out = None
225        for crate in self.crates.values():
226            for e in crate.examples:
227                if e == example:
228                    if out is not None:
229                        raise ValueError(
230                            f"example {example} appears more than once in cargo workspace"
231                        )
232                    out = crate
233        if out is None:
234            raise ValueError(f"example {example} does not exist in cargo workspace")
235        return out
236
237    def transitive_path_dependencies(
238        self, crate: Crate, dev: bool = False
239    ) -> set[Crate]:
240        """Collects the transitive path dependencies of the requested crate.
241
242        Note that only _path_ dependencies are collected. Other types of
243        dependencies, like registry or Git dependencies, are not collected.
244
245        Args:
246            crate: The crate object from which to start the dependency crawl.
247            dev: Whether to consider dev dependencies in the root crate.
248
249        Returns:
250            crate_set: A set of all of the crates in this Cargo workspace upon
251                which the input crate depended upon, whether directly or
252                transitively.
253
254        Raises:
255            IndexError: The input crate did not exist.
256        """
257        deps = set()
258
259        @cache
260        def visit(c: Crate) -> None:
261            deps.add(c)
262            for d in c.path_dependencies:
263                visit(self.crates[d])
264            for d in c.path_build_dependencies:
265                visit(self.crates[d])
266
267        visit(crate)
268        if dev:
269            for d in crate.path_dev_dependencies:
270                visit(self.crates[d])
271        return deps
272
273    def precompute_crate_inputs(self) -> None:
274        """Pre-fetch all crate input files in a single batched git call.
275
276        This replaces ~118 individual pairs of git subprocess calls with
277        a single pair, then partitions the results by crate path in Python.
278        """
279        from materialize import spawn
280
281        root = next(iter(self.all_crates.values())).root
282        # Use paths relative to root for git specs and partitioning, since
283        # git --relative outputs paths relative to cwd (root). Crate paths
284        # may be absolute when MZ_ROOT is an absolute path.
285        crate_rel_paths = sorted(
286            set(str(c.path.relative_to(root)) for c in self.all_crates.values())
287        )
288
289        specs = []
290        for p in crate_rel_paths:
291            specs.append(f"{p}/**")
292            specs.append(f":(exclude){p}/mzcompose")
293            specs.append(f":(exclude){p}/mzcompose.py")
294
295        empty_tree = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
296        diff_files = spawn.capture(
297            ["git", "diff", "--name-only", "-z", "--relative", empty_tree, "--"]
298            + specs,
299            cwd=root,
300        )
301        ls_files = spawn.capture(
302            ["git", "ls-files", "--others", "--exclude-standard", "-z", "--"] + specs,
303            cwd=root,
304        )
305        all_files = set(
306            f for f in (diff_files + ls_files).split("\0") if f.strip() != ""
307        )
308
309        # Partition files by crate path (longest match first for nested crates)
310        crate_file_map: dict[str, set[str]] = {p: set() for p in crate_rel_paths}
311        sorted_paths = sorted(crate_rel_paths, key=len, reverse=True)
312        for f in all_files:
313            for cp in sorted_paths:
314                if f.startswith(cp + "/"):
315                    crate_file_map[cp].add(f)
316                    break
317
318        # Inject cached results into each Crate object
319        for crate in self.all_crates.values():
320            rel = str(crate.path.relative_to(root))
321            crate._inputs_cache = crate_file_map.get(rel, set())