misc.python.materialize.cargo

  1# Copyright Materialize, Inc. and contributors. All rights reserved.
  2#
  3# Use of this software is governed by the Business Source License
  4# included in the LICENSE file at the root of this repository.
  5#
  6# As of the Change Date specified in that file, in accordance with
  7# the Business Source License, use of this software will be governed
  8# by the Apache License, Version 2.0.
  9
 10"""A pure Python metadata parser for Cargo, Rust's package manager.
 11
 12See the [Cargo][] documentation for details. Only the features that are presently
 13necessary to support this repository are implemented.
 14
 15[Cargo]: https://doc.rust-lang.org/cargo/
 16"""
 17
 18from functools import cache
 19from pathlib import Path
 20
 21import toml
 22
 23from materialize import git
 24
 25
 26class Crate:
 27    """A Cargo crate.
 28
 29    A crate directory must contain a `Cargo.toml` file with `package.name` and
 30    `package.version` keys.
 31
 32    Args:
 33        root: The path to the root of the workspace.
 34        path: The path to the crate directory.
 35
 36    Attributes:
 37        name: The name of the crate.
 38        version: The version of the crate.
 39        features: The features of the crate.
 40        path: The path to the crate.
 41        path_build_dependencies: The build dependencies which are declared
 42            using paths.
 43        path_dev_dependencies: The dev dependencies which are declared using
 44            paths.
 45        path_dependencies: The dependencies which are declared using paths.
 46        rust_version: The minimum Rust version declared in the crate, if any.
 47        bins: The names of all binaries in the crate.
 48        examples: The names of all examples in the crate.
 49    """
 50
 51    _inputs_cache: set[str] | None
 52
 53    def __init__(self, root: Path, path: Path):
 54        self.root = root
 55        self._inputs_cache = None
 56        with open(path / "Cargo.toml") as f:
 57            config = toml.load(f)
 58        self.name = config["package"]["name"]
 59        self.version_string = config["package"]["version"]
 60        self.features = config.get("features", {})
 61        self.path = path
 62        self.path_build_dependencies: set[str] = set()
 63        self.path_dev_dependencies: set[str] = set()
 64        self.path_dependencies: set[str] = set()
 65        for dep_type, field in [
 66            ("build-dependencies", self.path_build_dependencies),
 67            ("dev-dependencies", self.path_dev_dependencies),
 68            ("dependencies", self.path_dependencies),
 69        ]:
 70            if dep_type in config:
 71                field.update(
 72                    c.get("package", name)
 73                    for name, c in config[dep_type].items()
 74                    if "path" in c
 75                )
 76        self.rust_version: str | None = None
 77        try:
 78            self.rust_version = str(config["package"]["rust-version"])
 79        except KeyError:
 80            pass
 81        self.bins = []
 82        if "bin" in config:
 83            for bin in config["bin"]:
 84                self.bins.append(bin["name"])
 85        if config["package"].get("autobins", True):
 86            if (path / "src" / "main.rs").exists():
 87                self.bins.append(self.name)
 88            for p in (path / "src" / "bin").glob("*.rs"):
 89                self.bins.append(p.stem)
 90            for p in (path / "src" / "bin").glob("*/main.rs"):
 91                self.bins.append(p.parent.stem)
 92        self.examples = []
 93        if "example" in config:
 94            for example in config["example"]:
 95                self.examples.append(example["name"])
 96        if config["package"].get("autoexamples", True):
 97            for p in (path / "examples").glob("*.rs"):
 98                self.examples.append(p.stem)
 99            for p in (path / "examples").glob("*/main.rs"):
100                self.examples.append(p.parent.stem)
101
102    def inputs(self) -> set[str]:
103        """Compute the files that can impact the compilation of this crate.
104
105        Note that the returned list may have false positives (i.e., include
106        files that do not in fact impact the compilation of this crate), but it
107        is not believed to have false negatives.
108
109        Returns:
110            inputs: A list of input files, relative to the root of the
111                Cargo workspace.
112        """
113        # NOTE(benesch): it would be nice to have fine-grained tracking of only
114        # exactly the files that go into a Rust crate, but doing this properly
115        # requires parsing Rust code, and we don't want to force a dependency on
116        # a Rust toolchain for users running demos. Instead, we assume that all†
117        # files in a crate's directory are inputs to that crate.
118        #
119        # † As a development convenience, we omit mzcompose configuration files
120        # within a crate. This is technically incorrect if someone writes
121        # `include!("mzcompose.py")`, but that seems like a crazy thing to do.
122        if self._inputs_cache is not None:
123            return self._inputs_cache
124        return git.expand_globs(
125            self.root,
126            f"{self.path}/**",
127            f":(exclude){self.path}/mzcompose",
128            f":(exclude){self.path}/mzcompose.py",
129        )
130
131
132class Workspace:
133    """A Cargo workspace.
134
135    A workspace directory must contain a `Cargo.toml` file with a
136    `workspace.members` key.
137
138    Args:
139        root: The path to the root of the workspace.
140
141    Attributes:
142        crates: A mapping from name to crate definition.
143    """
144
145    def __init__(self, root: Path):
146        with open(root / "Cargo.toml") as f:
147            config = toml.load(f)
148
149        workspace_config = config["workspace"]
150
151        self.crates: dict[str, Crate] = {}
152        for path in workspace_config["members"]:
153            crate = Crate(root, root / path)
154            self.crates[crate.name] = crate
155        self.exclude: dict[str, Crate] = {}
156        for path in workspace_config.get("exclude", []):
157            if path.endswith("*") and (root / path.rstrip("*")).exists():
158                for item in (root / path.rstrip("*")).iterdir():
159                    if item.is_dir() and (item / "Cargo.toml").exists():
160                        crate = Crate(root, root / item)
161                        self.exclude[crate.name] = crate
162        self.all_crates = self.crates | self.exclude
163
164        self.default_members: list[str] = workspace_config.get("default-members", [])
165
166        self.rust_version: str | None = None
167        try:
168            self.rust_version = workspace_config["package"].get("rust-version")
169        except KeyError:
170            pass
171
172    def crate_for_bin(self, bin: str) -> Crate:
173        """Find the crate containing the named binary.
174
175        Args:
176            bin: The name of the binary to find.
177
178        Raises:
179            ValueError: The named binary did not exist in exactly one crate in
180                the Cargo workspace.
181        """
182        out = None
183        for crate in self.crates.values():
184            for b in crate.bins:
185                if b == bin:
186                    if out is not None:
187                        raise ValueError(
188                            f"bin {bin} appears more than once in cargo workspace"
189                        )
190                    out = crate
191        if out is None:
192            raise ValueError(f"bin {bin} does not exist in cargo workspace")
193        return out
194
195    def crate_for_example(self, example: str) -> Crate:
196        """Find the crate containing the named example.
197
198        Args:
199            example: The name of the example to find.
200
201        Raises:
202            ValueError: The named example did not exist in exactly one crate in
203                the Cargo workspace.
204        """
205        out = None
206        for crate in self.crates.values():
207            for e in crate.examples:
208                if e == example:
209                    if out is not None:
210                        raise ValueError(
211                            f"example {example} appears more than once in cargo workspace"
212                        )
213                    out = crate
214        if out is None:
215            raise ValueError(f"example {example} does not exist in cargo workspace")
216        return out
217
218    def transitive_path_dependencies(
219        self, crate: Crate, dev: bool = False
220    ) -> set[Crate]:
221        """Collects the transitive path dependencies of the requested crate.
222
223        Note that only _path_ dependencies are collected. Other types of
224        dependencies, like registry or Git dependencies, are not collected.
225
226        Args:
227            crate: The crate object from which to start the dependency crawl.
228            dev: Whether to consider dev dependencies in the root crate.
229
230        Returns:
231            crate_set: A set of all of the crates in this Cargo workspace upon
232                which the input crate depended upon, whether directly or
233                transitively.
234
235        Raises:
236            IndexError: The input crate did not exist.
237        """
238        deps = set()
239
240        @cache
241        def visit(c: Crate) -> None:
242            deps.add(c)
243            for d in c.path_dependencies:
244                visit(self.crates[d])
245            for d in c.path_build_dependencies:
246                visit(self.crates[d])
247
248        visit(crate)
249        if dev:
250            for d in crate.path_dev_dependencies:
251                visit(self.crates[d])
252        return deps
253
254    def precompute_crate_inputs(self) -> None:
255        """Pre-fetch all crate input files in a single batched git call.
256
257        This replaces ~118 individual pairs of git subprocess calls with
258        a single pair, then partitions the results by crate path in Python.
259        """
260        from materialize import spawn
261
262        root = next(iter(self.all_crates.values())).root
263        # Use paths relative to root for git specs and partitioning, since
264        # git --relative outputs paths relative to cwd (root). Crate paths
265        # may be absolute when MZ_ROOT is an absolute path.
266        crate_rel_paths = sorted(
267            set(str(c.path.relative_to(root)) for c in self.all_crates.values())
268        )
269
270        specs = []
271        for p in crate_rel_paths:
272            specs.append(f"{p}/**")
273            specs.append(f":(exclude){p}/mzcompose")
274            specs.append(f":(exclude){p}/mzcompose.py")
275
276        empty_tree = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
277        diff_files = spawn.capture(
278            ["git", "diff", "--name-only", "-z", "--relative", empty_tree, "--"]
279            + specs,
280            cwd=root,
281        )
282        ls_files = spawn.capture(
283            ["git", "ls-files", "--others", "--exclude-standard", "-z", "--"] + specs,
284            cwd=root,
285        )
286        all_files = set(
287            f for f in (diff_files + ls_files).split("\0") if f.strip() != ""
288        )
289
290        # Partition files by crate path (longest match first for nested crates)
291        crate_file_map: dict[str, set[str]] = {p: set() for p in crate_rel_paths}
292        sorted_paths = sorted(crate_rel_paths, key=len, reverse=True)
293        for f in all_files:
294            for cp in sorted_paths:
295                if f.startswith(cp + "/"):
296                    crate_file_map[cp].add(f)
297                    break
298
299        # Inject cached results into each Crate object
300        for crate in self.all_crates.values():
301            rel = str(crate.path.relative_to(root))
302            crate._inputs_cache = crate_file_map.get(rel, set())