misc.python.materialize.cargo

  1# Copyright Materialize, Inc. and contributors. All rights reserved.
  2#
  3# Use of this software is governed by the Business Source License
  4# included in the LICENSE file at the root of this repository.
  5#
  6# As of the Change Date specified in that file, in accordance with
  7# the Business Source License, use of this software will be governed
  8# by the Apache License, Version 2.0.
  9
 10"""A pure Python metadata parser for Cargo, Rust's package manager.
 11
 12See the [Cargo][] documentation for details. Only the features that are presently
 13necessary to support this repository are implemented.
 14
 15[Cargo]: https://doc.rust-lang.org/cargo/
 16"""
 17
 18from functools import cache
 19from pathlib import Path
 20
 21import toml
 22
 23from materialize import git
 24
 25
 26class Crate:
 27    """A Cargo crate.
 28
 29    A crate directory must contain a `Cargo.toml` file with `package.name` and
 30    `package.version` keys.
 31
 32    Args:
 33        root: The path to the root of the workspace.
 34        path: The path to the crate directory.
 35
 36    Attributes:
 37        name: The name of the crate.
 38        version: The version of the crate.
 39        features: The features of the crate.
 40        path: The path to the crate.
 41        path_build_dependencies: The build dependencies which are declared
 42            using paths.
 43        path_dev_dependencies: The dev dependencies which are declared using
 44            paths.
 45        path_dependencies: The dependencies which are declared using paths.
 46        rust_version: The minimum Rust version declared in the crate, if any.
 47        bins: The names of all binaries in the crate.
 48        examples: The names of all examples in the crate.
 49    """
 50
 51    _inputs_cache: set[str] | None
 52
 53    def __init__(self, root: Path, path: Path):
 54        self.root = root
 55        self._inputs_cache = None
 56        with open(path / "Cargo.toml") as f:
 57            config = toml.load(f)
 58        self.name = config["package"]["name"]
 59        self.version_string = config["package"]["version"]
 60        self.features = config.get("features", {})
 61        self.path = path
 62        self.path_build_dependencies: set[str] = set()
 63        self.path_dev_dependencies: set[str] = set()
 64        self.path_dependencies: set[str] = set()
 65        self.non_workspace_deps: dict[str, list[str]] = {}
 66        for dep_type, field in [
 67            ("build-dependencies", self.path_build_dependencies),
 68            ("dev-dependencies", self.path_dev_dependencies),
 69            ("dependencies", self.path_dependencies),
 70        ]:
 71            if dep_type in config:
 72                for name, c in config[dep_type].items():
 73                    if isinstance(c, dict) and "path" in c:
 74                        field.add(c.get("package", name))
 75                    elif isinstance(c, dict) and c.get("workspace"):
 76                        pass
 77                    else:
 78                        self.non_workspace_deps.setdefault(name, []).append(dep_type)
 79        self.rust_version: str | None = None
 80        try:
 81            self.rust_version = str(config["package"]["rust-version"])
 82        except KeyError:
 83            pass
 84        self.bins = []
 85        if "bin" in config:
 86            for bin in config["bin"]:
 87                self.bins.append(bin["name"])
 88        if config["package"].get("autobins", True):
 89            if (path / "src" / "main.rs").exists():
 90                self.bins.append(self.name)
 91            for p in (path / "src" / "bin").glob("*.rs"):
 92                self.bins.append(p.stem)
 93            for p in (path / "src" / "bin").glob("*/main.rs"):
 94                self.bins.append(p.parent.stem)
 95        self.examples = []
 96        if "example" in config:
 97            for example in config["example"]:
 98                self.examples.append(example["name"])
 99        if config["package"].get("autoexamples", True):
100            for p in (path / "examples").glob("*.rs"):
101                self.examples.append(p.stem)
102            for p in (path / "examples").glob("*/main.rs"):
103                self.examples.append(p.parent.stem)
104
105    def inputs(self) -> set[str]:
106        """Compute the files that can impact the compilation of this crate.
107
108        Note that the returned list may have false positives (i.e., include
109        files that do not in fact impact the compilation of this crate), but it
110        is not believed to have false negatives.
111
112        Returns:
113            inputs: A list of input files, relative to the root of the
114                Cargo workspace.
115        """
116        # NOTE(benesch): it would be nice to have fine-grained tracking of only
117        # exactly the files that go into a Rust crate, but doing this properly
118        # requires parsing Rust code, and we don't want to force a dependency on
119        # a Rust toolchain for users running demos. Instead, we assume that all†
120        # files in a crate's directory are inputs to that crate.
121        #
122        # † As a development convenience, we omit mzcompose configuration files
123        # within a crate. This is technically incorrect if someone writes
124        # `include!("mzcompose.py")`, but that seems like a crazy thing to do.
125        if self._inputs_cache is not None:
126            return self._inputs_cache
127        return git.expand_globs(
128            self.root,
129            f"{self.path}/**",
130            f":(exclude){self.path}/mzcompose",
131            f":(exclude){self.path}/mzcompose.py",
132        )
133
134
135class Workspace:
136    """A Cargo workspace.
137
138    A workspace directory must contain a `Cargo.toml` file with a
139    `workspace.members` key.
140
141    Args:
142        root: The path to the root of the workspace.
143
144    Attributes:
145        crates: A mapping from name to crate definition.
146    """
147
148    def __init__(self, root: Path):
149        with open(root / "Cargo.toml") as f:
150            config = toml.load(f)
151
152        workspace_config = config["workspace"]
153
154        self.crates: dict[str, Crate] = {}
155        for path in workspace_config["members"]:
156            crate = Crate(root, root / path)
157            self.crates[crate.name] = crate
158        self.exclude: dict[str, Crate] = {}
159        for path in workspace_config.get("exclude", []):
160            if path.endswith("*") and (root / path.rstrip("*")).exists():
161                for item in (root / path.rstrip("*")).iterdir():
162                    if item.is_dir() and (item / "Cargo.toml").exists():
163                        crate = Crate(root, root / item)
164                        self.exclude[crate.name] = crate
165        self.all_crates = self.crates | self.exclude
166
167        self.default_members: list[str] = workspace_config.get("default-members", [])
168        self.workspace_dependencies: dict[str, object] = workspace_config.get(
169            "dependencies", {}
170        )
171
172        self.rust_version: str | None = None
173        try:
174            self.rust_version = workspace_config["package"].get("rust-version")
175        except KeyError:
176            pass
177
178    def crate_for_bin(self, bin: str) -> Crate:
179        """Find the crate containing the named binary.
180
181        Args:
182            bin: The name of the binary to find.
183
184        Raises:
185            ValueError: The named binary did not exist in exactly one crate in
186                the Cargo workspace.
187        """
188        out = None
189        for crate in self.crates.values():
190            for b in crate.bins:
191                if b == bin:
192                    if out is not None:
193                        raise ValueError(
194                            f"bin {bin} appears more than once in cargo workspace"
195                        )
196                    out = crate
197        if out is None:
198            raise ValueError(f"bin {bin} does not exist in cargo workspace")
199        return out
200
201    def crate_for_example(self, example: str) -> Crate:
202        """Find the crate containing the named example.
203
204        Args:
205            example: The name of the example to find.
206
207        Raises:
208            ValueError: The named example did not exist in exactly one crate in
209                the Cargo workspace.
210        """
211        out = None
212        for crate in self.crates.values():
213            for e in crate.examples:
214                if e == example:
215                    if out is not None:
216                        raise ValueError(
217                            f"example {example} appears more than once in cargo workspace"
218                        )
219                    out = crate
220        if out is None:
221            raise ValueError(f"example {example} does not exist in cargo workspace")
222        return out
223
224    def transitive_path_dependencies(
225        self, crate: Crate, dev: bool = False
226    ) -> set[Crate]:
227        """Collects the transitive path dependencies of the requested crate.
228
229        Note that only _path_ dependencies are collected. Other types of
230        dependencies, like registry or Git dependencies, are not collected.
231
232        Args:
233            crate: The crate object from which to start the dependency crawl.
234            dev: Whether to consider dev dependencies in the root crate.
235
236        Returns:
237            crate_set: A set of all of the crates in this Cargo workspace upon
238                which the input crate depended upon, whether directly or
239                transitively.
240
241        Raises:
242            IndexError: The input crate did not exist.
243        """
244        deps = set()
245
246        @cache
247        def visit(c: Crate) -> None:
248            deps.add(c)
249            for d in c.path_dependencies:
250                visit(self.crates[d])
251            for d in c.path_build_dependencies:
252                visit(self.crates[d])
253
254        visit(crate)
255        if dev:
256            for d in crate.path_dev_dependencies:
257                visit(self.crates[d])
258        return deps
259
260    def precompute_crate_inputs(self) -> None:
261        """Pre-fetch all crate input files in a single batched git call.
262
263        This replaces ~118 individual pairs of git subprocess calls with
264        a single pair, then partitions the results by crate path in Python.
265        """
266        from materialize import spawn
267
268        root = next(iter(self.all_crates.values())).root
269        # Use paths relative to root for git specs and partitioning, since
270        # git --relative outputs paths relative to cwd (root). Crate paths
271        # may be absolute when MZ_ROOT is an absolute path.
272        crate_rel_paths = sorted(
273            set(str(c.path.relative_to(root)) for c in self.all_crates.values())
274        )
275
276        specs = []
277        for p in crate_rel_paths:
278            specs.append(f"{p}/**")
279            specs.append(f":(exclude){p}/mzcompose")
280            specs.append(f":(exclude){p}/mzcompose.py")
281
282        empty_tree = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
283        diff_files = spawn.capture(
284            ["git", "diff", "--name-only", "-z", "--relative", empty_tree, "--"]
285            + specs,
286            cwd=root,
287        )
288        ls_files = spawn.capture(
289            ["git", "ls-files", "--others", "--exclude-standard", "-z", "--"] + specs,
290            cwd=root,
291        )
292        all_files = set(
293            f for f in (diff_files + ls_files).split("\0") if f.strip() != ""
294        )
295
296        # Partition files by crate path (longest match first for nested crates)
297        crate_file_map: dict[str, set[str]] = {p: set() for p in crate_rel_paths}
298        sorted_paths = sorted(crate_rel_paths, key=len, reverse=True)
299        for f in all_files:
300            for cp in sorted_paths:
301                if f.startswith(cp + "/"):
302                    crate_file_map[cp].add(f)
303                    break
304
305        # Inject cached results into each Crate object
306        for crate in self.all_crates.values():
307            rel = str(crate.path.relative_to(root))
308            crate._inputs_cache = crate_file_map.get(rel, set())