From 07dd370f28c224aaa23d7d7ee46637e890426397 Mon Sep 17 00:00:00 2001 From: Lukasz Piatkowski Date: Thu, 13 Feb 2020 00:11:03 -0800 Subject: [PATCH] mononoke: add README.md and the missing pieces for supporting cargo (#13) Summary: Take the README.md from https://github.com/facebookexperimental/mononoke/blob/7ead0e29e41aec0771531a4650b6170bc1ff6566/README.md and apply it on Eden repo. Re-add the Cargo.toml file that declares Cargo workspace. Re-add fbcode_builder/getdeps manifest for Mononoke Pull Request resolved: https://github.com/facebookexperimental/eden/pull/13 Test Plan: ./build/fbcode_builder/getdeps.py build mononoke ./build/fbcode_builder/getdeps.py test mononoke Reviewed By: ahornby Differential Revision: D19833059 Pulled By: lukaspiatkowski fbshipit-source-id: fb37e13306c0b9969a7c4e52b05e1a66a577022f --- .gitignore | 4 +++ build/fbcode_builder/getdeps/builder.py | 41 ++++++++++++++++-------- build/fbcode_builder/getdeps/manifest.py | 7 +++- build/fbcode_builder/manifests/eden | 1 + build/fbcode_builder/manifests/mononoke | 37 +++++++++++++++++++++ eden/mononoke/Cargo.toml | 6 ++++ eden/mononoke/README.md | 23 +++++++++++++ 7 files changed, 104 insertions(+), 15 deletions(-) create mode 100644 build/fbcode_builder/manifests/mononoke create mode 100644 eden/mononoke/Cargo.toml create mode 100644 eden/mononoke/README.md diff --git a/.gitignore b/.gitignore index 4771a116f9..64561607a1 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,7 @@ /external # cmake /_build + +# Rust libraries and extensions +Cargo.lock +target/ diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index d64452e27b..a077a3c035 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -813,12 +813,22 @@ install(FILES sqlite3.h sqlite3ext.h DESTINATION include) class CargoBuilder(BuilderBase): def __init__( - self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, build_doc, loader + self, + build_opts, + ctx, + manifest, + src_dir, + build_dir, + inst_dir, + build_doc, + workspace_dir, + loader, ): super(CargoBuilder, self).__init__( build_opts, ctx, manifest, src_dir, build_dir, inst_dir ) self.build_doc = build_doc + self.ws_dir = workspace_dir self.loader = loader def run_cargo(self, install_dirs, operation, args=None): @@ -832,11 +842,14 @@ class CargoBuilder(BuilderBase): "--workspace", "-j%s" % self.build_opts.num_jobs, ] + args - self._run_cmd(cmd, cwd=self.build_source_dir(), env=env) + self._run_cmd(cmd, cwd=self.workspace_dir(), env=env) def build_source_dir(self): return os.path.join(self.build_dir, "source") + def workspace_dir(self): + return os.path.join(self.build_source_dir(), self.ws_dir) + def recreate_dir(self, src, dst): if os.path.isdir(dst): shutil.rmtree(dst) @@ -862,7 +875,7 @@ git-fetch-with-cli = true ) ) - self._patchup_workspace(build_source_dir) + self._patchup_workspace() try: from getdeps.facebook.rust import vendored_crates @@ -881,15 +894,14 @@ git-fetch-with-cli = true if self.build_doc: self.run_cargo(install_dirs, "doc", ["--no-deps"]) - def _patchup_workspace(self, build_source_dir): + def _patchup_workspace(self): """ - This method makes a lot of assumptions about the state of the project - and its cargo dependendies: - 1. There is a virtual manifest with workspace in the root of this project - 2. Crates from cargo dependencies can be extracted from Cargo.toml files + This method makes some assumptions about the state of the project and + its cargo dependendies: + 1. Crates from cargo dependencies can be extracted from Cargo.toml files using _extract_crates function. It is using a heuristic so check its code to understand how it is done. - 3. The extracted cargo dependencies crates can be found in the + 2. The extracted cargo dependencies crates can be found in the dependency's install dir using _resolve_crate_to_path function which again is using a heuristic. @@ -902,9 +914,10 @@ git-fetch-with-cli = true Exception. There migh be more cases where the code will silently pass producing bad results. """ - config = self._resolve_config(build_source_dir) + workspace_dir = self.workspace_dir() + config = self._resolve_config() if config: - with open(os.path.join(build_source_dir, "Cargo.toml"), "a") as f: + with open(os.path.join(workspace_dir, "Cargo.toml"), "a") as f: # A fake manifest has to be crated to change the virtual # manifest into a non-virtual. The virtual manifests are limited # in many ways and the inability to define patches on them is @@ -923,7 +936,7 @@ path = "/dev/null" ) f.write(config) - def _resolve_config(self, build_source_dir): + def _resolve_config(self): """ Returns a configuration to be put inside root Cargo.toml file which patches the dependencies git code with local getdeps versions. @@ -931,7 +944,7 @@ path = "/dev/null" """ dep_to_git = self._resolve_dep_to_git() dep_to_crates = CargoBuilder._resolve_dep_to_crates( - build_source_dir, dep_to_git + self.build_source_dir(), dep_to_git ) config = [] @@ -960,7 +973,7 @@ path = "/dev/null" """ For each direct dependency of the currently build manifest check if it is also cargo-builded and if yes then extract it's git configs and - install dir + install dir """ dependencies = self.manifest.get_section_as_dict("dependencies", ctx=self.ctx) if not dependencies: diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index 1b26b58dfa..6c9e25fb20 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -69,7 +69,10 @@ SCHEMA = { }, }, "msbuild": {"optional_section": True, "fields": {"project": REQUIRED}}, - "cargo": {"optional_section": True, "fields": {"build_doc": OPTIONAL}}, + "cargo": { + "optional_section": True, + "fields": {"build_doc": OPTIONAL, "workspace_dir": OPTIONAL}, + }, "cmake.defines": {"optional_section": True}, "autoconf.args": {"optional_section": True}, "b2.args": {"optional_section": True}, @@ -421,6 +424,7 @@ class ManifestParser(object): if builder == "cargo": build_doc = self.get("cargo", "build_doc", False, ctx) + workspace_dir = self.get("cargo", "workspace_dir", "", ctx) return CargoBuilder( build_options, ctx, @@ -429,6 +433,7 @@ class ManifestParser(object): build_dir, inst_dir, build_doc, + workspace_dir, loader, ) diff --git a/build/fbcode_builder/manifests/eden b/build/fbcode_builder/manifests/eden index 7fd209b94d..c04b3c30de 100644 --- a/build/fbcode_builder/manifests/eden +++ b/build/fbcode_builder/manifests/eden @@ -48,6 +48,7 @@ fbcode/tools/lfs = tools/lfs [shipit.strip] ^fbcode/eden/fs/eden-config\.h$ ^fbcode/eden/hg/.*$ +^fbcode/eden/mononoke/.*$ [cmake.defines.all(fb=on,os=windows)] INSTALL_PYTHON_LIB=ON diff --git a/build/fbcode_builder/manifests/mononoke b/build/fbcode_builder/manifests/mononoke new file mode 100644 index 0000000000..87a8d5f4ae --- /dev/null +++ b/build/fbcode_builder/manifests/mononoke @@ -0,0 +1,37 @@ +[manifest] +name = mononoke +fbsource_path = fbcode/eden +shipit_project = eden +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebookexperimental/eden.git + +[build.not(os=windows)] +builder = cargo + +[build.os=windows] +# building Mononoke on windows is not supported +builder = nop + +[cargo] +build_doc = true +workspace_dir = eden/mononoke + +[shipit.pathmap] +fbcode/eden/oss = . +fbcode/eden = eden +fbcode/eden/mononoke/public_autocargo = eden/mononoke +fbcode/tools/lfs = tools/lfs +tools/rust/ossconfigs = . + +[shipit.strip] +# strip all code unrelated to mononoke to prevent triggering unnecessary checks +^fbcode/eden/(?!mononoke)/.*$ +^fbcode/eden/mononoke/(?!public_autocargo).+/Cargo\.toml$ + +[dependencies] +rust-shed + +[dependencies.fb=on] +rust diff --git a/eden/mononoke/Cargo.toml b/eden/mononoke/Cargo.toml new file mode 100644 index 0000000000..ac91087f81 --- /dev/null +++ b/eden/mononoke/Cargo.toml @@ -0,0 +1,6 @@ +[workspace] + +members = [ + "server/session_id", + "sshrelay", +] diff --git a/eden/mononoke/README.md b/eden/mononoke/README.md new file mode 100644 index 0000000000..8415a5b0cd --- /dev/null +++ b/eden/mononoke/README.md @@ -0,0 +1,23 @@ +# Mononoke + +Mononoke is a next-generation server for the [Mercurial source control +system](https://www.mercurial-scm.org/), meant to scale up to accepting +thousands of commits every hour across millions of files. It is primarily +written in the [Rust programming language](https://www.rust-lang.org/en-US/). + +## Caveat Emptor + +Mononoke is still in early stages of development. We are making it available now because we plan to +start making references to it from our other open source projects. + +**The version that we provide on GitHub does not build yet**. + +This is because the code is exported verbatim from an internal repository at Facebook, and +not all of the scaffolding from our internal repository can be easily extracted. The key areas +where we need to shore things up are: + +* Full support for a standard `cargo build`. +* Open source replacements for Facebook-internal services (blob store, logging etc). + +The current goal is to get Mononoke working on Linux. Other Unix-like OSes may +be supported in the future