2019-05-04 01:52:39 +03:00
|
|
|
#!/usr/bin/env python
|
|
|
|
# Copyright (c) 2019-present, Facebook, Inc.
|
|
|
|
# All rights reserved.
|
|
|
|
#
|
|
|
|
# This source code is licensed under the BSD-style license found in the
|
|
|
|
# LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
# of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
|
|
|
|
|
|
|
|
from __future__ import absolute_import, division, print_function, unicode_literals
|
|
|
|
|
|
|
|
import argparse
|
|
|
|
import os
|
2019-05-04 01:52:39 +03:00
|
|
|
import shutil
|
2019-05-04 01:52:39 +03:00
|
|
|
import subprocess
|
|
|
|
import sys
|
|
|
|
|
fbcode_builder: getdeps: add fetch subcommand
Summary:
Adds a command that can be used to trigger a fetch for a project
```
$ ./opensource/fbcode_builder/getdeps.py fetch zstd
Cloning https://github.com/facebook/zstd.git...
---
+ git clone --depth=100 https://github.com/facebook/zstd.git /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git
Cloning into '/data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git'...
remote: Enumerating objects: 3816, done.
remote: Counting objects: 100% (3816/3816), done.
remote: Compressing objects: 100% (1415/1415), done.
remote: Total 3816 (delta 2556), reused 3312 (delta 2288), pack-reused 0
Receiving objects: 100% (3816/3816), 2.93 MiB | 9.59 MiB/s, done.
Resolving deltas: 100% (2556/2556), done.
Updating /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git -> v1.3.8
---
+ git -C /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git fetch origin v1.3.8
From https://github.com/facebook/zstd
* tag v1.3.8 -> FETCH_HEAD
---
+ git -C /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git checkout FETCH_HEAD
Note: checking out 'FETCH_HEAD'.
You are in 'detached HEAD' state. You can look around, make experimental
changes and commit them, and you can discard any commits you make in this
state without impacting any branches by performing another checkout.
If you want to create a new branch to retain commits you create, you may
do so (now or later) by using -b with the checkout command again. Example:
git checkout -b <new-branch-name>
HEAD is now at 470344d Merge pull request #1479 from facebook/visualTest
---
+ git -C /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git submodule update --init
```
Reviewed By: simpkins
Differential Revision: D14691008
fbshipit-source-id: 3afa391360518a08ebd6ff97f5b8b4993f10c4e8
2019-05-04 01:52:39 +03:00
|
|
|
from getdeps.buildopts import setup_build_options
|
2019-05-04 01:52:39 +03:00
|
|
|
from getdeps.load import load_project, manifests_in_dependency_order
|
2019-05-04 01:52:39 +03:00
|
|
|
from getdeps.manifest import ManifestParser
|
2019-05-04 01:52:39 +03:00
|
|
|
from getdeps.platform import HostType, context_from_host_tuple
|
2019-05-04 01:52:39 +03:00
|
|
|
from getdeps.subcmd import SubCmd, add_subcommands, cmd
|
|
|
|
|
|
|
|
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "getdeps"))
|
|
|
|
|
|
|
|
|
|
|
|
@cmd("validate-manifest", "parse a manifest and validate that it is correct")
|
|
|
|
class ValidateManifest(SubCmd):
|
|
|
|
def run(self, args):
|
|
|
|
try:
|
|
|
|
ManifestParser(file_name=args.file_name)
|
|
|
|
print("OK", file=sys.stderr)
|
|
|
|
return 0
|
|
|
|
except Exception as exc:
|
|
|
|
print("ERROR: %s" % str(exc), file=sys.stderr)
|
|
|
|
return 1
|
|
|
|
|
|
|
|
def setup_parser(self, parser):
|
|
|
|
parser.add_argument("file_name", help="path to the manifest file")
|
|
|
|
|
|
|
|
|
2019-05-04 01:52:39 +03:00
|
|
|
@cmd("show-host-type", "outputs the host type tuple for the host machine")
|
|
|
|
class ShowHostType(SubCmd):
|
|
|
|
def run(self, args):
|
|
|
|
host = HostType()
|
|
|
|
print("%s" % host.as_tuple_string())
|
|
|
|
return 0
|
|
|
|
|
|
|
|
|
fbcode_builder: getdeps: add fetch subcommand
Summary:
Adds a command that can be used to trigger a fetch for a project
```
$ ./opensource/fbcode_builder/getdeps.py fetch zstd
Cloning https://github.com/facebook/zstd.git...
---
+ git clone --depth=100 https://github.com/facebook/zstd.git /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git
Cloning into '/data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git'...
remote: Enumerating objects: 3816, done.
remote: Counting objects: 100% (3816/3816), done.
remote: Compressing objects: 100% (1415/1415), done.
remote: Total 3816 (delta 2556), reused 3312 (delta 2288), pack-reused 0
Receiving objects: 100% (3816/3816), 2.93 MiB | 9.59 MiB/s, done.
Resolving deltas: 100% (2556/2556), done.
Updating /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git -> v1.3.8
---
+ git -C /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git fetch origin v1.3.8
From https://github.com/facebook/zstd
* tag v1.3.8 -> FETCH_HEAD
---
+ git -C /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git checkout FETCH_HEAD
Note: checking out 'FETCH_HEAD'.
You are in 'detached HEAD' state. You can look around, make experimental
changes and commit them, and you can discard any commits you make in this
state without impacting any branches by performing another checkout.
If you want to create a new branch to retain commits you create, you may
do so (now or later) by using -b with the checkout command again. Example:
git checkout -b <new-branch-name>
HEAD is now at 470344d Merge pull request #1479 from facebook/visualTest
---
+ git -C /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git submodule update --init
```
Reviewed By: simpkins
Differential Revision: D14691008
fbshipit-source-id: 3afa391360518a08ebd6ff97f5b8b4993f10c4e8
2019-05-04 01:52:39 +03:00
|
|
|
@cmd("fetch", "fetch the code for a given project")
|
|
|
|
class FetchCmd(SubCmd):
|
|
|
|
def setup_parser(self, parser):
|
|
|
|
parser.add_argument(
|
|
|
|
"project",
|
|
|
|
help=(
|
|
|
|
"name of the project or path to a manifest "
|
|
|
|
"file describing the project"
|
|
|
|
),
|
|
|
|
)
|
2019-05-04 01:52:39 +03:00
|
|
|
parser.add_argument(
|
|
|
|
"--recursive",
|
|
|
|
help="fetch the transitive deps also",
|
|
|
|
action="store_true",
|
|
|
|
default=False,
|
|
|
|
)
|
|
|
|
parser.add_argument(
|
|
|
|
"--host-type",
|
|
|
|
help=(
|
|
|
|
"When recursively fetching, fetch deps for "
|
|
|
|
"this host type rather than the current system"
|
|
|
|
),
|
|
|
|
)
|
fbcode_builder: getdeps: add fetch subcommand
Summary:
Adds a command that can be used to trigger a fetch for a project
```
$ ./opensource/fbcode_builder/getdeps.py fetch zstd
Cloning https://github.com/facebook/zstd.git...
---
+ git clone --depth=100 https://github.com/facebook/zstd.git /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git
Cloning into '/data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git'...
remote: Enumerating objects: 3816, done.
remote: Counting objects: 100% (3816/3816), done.
remote: Compressing objects: 100% (1415/1415), done.
remote: Total 3816 (delta 2556), reused 3312 (delta 2288), pack-reused 0
Receiving objects: 100% (3816/3816), 2.93 MiB | 9.59 MiB/s, done.
Resolving deltas: 100% (2556/2556), done.
Updating /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git -> v1.3.8
---
+ git -C /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git fetch origin v1.3.8
From https://github.com/facebook/zstd
* tag v1.3.8 -> FETCH_HEAD
---
+ git -C /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git checkout FETCH_HEAD
Note: checking out 'FETCH_HEAD'.
You are in 'detached HEAD' state. You can look around, make experimental
changes and commit them, and you can discard any commits you make in this
state without impacting any branches by performing another checkout.
If you want to create a new branch to retain commits you create, you may
do so (now or later) by using -b with the checkout command again. Example:
git checkout -b <new-branch-name>
HEAD is now at 470344d Merge pull request #1479 from facebook/visualTest
---
+ git -C /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git submodule update --init
```
Reviewed By: simpkins
Differential Revision: D14691008
fbshipit-source-id: 3afa391360518a08ebd6ff97f5b8b4993f10c4e8
2019-05-04 01:52:39 +03:00
|
|
|
|
|
|
|
def run(self, args):
|
|
|
|
opts = setup_build_options(args)
|
2019-05-04 01:52:39 +03:00
|
|
|
manifest = load_project(opts, args.project)
|
|
|
|
ctx = context_from_host_tuple(args.host_type)
|
|
|
|
if args.recursive:
|
|
|
|
projects = manifests_in_dependency_order(opts, manifest, ctx)
|
|
|
|
else:
|
|
|
|
projects = [manifest]
|
|
|
|
for m in projects:
|
|
|
|
fetcher = m.create_fetcher(opts, ctx)
|
|
|
|
fetcher.update()
|
|
|
|
|
|
|
|
|
|
|
|
@cmd("list-deps", "lists the transitive deps for a given project")
|
|
|
|
class ListDepsCmd(SubCmd):
|
|
|
|
def run(self, args):
|
|
|
|
opts = setup_build_options(args)
|
|
|
|
manifest = load_project(opts, args.project)
|
|
|
|
ctx = context_from_host_tuple(args.host_type)
|
|
|
|
for m in manifests_in_dependency_order(opts, manifest, ctx):
|
|
|
|
print(m.name)
|
|
|
|
return 0
|
|
|
|
|
|
|
|
def setup_parser(self, parser):
|
|
|
|
parser.add_argument(
|
|
|
|
"--host-type",
|
|
|
|
help=(
|
|
|
|
"Produce the list for the specified host type, "
|
|
|
|
"rather than that of the current system"
|
|
|
|
),
|
|
|
|
)
|
|
|
|
parser.add_argument(
|
|
|
|
"project",
|
|
|
|
help=(
|
|
|
|
"name of the project or path to a manifest "
|
|
|
|
"file describing the project"
|
|
|
|
),
|
|
|
|
)
|
fbcode_builder: getdeps: add fetch subcommand
Summary:
Adds a command that can be used to trigger a fetch for a project
```
$ ./opensource/fbcode_builder/getdeps.py fetch zstd
Cloning https://github.com/facebook/zstd.git...
---
+ git clone --depth=100 https://github.com/facebook/zstd.git /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git
Cloning into '/data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git'...
remote: Enumerating objects: 3816, done.
remote: Counting objects: 100% (3816/3816), done.
remote: Compressing objects: 100% (1415/1415), done.
remote: Total 3816 (delta 2556), reused 3312 (delta 2288), pack-reused 0
Receiving objects: 100% (3816/3816), 2.93 MiB | 9.59 MiB/s, done.
Resolving deltas: 100% (2556/2556), done.
Updating /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git -> v1.3.8
---
+ git -C /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git fetch origin v1.3.8
From https://github.com/facebook/zstd
* tag v1.3.8 -> FETCH_HEAD
---
+ git -C /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git checkout FETCH_HEAD
Note: checking out 'FETCH_HEAD'.
You are in 'detached HEAD' state. You can look around, make experimental
changes and commit them, and you can discard any commits you make in this
state without impacting any branches by performing another checkout.
If you want to create a new branch to retain commits you create, you may
do so (now or later) by using -b with the checkout command again. Example:
git checkout -b <new-branch-name>
HEAD is now at 470344d Merge pull request #1479 from facebook/visualTest
---
+ git -C /data/users/wez/scratch/dataZusersZwezZfbsource/fbcode_builder_getdeps/repos/github.com-facebook-zstd.git submodule update --init
```
Reviewed By: simpkins
Differential Revision: D14691008
fbshipit-source-id: 3afa391360518a08ebd6ff97f5b8b4993f10c4e8
2019-05-04 01:52:39 +03:00
|
|
|
|
|
|
|
|
2019-05-04 01:52:39 +03:00
|
|
|
@cmd("build", "build a given project")
|
|
|
|
class BuildCmd(SubCmd):
|
|
|
|
def run(self, args):
|
|
|
|
opts = setup_build_options(args)
|
|
|
|
if args.clean:
|
|
|
|
for d in ["build", "installed", "extracted", "shipit"]:
|
|
|
|
d = os.path.join(opts.scratch_dir, d)
|
|
|
|
print("Cleaning %s..." % d)
|
|
|
|
if os.path.exists(d):
|
|
|
|
shutil.rmtree(d)
|
|
|
|
|
|
|
|
manifest = load_project(opts, args.project)
|
|
|
|
|
|
|
|
ctx = context_from_host_tuple()
|
|
|
|
projects = manifests_in_dependency_order(opts, manifest, ctx)
|
|
|
|
|
|
|
|
# Accumulate the install directories so that the build steps
|
|
|
|
# can find their dep installation
|
|
|
|
install_dirs = []
|
|
|
|
|
|
|
|
for m in projects:
|
|
|
|
print("Assessing %s..." % m.name)
|
|
|
|
fetcher = m.create_fetcher(opts, ctx)
|
|
|
|
|
|
|
|
if args.clean:
|
|
|
|
fetcher.clean()
|
|
|
|
change_status = fetcher.update()
|
|
|
|
reconfigure = change_status.build_changed()
|
|
|
|
sources_changed = change_status.sources_changed()
|
|
|
|
|
|
|
|
hash = fetcher.hash()
|
|
|
|
directory = "%s-%s" % (m.name, hash)
|
|
|
|
build_dir = os.path.join(opts.scratch_dir, "build", directory)
|
|
|
|
inst_dir = os.path.join(opts.scratch_dir, "installed", directory)
|
|
|
|
|
|
|
|
built_marker = os.path.join(inst_dir, ".built-by-getdeps")
|
|
|
|
if os.path.exists(built_marker):
|
|
|
|
with open(built_marker, "r") as f:
|
|
|
|
built_hash = f.read().strip()
|
|
|
|
if built_hash != hash:
|
|
|
|
# Some kind of inconsistency with a prior build,
|
|
|
|
# let's run it again to be sure
|
|
|
|
os.unlink(built_marker)
|
|
|
|
|
|
|
|
if sources_changed or reconfigure or not os.path.exists(built_marker):
|
|
|
|
if os.path.exists(built_marker):
|
|
|
|
os.unlink(built_marker)
|
|
|
|
src_dir = fetcher.get_src_dir()
|
|
|
|
builder = m.create_builder(opts, src_dir, build_dir, inst_dir, ctx)
|
|
|
|
builder.build(install_dirs, reconfigure=reconfigure)
|
|
|
|
|
|
|
|
with open(built_marker, "w") as f:
|
|
|
|
f.write(hash)
|
|
|
|
|
|
|
|
install_dirs.append(inst_dir)
|
|
|
|
|
|
|
|
def setup_parser(self, parser):
|
|
|
|
parser.add_argument(
|
|
|
|
"project",
|
|
|
|
help=(
|
|
|
|
"name of the project or path to a manifest "
|
|
|
|
"file describing the project"
|
|
|
|
),
|
|
|
|
)
|
|
|
|
parser.add_argument(
|
|
|
|
"--clean",
|
|
|
|
action="store_true",
|
|
|
|
default=False,
|
|
|
|
help=(
|
|
|
|
"Clean up the build and installation area prior to building, "
|
|
|
|
"causing the projects to be built from scratch"
|
|
|
|
),
|
|
|
|
)
|
|
|
|
|
|
|
|
|
2019-05-04 01:52:39 +03:00
|
|
|
def build_argparser():
|
2019-05-04 01:52:39 +03:00
|
|
|
common_args = argparse.ArgumentParser(add_help=False)
|
|
|
|
common_args.add_argument(
|
|
|
|
"--scratch-path", help="Where to maintain checkouts and build dirs"
|
|
|
|
)
|
|
|
|
common_args.add_argument(
|
|
|
|
"--install-prefix",
|
|
|
|
help=(
|
|
|
|
"Where the final build products will be installed "
|
|
|
|
"(default is [scratch-path]/installed)"
|
|
|
|
),
|
|
|
|
)
|
|
|
|
common_args.add_argument(
|
|
|
|
"--num-jobs",
|
|
|
|
type=int,
|
|
|
|
help=(
|
|
|
|
"Number of concurrent jobs to use while building. "
|
|
|
|
"(default=number of cpu cores)"
|
|
|
|
),
|
|
|
|
)
|
2019-05-04 01:52:39 +03:00
|
|
|
common_args.add_argument(
|
|
|
|
"--use-shipit",
|
|
|
|
help="use the real ShipIt instead of the simple shipit transformer",
|
|
|
|
action="store_true",
|
|
|
|
default=False,
|
|
|
|
)
|
2019-05-04 01:52:39 +03:00
|
|
|
|
|
|
|
ap = argparse.ArgumentParser(
|
|
|
|
description="Get and build dependencies and projects", parents=[common_args]
|
|
|
|
)
|
2019-05-04 01:52:39 +03:00
|
|
|
sub = ap.add_subparsers(
|
|
|
|
# metavar suppresses the long and ugly default list of subcommands on a
|
|
|
|
# single line. We still render the nicer list below where we would
|
|
|
|
# have shown the nasty one.
|
|
|
|
metavar="",
|
|
|
|
title="Available commands",
|
|
|
|
help="",
|
|
|
|
)
|
|
|
|
|
2019-05-04 01:52:39 +03:00
|
|
|
add_subcommands(sub, common_args)
|
2019-05-04 01:52:39 +03:00
|
|
|
|
|
|
|
return ap
|
|
|
|
|
|
|
|
|
|
|
|
def main():
|
|
|
|
ap = build_argparser()
|
|
|
|
args = ap.parse_args()
|
|
|
|
if getattr(args, "func", None) is None:
|
|
|
|
ap.print_help()
|
|
|
|
return 0
|
|
|
|
try:
|
|
|
|
return args.func(args)
|
|
|
|
except subprocess.CalledProcessError:
|
|
|
|
print("!! Failed", file=sys.stderr)
|
|
|
|
return 1
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
sys.exit(main())
|