diff --git a/PW_PLUGINS b/PW_PLUGINS index d789bf886..f588ded48 100644 --- a/PW_PLUGINS +++ b/PW_PLUGINS @@ -14,3 +14,4 @@ presubmit pw_presubmit.pigweed_presubmit main heap-viewer pw_allocator.heap_viewer main rpc pw_hdlc_lite.rpc_console main +package pw_package.pigweed_packages main diff --git a/docs/BUILD.gn b/docs/BUILD.gn index cc63f0652..50abaccb7 100644 --- a/docs/BUILD.gn +++ b/docs/BUILD.gn @@ -79,6 +79,7 @@ group("module_docs") { "$dir_pw_metric:docs", "$dir_pw_minimal_cpp_stdlib:docs", "$dir_pw_module:docs", + "$dir_pw_package:docs", "$dir_pw_polyfill:docs", "$dir_pw_preprocessor:docs", "$dir_pw_presubmit:docs", diff --git a/modules.gni b/modules.gni index e441a0f02..974c2a400 100644 --- a/modules.gni +++ b/modules.gni @@ -51,6 +51,7 @@ declare_args() { dir_pw_minimal_cpp_stdlib = get_path_info("pw_minimal_cpp_stdlib", "abspath") dir_pw_module = get_path_info("pw_module", "abspath") dir_pw_fuzzer = get_path_info("pw_fuzzer", "abspath") + dir_pw_package = get_path_info("pw_package", "abspath") dir_pw_polyfill = get_path_info("pw_polyfill", "abspath") dir_pw_preprocessor = get_path_info("pw_preprocessor", "abspath") dir_pw_presubmit = get_path_info("pw_presubmit", "abspath") diff --git a/pw_package/BUILD.gn b/pw_package/BUILD.gn new file mode 100644 index 000000000..dd021e82f --- /dev/null +++ b/pw_package/BUILD.gn @@ -0,0 +1,21 @@ +# Copyright 2020 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +import("//build_overrides/pigweed.gni") + +import("$dir_pw_docgen/docs.gni") + +pw_doc_group("docs") { + sources = [ "docs.rst" ] +} diff --git a/pw_package/docs.rst b/pw_package/docs.rst new file mode 100644 index 000000000..b3055db97 --- /dev/null +++ b/pw_package/docs.rst @@ -0,0 +1,110 @@ +.. _module-pw_package: + +========== +pw_package +========== +The package module provides a mechanism to install additional tools used by +Pigweed. Most Pigweed dependencies should be installed using +:ref:`module-pw_env_setup`. Examples of reasons packages should be managed using +this module instead are listed below. + +* The dependency is extremely large and not commonly used. +* The dependency has a number of compatible versions and we want to allow + downstream projects to pick a version rather than being forced to use ours. +* The dependency has license issues that make it complicated for Google to + include it directly as a submodule or distribute it as a CIPD package. +* The dependency needs to be "installed" into the system in some manner beyond + just extraction and thus isn't a good match for distribution with CIPD. + +----- +Usage +----- +The package module can be accessed through the ``pw package`` command. This +has several subcommands. + +``pw package list`` + Lists all the packages installed followed by all the packages available. + +``pw package install `` + Installs ````. Exactly how this works is package-dependent, + and packages can decide to do nothing because the package is current, do an + incremental update, or delete the current version and install anew. Use + ``--force`` to remove the package before installing. + +``pw package status `` + Indicates whether ```` is installed. + +``pw package remove `` + Removes ````. + +----------- +Configuring +----------- + +Compatibility +~~~~~~~~~~~~~ +Python 3 + +Adding a New Package +~~~~~~~~~~~~~~~~~~~~ +To add a new package create a class that subclasses ``Package`` from +``pw_package/package_manager.py``. + +.. code-block:: python + + class Package: + """Package to be installed. + + Subclass this to implement installation of a specific package. + """ + def __init__(self, name): + self._name = name + + @property + def name(self): + return self._name + + def install(self, path: pathlib.Path) -> None: + """Install the package at path. + + Install the package in path. Cannot assume this directory is empty—it + may need to be deleted or updated. + """ + + def remove(self, path: pathlib.Path) -> None: + """Remove the package from path. + + Removes the directory containing the package. For most packages this + should be sufficient to remove the package, and subclasses should not + need to override this package. + """ + if os.path.exists(path): + shutil.rmtree(path) + + def status(self, path: pathlib.Path) -> bool: + """Returns if package is installed at path and current. + + This method will be skipped if the directory does not exist. + """ + +There's also a helper class for retrieving specific revisions of Git +repositories in ``pw_package/git_repo.py``. + +Then call ``pw_package.package_manager.register(PackageClass)`` to register +the class with the package manager. + +Setting up a Project +~~~~~~~~~~~~~~~~~~~~ +To set up the package manager for a new project create a file like below and +add it to the ``PW_PLUGINS`` file (see :ref:`module-pw_cli` for details). This +file is based off of ``pw_package/pigweed_packages.py``. + +.. code-block:: python + + from pw_package import package_manager + # These modules register themselves so must be imported despite appearing + # unused. + from pw_package.packages import nanopb + + def main(argv=None) -> int: + return package_manager.run(**vars(package_manager.parse_args(argv))) diff --git a/pw_package/py/pw_package/git_repo.py b/pw_package/py/pw_package/git_repo.py new file mode 100644 index 000000000..4b983663a --- /dev/null +++ b/pw_package/py/pw_package/git_repo.py @@ -0,0 +1,71 @@ +# Copyright 2020 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Install and check status of Git repository-based packages.""" + +import os +import pathlib +import shutil +import subprocess +from typing import Union + +import pw_package.package_manager + +PathOrStr = Union[pathlib.Path, str] + + +def git_stdout(*args: PathOrStr, + show_stderr=False, + repo: PathOrStr = '.') -> str: + return subprocess.run(['git', '-C', repo, *args], + stdout=subprocess.PIPE, + stderr=None if show_stderr else subprocess.DEVNULL, + check=True).stdout.decode().strip() + + +def git(*args: PathOrStr, + repo: PathOrStr = '.') -> subprocess.CompletedProcess: + return subprocess.run(['git', '-C', repo, *args], check=True) + + +class GitRepo(pw_package.package_manager.Package): + """Install and check status of Git repository-based packages.""" + def __init__(self, url, commit, *args, **kwargs): + super().__init__(*args, **kwargs) + self._url = url + self._commit = commit + + def status(self, path: pathlib.Path) -> bool: + if not os.path.isdir(path / '.git'): + return False + + remote = git_stdout('remote', 'get-url', 'origin', repo=path) + commit = git_stdout('rev-parse', 'HEAD', repo=path) + status = git_stdout('status', '--porcelain=v1', repo=path) + return remote == self._url and commit == self._commit and not status + + def install(self, path: pathlib.Path) -> None: + # If already installed and at correct version exit now. + if self.status(path): + return + + # Otherwise delete current version and clone again. + if os.path.isdir(path): + shutil.rmtree(path) + + # --filter=blob:none means we don't get history, just the current + # revision. If we later run commands that need history it will be + # retrieved on-demand. For small repositories the effect is negligible + # but for large repositories this should be a significant improvement. + git('clone', '--filter=blob:none', self._url, path) + git('reset', '--hard', self._commit, repo=path) diff --git a/pw_package/py/pw_package/package_manager.py b/pw_package/py/pw_package/package_manager.py new file mode 100644 index 000000000..dd20b44d2 --- /dev/null +++ b/pw_package/py/pw_package/package_manager.py @@ -0,0 +1,147 @@ +# Copyright 2020 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Install and remove optional packages.""" + +import argparse +import logging +import os +import pathlib +import shutil +from typing import List + +_LOG: logging.Logger = logging.getLogger(__name__) + + +class Package: + """Package to be installed. + + Subclass this to implement installation of a specific package. + """ + def __init__(self, name): + self._name = name + + @property + def name(self): + return self._name + + def install(self, path: pathlib.Path) -> None: # pylint: disable=no-self-use + """Install the package at path. + + Install the package in path. Cannot assume this directory is empty—it + may need to be deleted or updated. + """ + + def remove(self, path: pathlib.Path) -> None: # pylint: disable=no-self-use + """Remove the package from path. + + Removes the directory containing the package. For most packages this + should be sufficient to remove the package, and subclasses should not + need to override this package. + """ + if os.path.exists(path): + shutil.rmtree(path) + + def status(self, path: pathlib.Path) -> bool: # pylint: disable=no-self-use + """Returns if package is installed at path and current. + + This method will be skipped if the directory does not exist. + """ + + +_PACKAGES = {} + + +def register(package_class: type) -> None: + obj = package_class() + _PACKAGES[obj.name] = obj + + +class PackageManager: + """Install and remove optional packages.""" + def __init__(self): + self._pkg_root: pathlib.Path = None + + def install(self, package: str, force=False): + pkg = _PACKAGES[package] + if force: + self.remove(package) + _LOG.info('Installing %s...', pkg.name) + pkg.install(self._pkg_root / pkg.name) + _LOG.info('Installing %s...done.', pkg.name) + return 0 + + def remove(self, package: str): # pylint: disable=no-self-use + pkg = _PACKAGES[package] + _LOG.info('Removing %s...', pkg.name) + pkg.remove(self._pkg_root / pkg.name) + _LOG.info('Removing %s...done.', pkg.name) + return 0 + + def status(self, package: str): # pylint: disable=no-self-use + pkg = _PACKAGES[package] + path = self._pkg_root / pkg.name + if os.path.isdir(path) and pkg.status(path): + _LOG.info('%s is installed.', pkg.name) + return 0 + + _LOG.info('%s is not installed.', pkg.name) + return -1 + + def list(self): # pylint: disable=no-self-use + _LOG.info('Installed packages:') + available = [] + for package in sorted(_PACKAGES.keys()): + pkg = _PACKAGES[package] + if pkg.status(self._pkg_root / pkg.name): + _LOG.info(' %s', pkg.name) + else: + available.append(pkg.name) + _LOG.info('') + + _LOG.info('Available packages:') + for pkg_name in available: + _LOG.info(' %s', pkg_name) + _LOG.info('') + + return 0 + + def run(self, command: str, pkg_root: pathlib.Path, **kwargs): + os.makedirs(pkg_root, exist_ok=True) + self._pkg_root = pkg_root + return getattr(self, command)(**kwargs) + + +def parse_args(argv: List[str] = None) -> argparse.Namespace: + parser = argparse.ArgumentParser("Manage packages.") + parser.add_argument( + '--package-root', + '-e', + dest='pkg_root', + type=pathlib.Path, + default=(pathlib.Path(os.environ['_PW_ACTUAL_ENVIRONMENT_ROOT']) / + 'packages'), + ) + subparsers = parser.add_subparsers(dest='command', required=True) + install = subparsers.add_parser('install') + install.add_argument('--force', '-f', action='store_true') + remove = subparsers.add_parser('remove') + status = subparsers.add_parser('status') + for cmd in (install, remove, status): + cmd.add_argument('package', choices=_PACKAGES.keys()) + _ = subparsers.add_parser('list') + return parser.parse_args(argv) + + +def run(**kwargs): + return PackageManager().run(**kwargs) diff --git a/pw_package/py/pw_package/packages/__init__.py b/pw_package/py/pw_package/packages/__init__.py new file mode 100644 index 000000000..2c8334fad --- /dev/null +++ b/pw_package/py/pw_package/packages/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. diff --git a/pw_package/py/pw_package/packages/nanopb.py b/pw_package/py/pw_package/packages/nanopb.py new file mode 100644 index 000000000..96955bde4 --- /dev/null +++ b/pw_package/py/pw_package/packages/nanopb.py @@ -0,0 +1,30 @@ +# Copyright 2020 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Install and check status of nanopb.""" + +import pw_package.git_repo +import pw_package.package_manager + + +class NanoPB(pw_package.git_repo.GitRepo): + """Install and check status of nanopb.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, + name='nanopb', + url='https://github.com/nanopb/nanopb.git', + commit='9f57cc871d8a025039019c2d2fde217591f4e30d', + **kwargs) + + +pw_package.package_manager.register(NanoPB) diff --git a/pw_package/py/pw_package/pigweed_packages.py b/pw_package/py/pw_package/pigweed_packages.py new file mode 100644 index 000000000..734b6d7cb --- /dev/null +++ b/pw_package/py/pw_package/pigweed_packages.py @@ -0,0 +1,29 @@ +# Copyright 2020 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Install and remove optional packages for Pigweed.""" + +import sys + +from pw_package import package_manager +# These modules register themselves so must be imported despite appearing +# unused. +from pw_package.packages import nanopb # pylint: disable=unused-import + + +def main(argv=None) -> int: + return package_manager.run(**vars(package_manager.parse_args(argv))) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/pw_package/py/setup.py b/pw_package/py/setup.py new file mode 100644 index 000000000..682e0130f --- /dev/null +++ b/pw_package/py/setup.py @@ -0,0 +1,26 @@ +# Copyright 2020 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""The pw_package package.""" + +import setuptools + +setuptools.setup( + name='pw_package', + version='0.0.1', + author='Pigweed Authors', + author_email='pigweed-developers@googlegroups.com', + description='Tools for installing optional packages', + install_requires=[], + packages=setuptools.find_packages(), +)