diff --git a/README.md b/README.md index 4576a3ac..dfdbcaa7 100644 --- a/README.md +++ b/README.md @@ -277,6 +277,120 @@ _PVECONTROL_COMPLETE=zsh_source pvecontrol > "${HOME}/.zsh/completions/_pvecontr _PVECONTROL_COMPLETE=fish_source pvecontrol > {$HOME}/.config/fish/completions/pvecontrol.fish ``` +--- + +# Proxmox Backup Server Control + +`pbscontrol` is a companion CLI tool to manage Proxmox Backup Server (PBS) instances via their API. It follows the same conventions and configuration style as `pvecontrol`. + +## Installation + +`pbscontrol` is bundled in the same package as `pvecontrol` and installed alongside it: + +```shell +pipx install pvecontrol +``` + +## Configuration + +`pbscontrol` reads its configuration from `$HOME/.config/pbscontrol/config.yaml`. The structure mirrors `pvecontrol`'s configuration, with a `servers` list instead of `clusters`. + +```yaml +servers: + - name: my-pbs + host: pbs.example.com + user: root@pam + password: my.password.is.weak + ssl_verify: false +``` + +### API tokens + +Authentication via API tokens is also supported: + +```yaml +servers: + - name: my-pbs + host: pbs.example.com + user: pbscontrol@pbs + token_name: mytoken + token_value: randomtokenvalue +``` + +### Better security + +As with `pvecontrol`, the shell command substitution syntax `$(...)` is supported in the `user`, `password`, `token_name`, and `token_value` fields: + +```yaml +servers: + - name: my-pbs + host: pbs.example.com + user: root@pam + password: $(pass show pbs/my-pbs) +``` + +### Connection options + +| Option | Default | Description | +|---|---|---| +| `port` | `8007` | PBS API port | +| `timeout` | `60` | Request timeout in seconds | +| `ssl_verify` | `false` | Verify TLS certificate | + +## Usage + +```shell +$ pbscontrol --help +Usage: pbscontrol [OPTIONS] COMMAND [ARGS]... + + Proxmox Backup Server control CLI, version: x.y.z + +Options: + -d, --debug + -o, --output [text|json|csv|yaml|md] + [default: text] + -s, --server NAME Proxmox Backup Server name as defined in + configuration [required] + --unicode / --no-unicode Use unicode characters for output + --color / --no-color Use colorized output + --help Show this message and exit. + +Commands: + status Show Proxmox Backup Server status + + Made with love by Enix.io +``` + +The simplest command to verify that `pbscontrol` is correctly configured is `status`: + +```shell +$ pbscontrol --server my-pbs status + + Name: my-pbs + Version: 4.1.1 + Datastores: + datastore1: 372.53 GiB/931.32 GiB (40.0%), available: 558.79 GiB, gc: ok + datastore2: 1.64 TiB/1.82 TiB (90.0%), available: 186.26 GiB, gc: ok +``` + +## Environment variables + +`pbscontrol` supports the following environment variables: +- `PBSCONTROL_SERVER`: the default server to use when no `-s` or `--server` option is specified. +- `PBSCONTROL_COLOR`: if set to `False`, it will disable all colorized output. +- `PBSCONTROL_UNICODE`: if set to `False`, it will disable all unicode output. + +## Shell completion + +```shell +# bash +_PBSCONTROL_COMPLETE=bash_source pbscontrol > "${BASH_COMPLETION_USER_DIR:-${XDG_DATA_HOME:-$HOME/.local/share}/bash-completion}/completions/pbscontrol" +# zsh +_PBSCONTROL_COMPLETE=zsh_source pbscontrol > "${HOME}/.zsh/completions/_pbscontrol" +# fish +_PBSCONTROL_COMPLETE=fish_source pbscontrol > {$HOME}/.config/fish/completions/pbscontrol.fish +``` + ## Development If you want to tinker with the code, all the required dependencies are listed in `requirements.txt`, and you can install them e.g. with pip: diff --git a/setup.cfg b/setup.cfg index 6b5c0d29..2ee30344 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,3 +25,4 @@ where=src [options.entry_points] console_scripts = pvecontrol = pvecontrol:main + pbscontrol = pbscontrol:main diff --git a/src/pbscontrol/__init__.py b/src/pbscontrol/__init__.py new file mode 100644 index 00000000..553ebc43 --- /dev/null +++ b/src/pbscontrol/__init__.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 + +import sys +import logging +import signal + +from types import SimpleNamespace +from importlib.metadata import version, PackageNotFoundError + +import click +import urllib3 + +from pbscontrol import actions +from pvecontrol.utils import OutputFormats + + +def get_leaf_command(cmd, ctx, args): + if len(args) == 0: + return cmd, [] + + parser = cmd.make_parser(ctx) + _, args_without_options, _ = parser.parse_args(list(args)) + + if len(args_without_options) == 0: + return cmd, args + + name, sub_cmd, sub_args = cmd.resolve_command(ctx, args_without_options) + if isinstance(sub_cmd, click.MultiCommand) and len(sub_args) > 0: + sub_ctx = sub_cmd.make_context(name, sub_args, parent=ctx) + return get_leaf_command(sub_cmd, sub_ctx, sub_args) + + return sub_cmd, sub_args + + +class IgnoreRequiredForHelp(click.Group): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.ignoring = False + + def _is_defaulting_to_help(self, ctx, args): + try: + leaf_cmd, leaf_args = get_leaf_command(self, ctx, args) + + if leaf_cmd is None or leaf_cmd is self: + return False + + return ( + "--help" in leaf_args + or (isinstance(leaf_cmd, click.MultiCommand) and not leaf_cmd.invoke_without_command) + or (leaf_cmd.no_args_is_help and len(leaf_args) == 0) + ) + except click.exceptions.UsageError: + return False + + def parse_args(self, ctx, args): + if self._is_defaulting_to_help(ctx, args): + self.ignoring = True + for param in self.params: + param.required = False + + return super().parse_args(ctx, args) + + def format_commands(self, ctx, formatter) -> None: + commands = [] + for subcommand in self.list_commands(ctx): + cmd = self.get_command(ctx, subcommand) + if cmd is None or cmd.hidden: + continue + + commands.append((subcommand, cmd)) + + if len(commands) > 0: + limit = formatter.width - 6 - max(len(cmd[0]) for cmd in commands) + + rows = [] + for subcommand, cmd in commands: + if not isinstance(cmd, click.MultiCommand): + cmd_help = cmd.get_short_help_str(limit) + rows.append((subcommand, cmd_help)) + continue + for subsubcommand in cmd.list_commands(ctx): + subcmd = cmd.get_command(ctx, subsubcommand) + if subcmd is None or subcmd.hidden: + continue + cmd_help = subcmd.get_short_help_str(limit) + rows.append((f"{subcommand} {subsubcommand}", cmd_help)) + + if rows: + with formatter.section("Commands"): + formatter.write_dl(rows) + + +try: + _version = version("pvecontrol") +except PackageNotFoundError: + _version = "unknown" + + +@click.group( + cls=IgnoreRequiredForHelp, + help=f"Proxmox Backup Server control CLI, version: {_version}", + epilog="Made with love by Enix.io", +) +@click.option("-d", "--debug", is_flag=True) +@click.option( + "-o", + "--output", + type=click.Choice([o.value for o in OutputFormats]), + show_default=True, + default=OutputFormats.TEXT.value, + callback=lambda *v: OutputFormats(v[2]), +) +@click.option( + "-s", + "--server", + metavar="NAME", + envvar="SERVER", + required=True, + help="Proxmox Backup Server name as defined in configuration", +) +@click.option( + "--unicode/--no-unicode", + envvar="UNICODE", + default=True, + help="Use unicode characters for output", +) +@click.option( + "--color/--no-color", + envvar="COLOR", + default=True, + help="Use colorized output", +) +@click.pass_context +def pbscontrol(ctx, debug, output, server, unicode, color): + signal.signal(signal.SIGINT, lambda *_: sys.exit(130)) + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + if not ctx.command.ignoring: + args = SimpleNamespace(output=output, server=server, unicode=unicode, color=color) + + logging.basicConfig(encoding="utf-8", level=logging.DEBUG if debug else logging.INFO) + logging.debug("Arguments: %s", args) + + ctx.ensure_object(dict) + ctx.obj["args"] = args + + +pbscontrol.add_command(cmd=actions.server.status, name="status") + + +def main(): + # pylint: disable=no-value-for-parameter + pbscontrol(auto_envvar_prefix="PBSCONTROL") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/pbscontrol/__main__.py b/src/pbscontrol/__main__.py new file mode 100644 index 00000000..b8ac4c73 --- /dev/null +++ b/src/pbscontrol/__main__.py @@ -0,0 +1,4 @@ +import pbscontrol + +if __name__ == "__main__": + pbscontrol.main() diff --git a/src/pbscontrol/actions/__init__.py b/src/pbscontrol/actions/__init__.py new file mode 100644 index 00000000..06a11bd1 --- /dev/null +++ b/src/pbscontrol/actions/__init__.py @@ -0,0 +1 @@ +from pbscontrol.actions import server diff --git a/src/pbscontrol/actions/server.py b/src/pbscontrol/actions/server.py new file mode 100644 index 00000000..8d525ea2 --- /dev/null +++ b/src/pbscontrol/actions/server.py @@ -0,0 +1,40 @@ +import click + +from humanize import naturalsize + +from pbscontrol.models.server import PBSServer +from pvecontrol.utils import OutputFormats, render_output + + +@click.command() +@click.pass_context +def status(ctx): + """Show Proxmox Backup Server status""" + pbs = PBSServer.create_from_config(ctx.obj["args"].server) + usage = pbs.datastore_usage + + if ctx.obj["args"].output == OutputFormats.TEXT: + print(f"""\n\ + Name: {pbs.name} + Version: {pbs.version.get('version', 'unknown')} + Datastores:""") + for ds in usage: + total = naturalsize(ds["total"], binary=True, format="%.2f") + used = naturalsize(ds["used"], binary=True, format="%.2f") + avail = naturalsize(ds["avail"], binary=True, format="%.2f") + percent = ds["used"] / ds["total"] * 100 if ds["total"] else 0 + print(f" {ds['store']}: {used}/{total} ({percent:.1f}%), available: {avail}, gc: {ds['gc-status']}") + print() + else: + render_table = [ + { + "store": ds["store"], + "total": ds["total"], + "used": ds["used"], + "avail": ds["avail"], + "percent": round(ds["used"] / ds["total"] * 100 if ds["total"] else 0, 1), + "gc-status": ds["gc-status"], + } + for ds in usage + ] + print(render_output(render_table, output=ctx.obj["args"].output)) diff --git a/src/pbscontrol/config.py b/src/pbscontrol/config.py new file mode 100644 index 00000000..6e8d387e --- /dev/null +++ b/src/pbscontrol/config.py @@ -0,0 +1,50 @@ +import logging +import sys +import confuse + +configtemplate = { + "servers": confuse.Sequence( # pylint: disable=abstract-class-instantiated + { + "name": str, + "host": str, + "user": str, + "password": confuse.Optional(str, None), + "token_name": confuse.Optional(str, None), + "token_value": confuse.Optional(str, None), + "proxy_certificate": confuse.Optional( + confuse.OneOf( + [ + str, + { + "cert": str, + "key": str, + }, + ] + ), + None, + ), + "port": confuse.Optional(int, default=8007), + "timeout": confuse.Optional(int, default=60), + "ssl_verify": confuse.Optional(bool, False), + } + ), +} + + +config = confuse.LazyConfig("pbscontrol", __name__) + + +def set_config(server_name): + validconfig = config.get(configtemplate) + logging.debug("configuration is %s", validconfig) + + serverconfig = False + for s in validconfig.servers: + if s.name == server_name: + serverconfig = s + if not serverconfig: + logging.error('No such server "%s"', server_name) + sys.exit(1) + logging.debug("serverconfig is %s", serverconfig) + + return serverconfig diff --git a/src/pbscontrol/models/__init__.py b/src/pbscontrol/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/pbscontrol/models/server.py b/src/pbscontrol/models/server.py new file mode 100644 index 00000000..1d410fd8 --- /dev/null +++ b/src/pbscontrol/models/server.py @@ -0,0 +1,61 @@ +import logging +import sys + +from proxmoxer import ProxmoxAPI +from requests.exceptions import SSLError + +from pvecontrol.utils import run_auth_commands +from pbscontrol.config import set_config + + +class PBSServer: + """Proxmox Backup Server""" + + def __init__(self, name, host, port, timeout, verify_ssl=False, **auth): + try: + self.api = ProxmoxAPI(host, port=port, timeout=timeout, verify_ssl=verify_ssl, service="pbs", **auth) + except SSLError as e: + print(e) + sys.exit(1) + self.name = name + self._initstatus() + + def _initstatus(self): + self.version = self.api.version.get() + self.datastores = self.api.admin.datastore.get() + + @staticmethod + def create_from_config(server_name): + logging.info("Proxmox Backup Server: %s", server_name) + + serverconfig = set_config(server_name) + auth = run_auth_commands(serverconfig) + return PBSServer( + serverconfig.name, + serverconfig.host, + port=serverconfig.port, + verify_ssl=serverconfig.ssl_verify, + timeout=serverconfig.timeout, + **auth, + ) + + @property + def datastore_usage(self): + usage = [] + for ds in self.datastores: + store = ds.get("store") + try: + status = self.api.admin.datastore(store).status.get() + except Exception as e: # pylint: disable=broad-exception-caught + logging.warning("Could not get status for datastore %s: %s", store, e) + status = {} + usage.append( + { + "store": store, + "total": status.get("total", 0), + "used": status.get("used", 0), + "avail": status.get("avail", 0), + "gc-status": status.get("gc-status", {}).get("error", "ok"), + } + ) + return usage diff --git a/src/tests/pbscontrol/__init__.py b/src/tests/pbscontrol/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/tests/pbscontrol/fixtures/__init__.py b/src/tests/pbscontrol/fixtures/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/tests/pbscontrol/fixtures/api.py b/src/tests/pbscontrol/fixtures/api.py new file mode 100644 index 00000000..08872c73 --- /dev/null +++ b/src/tests/pbscontrol/fixtures/api.py @@ -0,0 +1,77 @@ +import json +import requests +import responses + +PBS_BASE_URL = "https://host:8007" + + +def execute_route(routes, method, url, **kwargs): + print(f"{method} {url}") + print(f"params: {kwargs.get('params', {})}") + path = url.replace(PBS_BASE_URL, "") + assert path in routes, f"Route not found: {path}\nAvailable routes: {list(routes.keys())}" + + route = routes[path] + data = route(method, **kwargs) if callable(route) else route + + content = json.dumps({"data": data}) + print(content + "\n") + + return content + + +def create_response_wrapper(datastores): + routes = generate_routes(datastores) + + def wrapper(path, data=None, **kwargs): + kwargs["params"] = kwargs.get("params", {}) + url = PBS_BASE_URL + path + + if data is None: + body = execute_route(routes, "GET", url, **kwargs) + else: + body = json.dumps({"data": data}) + + responses.get(url, body=body) + + return wrapper + + +def generate_routes(datastores): + routes = { + "/api2/json/version": {"version": "4.1.1", "release": "4.1", "repoid": "aabbccdd"}, + "/api2/json/admin/datastore": [{"store": ds["store"], "path": ds["path"]} for ds in datastores], + **generate_datastore_status_routes(datastores), + } + + print("ROUTES:") + for route_path in routes.keys(): + print(route_path) + print("") + + return routes + + +def generate_datastore_status_routes(datastores): + return { + f"/api2/json/admin/datastore/{ds['store']}/status": { + "total": ds["total"], + "used": ds["used"], + "avail": ds["avail"], + "gc-status": ds.get("gc-status", {"upid": None}), + } + for ds in datastores + } + + +def fake_datastore(name, total=1_000_000_000_000, used=400_000_000_000, gc_error=None): + avail = total - used + gc_status = {"error": gc_error} if gc_error else {"upid": None} + return { + "store": name, + "path": f"/mnt/datastore/{name}", + "total": total, + "used": used, + "avail": avail, + "gc-status": gc_status, + } diff --git a/src/tests/pbscontrol/test_pbscontrol.py b/src/tests/pbscontrol/test_pbscontrol.py new file mode 100644 index 00000000..45a363c4 --- /dev/null +++ b/src/tests/pbscontrol/test_pbscontrol.py @@ -0,0 +1,18 @@ +from pbscontrol import pbscontrol, get_leaf_command +from pbscontrol.actions.server import status + + +def test_get_leaf_command(): + testcases = [ + (pbscontrol, [], []), + (pbscontrol, ["--debug"], ["--debug"]), + (status, ["status"], []), + (status, ["-o", "json", "status", "--help"], ["--help"]), + (None, ["foobar"], []), + ] + + for testcase in testcases: + ctx = pbscontrol.make_context(pbscontrol.name, list(testcase[1]), resilient_parsing=True) + leaf_cmd, leaf_args = get_leaf_command(pbscontrol, ctx, testcase[1]) + assert leaf_cmd == testcase[0] + assert leaf_args == testcase[2] diff --git a/src/tests/pbscontrol/test_server.py b/src/tests/pbscontrol/test_server.py new file mode 100644 index 00000000..f57beaef --- /dev/null +++ b/src/tests/pbscontrol/test_server.py @@ -0,0 +1,55 @@ +import responses + +from tests.pbscontrol.testcase import PBSControlTestcase + + +class PBSServerTestcase(PBSControlTestcase): + + def test_version(self): + assert self.server.version["version"] == "4.1.1" + assert self.server.version["release"] == "4.1" + + def test_datastores_loaded(self): + assert len(self.server.datastores) == len(self.datastores) + store_names = [ds["store"] for ds in self.server.datastores] + assert "datastore1" in store_names + assert "datastore2" in store_names + + @responses.activate + def test_datastore_usage(self): + for ds in self.datastores: + self.responses_get(f"/api2/json/admin/datastore/{ds['store']}/status") + + usage = self.server.datastore_usage + + assert len(usage) == len(self.datastores) + + ds1 = next(u for u in usage if u["store"] == "datastore1") + assert ds1["total"] == 1_000_000_000_000 + assert ds1["used"] == 400_000_000_000 + assert ds1["avail"] == 600_000_000_000 + assert ds1["gc-status"] == "ok" + + ds2 = next(u for u in usage if u["store"] == "datastore2") + assert ds2["total"] == 2_000_000_000_000 + assert ds2["used"] == 1_800_000_000_000 + assert ds2["avail"] == 200_000_000_000 + + @responses.activate + def test_datastore_usage_with_gc_error(self): + datastores_with_error = [ + {**self.datastores[0], "gc-status": {"error": "some gc error"}}, + self.datastores[1], + ] + from tests.pbscontrol.fixtures.api import create_response_wrapper + + responses_get = create_response_wrapper(datastores_with_error) + for ds in datastores_with_error: + responses_get(f"/api2/json/admin/datastore/{ds['store']}/status") + + usage = self.server.datastore_usage + ds1 = next(u for u in usage if u["store"] == "datastore1") + assert ds1["gc-status"] == "some gc error" + + ds2 = next(u for u in usage if u["store"] == "datastore2") + assert ds2["gc-status"] == "ok" diff --git a/src/tests/pbscontrol/testcase.py b/src/tests/pbscontrol/testcase.py new file mode 100644 index 00000000..4ce8ea3c --- /dev/null +++ b/src/tests/pbscontrol/testcase.py @@ -0,0 +1,38 @@ +import unittest + +from unittest.mock import patch + +import responses + +from tests.pbscontrol.fixtures.api import create_response_wrapper, fake_datastore +from pbscontrol.models.server import PBSServer + + +class PBSControlTestcase(unittest.TestCase): + + @responses.activate + def setUp(self): + self.datastores = [ + fake_datastore("datastore1", total=1_000_000_000_000, used=400_000_000_000), + fake_datastore("datastore2", total=2_000_000_000_000, used=1_800_000_000_000), + ] + + self.responses_get = create_response_wrapper(self.datastores) + + self.responses_get("/api2/json/version") + self.responses_get("/api2/json/admin/datastore") + for ds in self.datastores: + self.responses_get(f"/api2/json/admin/datastore/{ds['store']}/status") + + with patch("proxmoxer.backends.https.ProxmoxHTTPAuth") as mock_auth: + mock_auth_instance = mock_auth.return_value + mock_auth_instance.timeout = 1 + + self.server = PBSServer( + "test-pbs", + "host", + port=8007, + verify_ssl=False, + timeout=mock_auth_instance.timeout, + **{"user": "root@pam", "password": "password"}, + )