From b0fa1b7b2536fc6e66fa1cbd740620ce7467a1a0 Mon Sep 17 00:00:00 2001 From: ge Date: Sun, 3 Dec 2023 23:25:34 +0300 Subject: [PATCH] various improvements --- Makefile | 16 +- README.md | 20 +- compute/__init__.py | 6 +- compute/__main__.py | 4 +- compute/cli/control.py | 285 +++++++++++++++++------- compute/common.py | 4 +- compute/exceptions.py | 27 ++- compute/instance/__init__.py | 4 +- compute/instance/devices.py | 128 +++++++++++ compute/instance/guest_agent.py | 4 +- compute/instance/instance.py | 78 +++++-- compute/instance/schemas.py | 41 +++- compute/session.py | 80 ++++--- compute/storage/__init__.py | 6 +- compute/storage/pool.py | 4 +- compute/storage/volume.py | 55 +---- compute/utils/config_loader.py | 4 +- compute/utils/dictutil.py | 77 +++++++ compute/utils/ids.py | 4 +- compute/utils/units.py | 13 +- docs/source/_static/custom.css | 1 + docs/source/cli/index.rst | 8 + docs/source/cli/reference.rst | 7 + docs/source/cli/usage.rst | 102 +++++++++ docs/source/conf.py | 5 +- docs/source/index.rst | 9 +- docs/source/installation.rst | 47 ++++ docs/source/pyapi/index.rst | 34 +-- docs/source/pyapi/instance/devices.rst | 5 + docs/source/pyapi/instance/index.rst | 1 + docs/source/pyapi/utils.rst | 7 + packaging/Dockerfile | 1 + packaging/files/compute.bash-completion | 10 +- packaging/files/control | 1 + poetry.lock | 20 +- pyproject.toml | 1 + 36 files changed, 842 insertions(+), 277 deletions(-) create mode 100644 compute/instance/devices.py create mode 100644 compute/utils/dictutil.py create mode 100644 docs/source/_static/custom.css create mode 100644 docs/source/cli/index.rst create mode 100644 docs/source/cli/reference.rst create mode 100644 docs/source/cli/usage.rst create mode 100644 docs/source/installation.rst create mode 100644 docs/source/pyapi/instance/devices.rst diff --git a/Makefile b/Makefile index be82623..249c194 100644 --- a/Makefile +++ b/Makefile @@ -10,14 +10,17 @@ all: docs build-deb requirements.txt: poetry export -f requirements.txt -o requirements.txt -build: format lint - awk '/^version/{print $$3}' pyproject.toml \ - | xargs -I {} sed "s/__version__ =.*/__version__ = '{}'/" -i $(SRCDIR)/__init__.py +build: version format lint poetry build build-deb: build cd packaging && $(MAKE) +version: + VERSION=$$(awk '/^version/{print $$3}' pyproject.toml); \ + sed "s/__version__ =.*/__version__ = $$VERSION/" -i $(SRCDIR)/__init__.py; \ + sed "s/release =.*/release = $$VERSION/" -i $(DOCS_SRCDIR)/conf.py + format: poetry run isort $(SRCDIR) poetry run ruff format $(SRCDIR) @@ -32,7 +35,8 @@ docs-versions: poetry run sphinx-multiversion $(DOCS_SRCDIR) $(DOCS_BUILDDIR) serve-docs: - poetry run sphinx-autobuild $(DOCS_SRCDIR) $(DOCS_BUILDDIR) + poetry run sphinx-autobuild $(DOCS_SRCDIR) $(DOCS_BUILDDIR) \ + --pre-build 'make clean' clean: [ -d $(DISTDIR) ] && rm -rf $(DISTDIR) || true @@ -42,3 +46,7 @@ clean: test-build: build-deb scp packaging/build/compute*.deb vm:~ + +upload-docs: docs-versions + ssh root@hitomi 'rm -rf /srv/http/nixhacks.net/hstack/*' + scp -r $(DOCS_DUILDDIR) root@hitomi:/srv/http/nixhacks.net/hstack/ diff --git a/README.md b/README.md index ef5002a..22c71ce 100644 --- a/README.md +++ b/README.md @@ -9,23 +9,23 @@ Run `make serve-docs`. See [Development](#development) below. ## Roadmap - [x] Create instances -- [ ] CDROM +- [x] CDROM - [ ] cloud-init for provisioning instances -- [x] Instance power management -- [x] Instance pause and resume +- [x] Power management +- [x] Pause and resume - [x] vCPU hotplug - [x] Memory hotplug - [x] Hot disk resize [not tested] -- [ ] CPU topology customization - [x] CPU customization (emulation mode, model, vendor, features) +- [ ] CPU topology customization - [ ] BIOS/UEFI settings - [x] Device attaching - [x] Device detaching - [ ] GPU passthrough - [ ] CPU guarantied resource percent support - [x] QEMU Guest Agent management -- [ ] Instance resources usage stats -- [ ] SSH-keys management +- [ ] Resource usage stats +- [x] SSH-keys management - [x] Setting user passwords in guest - [x] QCOW2 disks support - [ ] ZVOL support @@ -35,10 +35,11 @@ Run `make serve-docs`. See [Development](#development) below. - [ ] Idempotency - [ ] CLI [in progress] - [ ] HTTP API -- [ ] Instance migrations -- [ ] Instance snapshots -- [ ] Instance backups +- [ ] Migrations +- [ ] Snapshots +- [ ] Backups - [ ] LXC +- [ ] Attaching CDROM from sources: block, (http|https|ftp|ftps|tftp):// ## Development @@ -78,6 +79,7 @@ After installation prepare environment, run following command to start libvirtd virsh pool-define-as $pool dir - - - - "/$pool" virsh pool-build $pool virsh pool-start $pool + virsh pool-autostart $pool done ``` diff --git a/compute/__init__.py b/compute/__init__.py index ffe06d7..e0722f1 100644 --- a/compute/__init__.py +++ b/compute/__init__.py @@ -5,17 +5,17 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# Ansible is distributed in the hope that it will be useful, +# Compute is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . +# along with Compute. If not, see . """Compute instances management library.""" -__version__ = '0.1.0-dev1' +__version__ = '0.1.0-dev2' from .instance import Instance, InstanceConfig, InstanceSchema from .session import Session diff --git a/compute/__main__.py b/compute/__main__.py index c50bd48..346aaaf 100644 --- a/compute/__main__.py +++ b/compute/__main__.py @@ -5,13 +5,13 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# Ansible is distributed in the hope that it will be useful, +# Compute is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . +# along with Compute. If not, see . """Command line interface for compute module.""" diff --git a/compute/cli/control.py b/compute/cli/control.py index a27d7ab..2d85cd7 100644 --- a/compute/cli/control.py +++ b/compute/cli/control.py @@ -5,25 +5,25 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# Ansible is distributed in the hope that it will be useful, +# Compute is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . +# along with Compute. If not, see . """Command line interface.""" import argparse -import io +import json import logging import os +import re import shlex +import string import sys -from collections import UserDict -from typing import Any -from uuid import uuid4 +import uuid import libvirt import yaml @@ -31,9 +31,10 @@ from pydantic import ValidationError from compute import __version__ from compute.exceptions import ComputeError, GuestAgentTimeoutError -from compute.instance import GuestAgent +from compute.instance import GuestAgent, Instance, InstanceSchema +from compute.instance.devices import DiskConfig, DiskDriver from compute.session import Session -from compute.utils import ids +from compute.utils import dictutil, ids log = logging.getLogger(__name__) @@ -128,78 +129,77 @@ def _exec_guest_agent_command( sys.exit(output.exitcode) -class _NotPresent: - """ - Type for representing non-existent dictionary keys. - - See :class:`_FillableDict`. - """ - - -class _FillableDict(UserDict): - """Use :method:`fill` to add key if not present.""" - - def __init__(self, data: dict): - self.data = data - - def fill(self, key: str, value: Any) -> None: # noqa: ANN401 - if self.data.get(key, _NotPresent) is _NotPresent: - self.data[key] = value - - -def _merge_dicts(a: dict, b: dict, path: list[str] | None = None) -> dict: - """Merge `b` into `a`. Return modified `a`.""" - if path is None: - path = [] - for key in b: - if key in a: - if isinstance(a[key], dict) and isinstance(b[key], dict): - _merge_dicts(a[key], b[key], [path + str(key)]) - elif a[key] == b[key]: - pass # same leaf value - else: - a[key] = b[key] # replace existing key's values - else: - a[key] = b[key] - return a - - -def _create_instance(session: Session, file: io.TextIOWrapper) -> None: +def _init_instance(session: Session, args: argparse.Namespace) -> None: try: - data = _FillableDict(yaml.load(file.read(), Loader=yaml.SafeLoader)) + data = yaml.load(args.file.read(), Loader=yaml.SafeLoader) log.debug('Read from file: %s', data) except yaml.YAMLError as e: sys.exit(f'error: cannot parse YAML: {e}') - capabilities = session.get_capabilities() node_info = session.get_node_info() - - data.fill('name', uuid4().hex) - data.fill('title', None) - data.fill('description', None) - data.fill('arch', capabilities.arch) - data.fill('machine', capabilities.machine) - data.fill('emulator', capabilities.emulator) - data.fill('max_vcpus', node_info.cpus) - data.fill('max_memory', node_info.memory) - data.fill('cpu', {}) - cpu = { - 'emulation_mode': 'host-passthrough', - 'model': None, - 'vendor': None, - 'topology': None, - 'features': None, + base_instance_config = { + 'name': str(uuid.uuid4()), + 'title': None, + 'description': None, + 'arch': capabilities.arch, + 'machine': capabilities.machine, + 'emulator': capabilities.emulator, + 'max_vcpus': node_info.cpus, + 'max_memory': node_info.memory, + 'cpu': { + 'emulation_mode': 'host-passthrough', + 'model': None, + 'vendor': None, + 'topology': None, + 'features': None, + }, + 'network_interfaces': [ + { + 'source': 'default', + 'mac': ids.random_mac(), + }, + ], + 'boot': {'order': ['cdrom', 'hd']}, } - data['cpu'] = _merge_dicts(data['cpu'], cpu) - data.fill( - 'network_interfaces', - [{'source': 'default', 'mac': ids.random_mac()}], - ) - data.fill('boot', {'order': ['cdrom', 'hd']}) - + data = dictutil.override(base_instance_config, data) + volumes = [] + for volume in data['volumes']: + base_disk_config = { + 'bus': 'virtio', + 'is_readonly': False, + 'driver': { + 'name': 'qemu', + 'type': 'qcow2', + 'cache': 'writethrough', + }, + } + base_cdrom_config = { + 'bus': 'ide', + 'target': 'hda', + 'is_readonly': True, + 'driver': { + 'name': 'qemu', + 'type': 'raw', + 'cache': 'writethrough', + }, + } + if volume.get('device') is None: + volume['device'] = 'disk' + if volume['device'] == 'disk': + volumes.append(dictutil.override(base_disk_config, volume)) + if volume['device'] == 'cdrom': + volumes.append(dictutil.override(base_cdrom_config, volume)) + data['volumes'] = volumes try: log.debug('Input data: %s', data) - session.create_instance(**data) + if args.test: + _ = InstanceSchema(**data) + print(json.dumps(dict(data), indent=4, sort_keys=True)) + sys.exit() + instance = session.create_instance(**data) + print(f'initialised: {instance.name}') + if args.start: + instance.start() except ValidationError as e: for error in e.errors(): fields = '.'.join([str(lc) for lc in error['loc']]) @@ -223,11 +223,84 @@ def _shutdown_instance(session: Session, args: argparse.Namespace) -> None: instance.shutdown(method) +def _confirm(message: str, *, default: bool | None = None) -> None: + while True: + match default: + case True: + prompt = 'default: yes' + case False: + prompt = 'default: no' + case _: + prompt = 'no default' + try: + answer = input(f'{message} ({prompt}) ') + except KeyboardInterrupt: + sys.exit('aborted') + if not answer and isinstance(default, bool): + return default + if re.match(r'^y(es)?$', answer, re.I): + return True + if re.match(r'^no?$', answer, re.I): + return False + print("Please respond 'yes' or 'no'") + + +def _delete_instance(session: Session, args: argparse.Namespace) -> None: + if args.yes is True or _confirm( + 'this action is irreversible, continue?', + default=False, + ): + instance = session.get_instance(args.instance) + if args.save_volumes is False: + instance.delete(with_volumes=True) + else: + instance.delete() + else: + print('aborted') + sys.exit() + + +def _get_disk_target(instance: Instance, prefix: str = 'hd') -> str: + disks_live = instance.list_disks(persistent=False) + disks_inactive = instance.list_disks(persistent=True) + disks = [d for d in disks_inactive if d not in disks_live] + devs = [d.target[-1] for d in disks if d.target.startswith(prefix)] + return prefix + [x for x in string.ascii_lowercase if x not in devs][0] # noqa: RUF015 + + +def _manage_cdrom(session: Session, args: argparse.Namespace) -> None: + instance = session.get_instance(args.instance) + if args.detach: + for disk in instance.list_disks(persistent=True): + if disk.device == 'cdrom' and disk.source == args.source: + instance.detach_disk(disk.target, live=False) + print( + f"disk '{disk.target}' detached, " + 'perform power reset to apply changes' + ) + return + target = _get_disk_target(instance, 'hd') + cdrom = DiskConfig( + type='file', + device='cdrom', + source=args.source, + target=target, + is_readonly=True, + bus='ide', + driver=DiskDriver('qemu', 'raw', 'writethrough'), + ) + instance.attach_device(cdrom, live=False) + print( + f"CDROM attached as disk '{target}', " + 'perform power reset to apply changes' + ) + + def main(session: Session, args: argparse.Namespace) -> None: """Perform actions.""" match args.command: case 'init': - _create_instance(session, args.file) + _init_instance(session, args) case 'exec': _exec_guest_agent_command(session, args) case 'ls': @@ -268,13 +341,17 @@ def main(session: Session, args: argparse.Namespace) -> None: args.password, encrypted=args.encrypted, ) + case 'setcdrom': + _manage_cdrom(session, args) + case 'delete': + _delete_instance(session, args) -def cli() -> None: # noqa: PLR0915 +def get_parser() -> argparse.ArgumentParser: # noqa: PLR0915 """Return command line arguments parser.""" root = argparse.ArgumentParser( prog='compute', - description='manage compute instances', + description='Manage compute instances.', formatter_class=argparse.RawTextHelpFormatter, ) root.add_argument( @@ -317,12 +394,27 @@ def cli() -> None: # noqa: PLR0915 default='instance.yaml', help='instance config [default: instance.yaml]', ) + init.add_argument( + '-s', + '--start', + action='store_true', + default=False, + help='start instance after init', + ) + init.add_argument( + '-t', + '--test', + action='store_true', + default=False, + help='just print resulting instance config as JSON and exit', + ) # exec subcommand execute = subparsers.add_parser( 'exec', help='execute command in guest via guest agent', description=( + 'Execute command in guest via guest agent. ' 'NOTE: any argument after instance name will be passed into ' 'guest as shell command.' ), @@ -463,27 +555,60 @@ def cli() -> None: # noqa: PLR0915 help='set it if password is already encrypted', ) + # setcdrom subcommand + setcdrom = subparsers.add_parser('setcdrom', help='manage CDROM devices') + setcdrom.add_argument('instance') + setcdrom.add_argument('source', help='source for CDROM') + setcdrom.add_argument( + '-d', + '--detach', + action='store_true', + default=False, + help='detach CDROM device', + ) + + # delete subcommand + delete = subparsers.add_parser( + 'delete', + help='delete instance', + ) + delete.add_argument('instance') + delete.add_argument( + '-y', + '--yes', + action='store_true', + default=False, + help='automatic yes to prompt', + ) + delete.add_argument( + '--save-volumes', + action='store_true', + default=False, + help='do not delete local storage volumes', + ) + + return root + + +def cli() -> None: + """Run arguments parser.""" + root = get_parser() args = root.parse_args() if args.command is None: root.print_help() sys.exit() - log_level = args.log_level or os.getenv('CMP_LOG') - if isinstance(log_level, str) and log_level.lower() in log_levels: logging.basicConfig( level=logging.getLevelNamesMapping()[log_level.upper()] ) - log.debug('CLI started with args: %s', args) - connect_uri = ( args.connect or os.getenv('CMP_LIBVIRT_URI') or os.getenv('LIBVIRT_DEFAULT_URI') or 'qemu:///system' ) - try: with Session(connect_uri) as session: main(session, args) @@ -493,8 +618,6 @@ def cli() -> None: # noqa: PLR0915 sys.exit() except SystemExit as e: sys.exit(e) - except Exception as e: # noqa: BLE001 - sys.exit(f'unexpected error {type(e)}: {e}') if __name__ == '__main__': diff --git a/compute/common.py b/compute/common.py index a967a5c..8bea4d9 100644 --- a/compute/common.py +++ b/compute/common.py @@ -5,13 +5,13 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# Ansible is distributed in the hope that it will be useful, +# Compute is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . +# along with Compute. If not, see . """Common symbols.""" diff --git a/compute/exceptions.py b/compute/exceptions.py index 1e25ddd..0d8b9af 100644 --- a/compute/exceptions.py +++ b/compute/exceptions.py @@ -5,13 +5,13 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# Ansible is distributed in the hope that it will be useful, +# Compute is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . +# along with Compute. If not, see . """Exceptions.""" @@ -80,9 +80,32 @@ class InstanceNotFoundError(InstanceError): super().__init__(f"compute instance '{msg}' not found") +class InvalidDeviceConfigError(ComputeError): + """ + Invalid device XML description. + + :class:`DeviceCoonfig` instance cannot be created because + device config in libvirt XML config is not valid. + """ + + def __init__(self, msg: str, xml: str): + """Initialise InvalidDeviceConfigError.""" + self.msg = f'Invalid device XML config: {msg}' + self.loc = f' {xml}' + super().__init__(f'{self.msg}\n:{self.loc}') + + class InvalidDataUnitError(ValueError, ComputeError): """Data unit is not valid.""" def __init__(self, msg: str, units: list): """Initialise InvalidDataUnitError.""" super().__init__(f'{msg}, valid units are: {", ".join(units)}') + + +class DictMergeConflictError(ComputeError): + """Conflict when merging dicts.""" + + def __init__(self, key: str): + """Initialise DictMergeConflictError.""" + super().__init__(f'Conflicting key: {key}') diff --git a/compute/instance/__init__.py b/compute/instance/__init__.py index 6e2b150..42ddaf2 100644 --- a/compute/instance/__init__.py +++ b/compute/instance/__init__.py @@ -5,13 +5,13 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# Ansible is distributed in the hope that it will be useful, +# Compute is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . +# along with Compute. If not, see . from .guest_agent import GuestAgent from .instance import Instance, InstanceConfig diff --git a/compute/instance/devices.py b/compute/instance/devices.py new file mode 100644 index 0000000..eba09f2 --- /dev/null +++ b/compute/instance/devices.py @@ -0,0 +1,128 @@ +# This file is part of Compute +# +# Compute is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Compute is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Compute. If not, see . + +# ruff: noqa: SIM211, UP007, A003 + +"""Virtual devices configs.""" + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Union + +from lxml import etree +from lxml.builder import E + +from compute.common import DeviceConfig +from compute.exceptions import InvalidDeviceConfigError + + +@dataclass +class DiskDriver: + """Disk driver description for libvirt.""" + + name: str + type: str + cache: str + + def __call__(self): + """Return self.""" + return self + + +@dataclass +class DiskConfig(DeviceConfig): + """ + Disk config builder. + + Generate XML config for attaching or detaching storage volumes + to compute instances. + """ + + type: str + source: str | Path + target: str + is_readonly: bool = False + device: str = 'disk' + bus: str = 'virtio' + driver: DiskDriver = field( + default_factory=DiskDriver( + name='qemu', + type='qcow2', + cache='writethrough', + ) + ) + + def to_xml(self) -> str: + """Return XML config for libvirt.""" + xml = E.disk(type=self.type, device=self.device) + xml.append( + E.driver( + name=self.driver.name, + type=self.driver.type, + cache=self.driver.cache, + ) + ) + if self.source and self.type == 'file': + xml.append(E.source(file=str(self.source))) + xml.append(E.target(dev=self.target, bus=self.bus)) + if self.is_readonly: + xml.append(E.readonly()) + return etree.tostring(xml, encoding='unicode', pretty_print=True) + + @classmethod + def from_xml(cls, xml: Union[str, etree.Element]) -> 'DiskConfig': + """ + Create :class:`DiskConfig` instance from XML config. + + :param xml: Disk device XML configuration as :class:`str` or lxml + :class:`etree.Element` object. + """ + if isinstance(xml, str): + xml_str = xml + xml = etree.fromstring(xml) + else: + xml_str = etree.tostring( + xml, + encoding='unicode', + pretty_print=True, + ).strip() + driver = xml.find('driver') + disk_params = { + 'type': xml.get('type'), + 'device': xml.get('device'), + 'driver': DiskDriver( + name=driver.get('name'), + type=driver.get('type'), + cache=driver.get('cache'), + ), + 'source': xml.find('source').get('file'), + 'target': xml.find('target').get('dev'), + 'bus': xml.find('target').get('bus'), + 'is_readonly': False if xml.find('readonly') is None else True, + } + for param in disk_params: + if disk_params[param] is None: + msg = f"missing XML tag '{param}'" + raise InvalidDeviceConfigError(msg, xml_str) + if param == 'driver': + driver = disk_params[param] + for driver_param in [driver.name, driver.type, driver.cache]: + if driver_param is None: + msg = ( + "'driver' tag must have " + "'name', 'type' and 'cache' attributes" + ) + raise InvalidDeviceConfigError(msg, xml_str) + return cls(**disk_params) diff --git a/compute/instance/guest_agent.py b/compute/instance/guest_agent.py index cf665cc..a6f62da 100644 --- a/compute/instance/guest_agent.py +++ b/compute/instance/guest_agent.py @@ -5,13 +5,13 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# Ansible is distributed in the hope that it will be useful, +# Compute is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . +# along with Compute. If not, see . """Interacting with the QEMU Guest Agent.""" diff --git a/compute/instance/instance.py b/compute/instance/instance.py index ad7afb5..a626a3c 100644 --- a/compute/instance/instance.py +++ b/compute/instance/instance.py @@ -5,19 +5,20 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# Ansible is distributed in the hope that it will be useful, +# Compute is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . +# along with Compute. If not, see . """Manage compute instances.""" __all__ = ['Instance', 'InstanceConfig', 'InstanceInfo'] import logging +import time from typing import NamedTuple import libvirt @@ -29,9 +30,9 @@ from compute.exceptions import ( GuestAgentCommandNotSupportedError, InstanceError, ) -from compute.storage import DiskConfig from compute.utils import units +from .devices import DiskConfig from .guest_agent import GuestAgent from .schemas import ( CPUEmulationMode, @@ -282,7 +283,7 @@ class Instance: def start(self) -> None: """Start defined instance.""" - log.info('Starting instnce=%s', self.name) + log.info("Starting instance '%s'", self.name) if self.is_running(): log.warning( 'Already started, nothing to do instance=%s', self.name @@ -292,7 +293,7 @@ class Instance: self.domain.create() except libvirt.libvirtError as e: raise InstanceError( - f'Cannot start instance={self.name}: {e}' + f"Cannot start instance '{self.name}': {e}" ) from e def shutdown(self, method: str | None = None) -> None: @@ -323,6 +324,7 @@ class Instance: :param method: Method used to shutdown instance """ if not self.is_running(): + log.warning('Instance is not running, nothing to do') return methods = { 'SOFT': libvirt.VIR_DOMAIN_SHUTDOWN_GUEST_AGENT, @@ -339,6 +341,7 @@ class Instance: method = method.upper() if method not in methods: raise ValueError(f"Unsupported shutdown method: '{method}'") + log.info("Performing instance shutdown with method '%s'", method) try: if method in ['SOFT', 'NORMAL']: self.domain.shutdownFlags(flags=methods[method]) @@ -346,7 +349,7 @@ class Instance: self.domain.destroyFlags(flags=methods[method]) except libvirt.libvirtError as e: raise InstanceError( - f'Cannot shutdown instance={self.name} ' f'{method=}: {e}' + f"Cannot shutdown instance '{self.name}' with '{method=}': {e}" ) from e def reboot(self) -> None: @@ -375,7 +378,7 @@ class Instance: self.domain.reset() except libvirt.libvirtError as e: raise InstanceError( - f'Cannot reset instance={self.name}: {e}' + f"Cannot reset instance '{self.name}': {e}" ) from e def power_reset(self) -> None: @@ -389,7 +392,13 @@ class Instance: configuration change in libvirt and you need to restart the instance to apply the new configuration. """ - self.shutdown(method='NORMAL') + log.debug("Performing power reset for instance '%s'", self.name) + self.shutdown('NORMAL') + time.sleep(3) + # TODO @ge: do safe shutdown insted of this shit + if self.is_running(): + self.shutdown('HARD') + time.sleep(1) self.start() def set_autostart(self, *, enabled: bool) -> None: @@ -550,7 +559,9 @@ class Instance: return self.domain.detachDeviceFlags(device.to_xml(), flags=flags) - def get_disk(self, name: str) -> DiskConfig | None: + def get_disk( + self, name: str, *, persistent: bool = False + ) -> DiskConfig | None: """ Return :class:`DiskConfig` by disk target name. @@ -558,20 +569,27 @@ class Instance: :param name: Disk name e.g. `vda`, `sda`, etc. This name may not match the name of the disk inside the guest OS. + :param persistent: If True get only persistent volumes described + in instance XML config. """ - xml = etree.fromstring(self.dump_xml()) + xml = etree.fromstring(self.dump_xml(inactive=persistent)) child = xml.xpath(f'/domain/devices/disk/target[@dev="{name}"]') if len(child) == 0: return None return DiskConfig.from_xml(child[0].getparent()) - def list_disks(self) -> list[DiskConfig]: - """Return list of attached disk devices.""" - xml = etree.fromstring(self.dump_xml()) + def list_disks(self, *, persistent: bool = False) -> list[DiskConfig]: + """ + Return list of attached disk devices. + + :param persistent: If True list only persistent volumes described + in instance XML config. + """ + xml = etree.fromstring(self.dump_xml(inactive=persistent)) disks = xml.xpath('/domain/devices/disk') return [DiskConfig.from_xml(disk) for disk in disks] - def detach_disk(self, name: str) -> None: + def detach_disk(self, name: str, *, live: bool = False) -> None: """ Detach disk device by target name. @@ -580,15 +598,17 @@ class Instance: :param name: Disk name e.g. `vda`, `sda`, etc. This name may not match the name of the disk inside the guest OS. + :param live: Affect a running instance. Not supported for CDROM + devices. """ - disk = self.get_disk(name) + disk = self.get_disk(name, persistent=live) if disk is None: log.warning( "Volume with target '%s' is already detached", name, ) return - self.detach_device(disk, live=True) + self.detach_device(disk, live=live) def resize_disk( self, name: str, capacity: int, unit: units.DataUnit @@ -601,6 +621,7 @@ class Instance: :param capacity: New capacity. :param unit: Capacity unit. """ + # TODO @ge: check actual size before making changes self.domain.blockResize( name, units.to_bytes(capacity, unit=unit), @@ -619,7 +640,7 @@ class Instance: def list_ssh_keys(self, user: str) -> list[str]: """ - Return list of SSH keys on guest for specific user. + Return list of authorized SSH keys in guest for specific user. :param user: Username. """ @@ -655,7 +676,7 @@ class Instance: append: bool = False, ) -> None: """ - Add SSH keys to guest for specific user. + Add authorized SSH keys to guest for specific user. :param user: Username. :param keys: List of authorized SSH keys. @@ -666,7 +687,7 @@ class Instance: qemu_ga_commands = ['guest-ssh-add-authorized-keys'] if remove and append: raise InstanceError( - "'append' and 'remove' parameters is mutually exclusive" + "'append' and 'remove' parameters are mutually exclusive" ) if not self.is_running(): raise InstanceError( @@ -693,7 +714,7 @@ class Instance: """ Set new user password in guest OS. - This action performs by guest agent inside the guest. + This action is performed by guest agent inside the guest. :param user: Username. :param password: Password. @@ -702,6 +723,7 @@ class Instance: """ self.guest_agent.raise_for_commands(['guest-set-user-password']) flags = libvirt.VIR_DOMAIN_PASSWORD_ENCRYPTED if encrypted else 0 + log.debug("Setting up password for user '%s'", user) self.domain.setUserPassword(user, password, flags=flags) def dump_xml(self, *, inactive: bool = False) -> str: @@ -709,11 +731,19 @@ class Instance: flags = libvirt.VIR_DOMAIN_XML_INACTIVE if inactive else 0 return self.domain.XMLDesc(flags) - def delete(self) -> None: - """Delete instance with local disks.""" + def delete(self, *, with_volumes: bool = False) -> None: + """ + Delete instance with local volumes. + + :param with_volumes: If True delete local volumes with instance. + """ self.shutdown(method='HARD') - for disk in self.list_disks(): - if disk.disk_type == 'file': + disks = self.list_disks(persistent=True) + log.debug('Disks list: %s', disks) + for disk in disks: + if with_volumes and disk.type == 'file': volume = self.connection.storageVolLookupByPath(disk.source) + log.debug('Delete volume: %s', volume.path()) volume.delete() + log.debug('Undefine instance') self.domain.undefine() diff --git a/compute/instance/schemas.py b/compute/instance/schemas.py index a13d486..1983826 100644 --- a/compute/instance/schemas.py +++ b/compute/instance/schemas.py @@ -5,13 +5,13 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# Ansible is distributed in the hope that it will be useful, +# Compute is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . +# along with Compute. If not, see . """Compute instance related objects schemas.""" @@ -19,7 +19,8 @@ import re from enum import StrEnum from pathlib import Path -from pydantic import validator +from pydantic import ValidationError, validator +from pydantic.error_wrappers import ErrorWrapper from compute.common import EntityModel from compute.utils.units import DataUnit @@ -73,15 +74,26 @@ class VolumeCapacitySchema(EntityModel): unit: DataUnit +class DiskDriverSchema(EntityModel): + """Virtual disk driver model.""" + + name: str + type: str # noqa: A003 + cache: str = 'writethrough' + + class VolumeSchema(EntityModel): """Storage volume model.""" type: VolumeType # noqa: A003 target: str - capacity: VolumeCapacitySchema + driver: DiskDriverSchema + capacity: VolumeCapacitySchema | None source: str | None = None is_readonly: bool = False is_system: bool = False + bus: str = 'virtio' + device: str = 'disk' class NetworkInterfaceSchema(EntityModel): @@ -118,10 +130,10 @@ class InstanceSchema(EntityModel): @validator('name') def _check_name(cls, value: str) -> str: # noqa: N805 - if not re.match(r'^[a-z0-9_]+$', value): + if not re.match(r'^[a-z0-9_-]+$', value): msg = ( - 'Name can contain only lowercase letters, numbers ' - 'and underscore.' + 'Name can contain only lowercase letters, numbers, ' + 'minus sign and underscore.' ) raise ValueError(msg) return value @@ -140,13 +152,22 @@ class InstanceSchema(EntityModel): if len([v for v in volumes if v.is_system is True]) != 1: msg = 'volumes list must contain one system volume' raise ValueError(msg) - vol_with_source = 0 for vol in volumes: + if vol.source is None and vol.capacity is None: + raise ValidationError( + [ + ErrorWrapper( + Exception( + "capacity is required if 'source' is unset" + ), + loc='X.capacity', + ) + ], + model=VolumeSchema, + ) if vol.is_system is True and vol.is_readonly is True: msg = 'volume marked as system cannot be readonly' raise ValueError(msg) - if vol.source is not None: - vol_with_source += 1 return volumes @validator('network_interfaces') diff --git a/compute/session.py b/compute/session.py index d33c655..5fae003 100644 --- a/compute/session.py +++ b/compute/session.py @@ -5,13 +5,13 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# Ansible is distributed in the hope that it will be useful, +# Compute is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . +# along with Compute. If not, see . """Hypervisor session manager.""" @@ -31,7 +31,8 @@ from .exceptions import ( StoragePoolNotFoundError, ) from .instance import Instance, InstanceConfig, InstanceSchema -from .storage import DiskConfig, StoragePool, VolumeConfig +from .instance.devices import DiskConfig, DiskDriver +from .storage import StoragePool, VolumeConfig from .utils import units @@ -164,7 +165,7 @@ class Session(AbstractContextManager): cpu_vendor=caps.xpath(f'{hprefix}/vendor/text()')[0], cpu_model=caps.xpath(f'{hprefix}/model/text()')[0], cpu_features=self._cap_get_cpu_features(caps), - usable_cpus=self._cap_get_cpus(caps), + usable_cpus=self._cap_get_usable_cpus(caps), ) def create_instance(self, **kwargs: Any) -> Instance: @@ -208,15 +209,15 @@ class Session(AbstractContextManager): config = InstanceConfig(data) log.info('Define XML...') log.info(config.to_xml()) - self.connection.defineXML(config.to_xml()) + try: + self.connection.defineXML(config.to_xml()) + except libvirt.libvirtError as e: + raise SessionError(f'Error defining instance: {e}') from e log.info('Getting instance...') instance = self.get_instance(config.name) - log.info('Creating volumes...') + log.info('Start processing volumes...') for volume in data.volumes: - log.info('Creating volume=%s', volume) - capacity = units.to_bytes( - volume.capacity.value, volume.capacity.unit - ) + log.info('Processing volume=%s', volume) log.info('Connecting to images pool...') images_pool = self.get_storage_pool(self.IMAGES_POOL) log.info('Connecting to volumes pool...') @@ -226,35 +227,48 @@ class Session(AbstractContextManager): vol_name = f'{uuid4()}.qcow2' else: vol_name = volume.source - vol_conf = VolumeConfig( - name=vol_name, - path=str(volumes_pool.path.joinpath(vol_name)), - capacity=capacity, - ) - log.info('Volume configuration is:\n %s', vol_conf.to_xml()) - if volume.is_system is True and data.image: - log.info( - "Volume is marked as 'system', start cloning image..." - ) - log.info('Get image %s', data.image) - image = images_pool.get_volume(data.image) - log.info('Cloning image into volumes pool...') - vol = volumes_pool.clone_volume(image, vol_conf) - log.info( - 'Resize cloned volume to specified size: %s', - capacity, - ) - vol.resize(capacity, unit=units.DataUnit.BYTES) + if volume.device == 'cdrom': + log.debug('Volume %s is CDROM device', vol_name) else: - log.info('Create volume...') - volumes_pool.create_volume(vol_conf) + capacity = units.to_bytes( + volume.capacity.value, volume.capacity.unit + ) + vol_conf = VolumeConfig( + name=vol_name, + path=str(volumes_pool.path.joinpath(vol_name)), + capacity=capacity, + ) + log.info('Volume configuration is:\n %s', vol_conf.to_xml()) + if volume.is_system is True and data.image: + log.info( + "Volume is marked as 'system', start cloning image..." + ) + log.info('Get image %s', data.image) + image = images_pool.get_volume(data.image) + log.info('Cloning image into volumes pool...') + vol = volumes_pool.clone_volume(image, vol_conf) + log.info( + 'Resize cloned volume to specified size: %s', + capacity, + ) + vol.resize(capacity, unit=units.DataUnit.BYTES) + else: + log.info('Create volume...') + volumes_pool.create_volume(vol_conf) log.info('Attaching volume to instance...') instance.attach_device( DiskConfig( - disk_type=volume.type, + type=volume.type, + device=volume.device, source=vol_conf.path, target=volume.target, - readonly=volume.is_readonly, + is_readonly=volume.is_readonly, + bus=volume.bus, + driver=DiskDriver( + volume.driver.name, + volume.driver.type, + volume.driver.cache, + ), ) ) return instance diff --git a/compute/storage/__init__.py b/compute/storage/__init__.py index 34aae30..29f1b2f 100644 --- a/compute/storage/__init__.py +++ b/compute/storage/__init__.py @@ -5,13 +5,13 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# Ansible is distributed in the hope that it will be useful, +# Compute is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . +# along with Compute. If not, see . from .pool import StoragePool -from .volume import DiskConfig, Volume, VolumeConfig +from .volume import Volume, VolumeConfig diff --git a/compute/storage/pool.py b/compute/storage/pool.py index 20c1d5a..3f906bd 100644 --- a/compute/storage/pool.py +++ b/compute/storage/pool.py @@ -5,13 +5,13 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# Ansible is distributed in the hope that it will be useful, +# Compute is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . +# along with Compute. If not, see . """Manage storage pools.""" diff --git a/compute/storage/volume.py b/compute/storage/volume.py index b090b56..bc028d0 100644 --- a/compute/storage/volume.py +++ b/compute/storage/volume.py @@ -5,26 +5,25 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# Ansible is distributed in the hope that it will be useful, +# Compute is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . +# along with Compute. If not, see . """Manage storage volumes.""" from dataclasses import dataclass from pathlib import Path from time import time -from typing import Union import libvirt from lxml import etree from lxml.builder import E -from compute.common import DeviceConfig, EntityConfig +from compute.common import EntityConfig from compute.utils import units @@ -64,54 +63,6 @@ class VolumeConfig(EntityConfig): return etree.tostring(xml, encoding='unicode', pretty_print=True) -@dataclass -class DiskConfig(DeviceConfig): - """ - Disk XML config builder. - - Generate XML config for attaching or detaching storage volumes - to compute instances. - """ - - disk_type: str - source: str | Path - target: str - readonly: bool = False - - def to_xml(self) -> str: - """Return XML config for libvirt.""" - xml = E.disk(type=self.disk_type, device='disk') - xml.append(E.driver(name='qemu', type='qcow2', cache='writethrough')) - if self.disk_type == 'file': - xml.append(E.source(file=str(self.source))) - xml.append(E.target(dev=self.target, bus='virtio')) - if self.readonly: - xml.append(E.readonly()) - return etree.tostring(xml, encoding='unicode', pretty_print=True) - - @classmethod - def from_xml(cls, xml: Union[str, etree.Element]) -> 'DiskConfig': # noqa: UP007 - """ - Return :class:`DiskConfig` instance using existing XML config. - - :param xml: Disk device XML configuration as :class:`str` or lxml - :class:`etree.Element` object. - """ - if isinstance(xml, str): - xml = etree.fromstring(xml) - disk_params = { - 'disk_type': xml.get('type'), - 'source': xml.find('source').get('file'), - 'target': xml.find('target').get('dev'), - 'readonly': False if xml.find('readonly') is None else True, # noqa: SIM211 - } - for param in disk_params: - if disk_params[param] is None: - msg = f"Bad XML config: parameter '{param}' is not defined" - raise ValueError(msg) - return cls(**disk_params) - - class Volume: """Storage volume manipulating class.""" diff --git a/compute/utils/config_loader.py b/compute/utils/config_loader.py index aaeb0fe..f6fcbd7 100644 --- a/compute/utils/config_loader.py +++ b/compute/utils/config_loader.py @@ -5,13 +5,13 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# Ansible is distributed in the hope that it will be useful, +# Compute is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . +# along with Compute. If not, see . """Configuration loader.""" diff --git a/compute/utils/dictutil.py b/compute/utils/dictutil.py new file mode 100644 index 0000000..104e158 --- /dev/null +++ b/compute/utils/dictutil.py @@ -0,0 +1,77 @@ +# This file is part of Compute +# +# Compute is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Compute is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Compute. If not, see . + +"""Dict tools.""" + +from compute.exceptions import DictMergeConflictError + + +def merge(a: dict, b: dict, path: list[str] | None = None) -> dict: + """ + Merge `b` into `a`. Return modified `a`. + + :raise: :class:`DictMergeConflictError` + """ + if path is None: + path = [] + for key in b: + if key in a: + if isinstance(a[key], dict) and isinstance(b[key], dict): + merge(a[key], b[key], [*path, str(key)]) + elif a[key] != b[key]: + raise DictMergeConflictError('.'.join([*path, str(key)])) + else: + a[key] = b[key] + return a + + +def override(a: dict, b: dict) -> dict: + """ + Override dict `a` by `b` values. + + Keys that not exists in `a`, but exists in `b` will be + appended to `a`. + + .. code-block:: shell-session + + >>> from compute.utils import dictutil + >>> default = { + ... 'bus': 'virtio', + ... 'driver': {'name': 'qemu', 'type': 'qcow2'} + ... } + >>> user = { + ... 'bus': 'ide', + ... 'target': 'vda', + ... 'driver': {'type': 'raw'} + ... } + >>> dictutil.override(default, user) + {'bus': 'ide', 'driver': {'name': 'qemu', 'type': 'raw'}, + 'target': 'vda'} + + NOTE: merging dicts contained in lists is not supported. + + :param a: Dict to be overwritten. + :param b: A dict whose values will be used to rewrite dict `a`. + :return: Modified `a` dict. + """ + for key in b: + if key in a: + if isinstance(a[key], dict) and isinstance(b[key], dict): + override(a[key], b[key]) + else: + a[key] = b[key] # replace existing key's values + else: + a[key] = b[key] + return a diff --git a/compute/utils/ids.py b/compute/utils/ids.py index 8a6454a..5cd9559 100644 --- a/compute/utils/ids.py +++ b/compute/utils/ids.py @@ -5,13 +5,13 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# Ansible is distributed in the hope that it will be useful, +# Compute is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . +# along with Compute. If not, see . """Random identificators.""" diff --git a/compute/utils/units.py b/compute/utils/units.py index 40c62c9..cec9a22 100644 --- a/compute/utils/units.py +++ b/compute/utils/units.py @@ -5,13 +5,13 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# Ansible is distributed in the hope that it will be useful, +# Compute is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . +# along with Compute. If not, see . """Tools for data units convertion.""" @@ -21,7 +21,7 @@ from compute.exceptions import InvalidDataUnitError class DataUnit(StrEnum): - """Data units enumerated.""" + """Data units enumeration.""" BYTES = 'bytes' KIB = 'KiB' @@ -29,6 +29,13 @@ class DataUnit(StrEnum): GIB = 'GiB' TIB = 'TiB' + @classmethod + def _missing_(cls, name: str) -> 'DataUnit': + for member in cls: + if member.name.lower() == name.lower(): + return member + return None + def to_bytes(value: int, unit: DataUnit = DataUnit.BYTES) -> int: """Convert value to bytes. See :class:`DataUnit`.""" diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css new file mode 100644 index 0000000..f7a1566 --- /dev/null +++ b/docs/source/_static/custom.css @@ -0,0 +1 @@ +div.code-block-caption {background: #d0d0d0;} diff --git a/docs/source/cli/index.rst b/docs/source/cli/index.rst new file mode 100644 index 0000000..2d70471 --- /dev/null +++ b/docs/source/cli/index.rst @@ -0,0 +1,8 @@ +CLI +=== + +.. toctree:: + :maxdepth: 1 + + usage + reference diff --git a/docs/source/cli/reference.rst b/docs/source/cli/reference.rst new file mode 100644 index 0000000..f663737 --- /dev/null +++ b/docs/source/cli/reference.rst @@ -0,0 +1,7 @@ +CLI Reference +============= + +.. argparse:: + :module: compute.cli.control + :func: get_parser + :prog: compute diff --git a/docs/source/cli/usage.rst b/docs/source/cli/usage.rst new file mode 100644 index 0000000..789feb6 --- /dev/null +++ b/docs/source/cli/usage.rst @@ -0,0 +1,102 @@ +Usage +===== + +Creating compute instances +-------------------------- + +First place your image into images pool path. + +Create :file:`inatance.yaml` config file with following content. Replace `debian_12.qcow2` with your actual image filename. + +.. code-block:: yaml + :caption: Using prebuilt QCOW2 disk image + :emphasize-lines: 4 + :linenos: + + name: myinstance + memory: 2048 + vcpus: 2 + image: debian_12.qcow2 + volumes: + - type: file + is_system: true + target: vda + capacity: + value: 10 + unit: GiB + +Check out what configuration will be applied when ``init``:: + + compute init -t + +Initialise instance with command:: + + compute init + +Also you can use following syntax:: + + compute init yourfile.yaml + +Start instance:: + + compute start myinstance + +Using ISO installation medium +````````````````````````````` + +Download ISO image and set it as source for ``cdrom`` device. + +Note that the ``image`` parameter is not used here. + +.. code-block:: yaml + :caption: Using ISO image + :emphasize-lines: 11-13 + :linenos: + + name: myinstance + memory: 2048 + vcpus: 2 + volumes: + - type: file + is_system: true + target: vda + capacity: + value: 10 + unit: GiB + - type: file + device: cdrom + source: /images/debian-12.2.0-amd64-netinst.iso + +:: + + compute init + +Now edit instance XML configuration to add VNC-server listen address:: + + virsh edit myinstance + +Add ``address`` attribute to start listen on all host network interfaces. + +.. code-block:: xml + :caption: libvirt XML config fragment + :emphasize-lines: 2 + :linenos: + + + + + +Also you can specify VNC server port. This is **5900** by default. + +Start instance and connect to VNC via any VNC client such as `Remmina `_ or something else. + +:: + + compute start myinstance + +Finish the OS installation over VNC and then do:: + + compute setcdrom myinstance /images/debian-12.2.0-amd64-netinst.iso --detach + compute powrst myinstance + +CDROM will be detached. ``powrst`` command will perform instance shutdown and start. Instance will booted from `vda` disk. diff --git a/docs/source/conf.py b/docs/source/conf.py index 38e4a1f..8fdd50d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,4 +1,3 @@ -# Add ../.. to path for autodoc Sphinx extension import os import sys sys.path.insert(0, os.path.abspath('../..')) @@ -7,16 +6,18 @@ sys.path.insert(0, os.path.abspath('../..')) project = 'Compute' copyright = '2023, Compute Authors' author = 'Compute Authors' -release = '0.1.0' +release = '0.1.0-dev2' # Sphinx general settings extensions = [ 'sphinx.ext.autodoc', 'sphinx_multiversion', + 'sphinxarg.ext', ] templates_path = ['_templates'] exclude_patterns = [] language = 'en' +#pygments_style = 'monokai' # HTML output settings html_theme = 'alabaster' diff --git a/docs/source/index.rst b/docs/source/index.rst index 81222c2..a3cb747 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -3,9 +3,14 @@ Compute Compute instances management library. -.. toctree:: - :maxdepth: 1 +Contents +-------- +.. toctree:: + :maxdepth: 2 + + installation + cli/index pyapi/index Indices and tables diff --git a/docs/source/installation.rst b/docs/source/installation.rst new file mode 100644 index 0000000..40220d3 --- /dev/null +++ b/docs/source/installation.rst @@ -0,0 +1,47 @@ +Installation +============ + +Install Debian 12 on your host system. If you want use virtual machine as host make sure that nested virtualization is enabled. + +1. Download or build ``compute`` DEB packages. +2. Install packages:: + + apt-get install ./compute* + +3. Make sure that ``libvirtd`` and ``dnsmasq`` are enabled and running:: + + systemctl enable --now libvirtd.service + systemctl enable --now dnsmasq.service + +4. Prepare storage pools. You need storage pool for images and for instance volumes. + + :: + + for pool in images volumes; do + virsh pool-define-as $pool dir - - - - "/$pool" + virsh pool-build $pool + virsh pool-start $pool + virsh pool-autostart $pool + done + +5. Prepare env. Set environment variables in your `~/.profile`, `~/.bashrc` or global in `/etc/profile.d/compute` or `/etc/bash.bashrc`: + + .. code-block:: sh + + export CMP_IMAGES_POOL=images + export CMP_VOLUMES_POOL=volumes + + Configuration file is yet not supported. + + Make sure the variables are exported to the environment:: + + printenv | grep CMP_ + + If the command didn't show anything source your rc files or relogin. + +6. Prepare network:: + + virsh net-start default + virsh net-autostart default + +7. Done. Now you can follow `CLI instructions `_ diff --git a/docs/source/pyapi/index.rst b/docs/source/pyapi/index.rst index fa5df88..c2bab9b 100644 --- a/docs/source/pyapi/index.rst +++ b/docs/source/pyapi/index.rst @@ -1,38 +1,8 @@ Python API ========== -The API allows you to perform actions on instances programmatically. - -.. code-block:: python - - import compute - - with compute.Session() as session: - instance = session.get_instance('myinstance') - info = instance.get_info() - - print(info) - - -:class:`Session` context manager provides an abstraction over :class:`libvirt.virConnect` -and returns objects of other classes of the present library. - -Entity representation ---------------------- - -Entities such as a compute-instance are represented as classes. These classes directly -call libvirt methods to perform operations on the hypervisor. An example class is -:class:`Volume`. - -The configuration files of various libvirt objects in `compute` are described by special -dataclasses. The dataclass stores object parameters in its properties and can return an -XML config for libvirt using the ``to_xml()`` method. For example :class:`VolumeConfig`. - -`Pydantic `_ models are used to validate input data. -For example :class:`VolumeSchema`. - -Modules documentation ---------------------- +API Reference +------------- .. toctree:: :maxdepth: 4 diff --git a/docs/source/pyapi/instance/devices.rst b/docs/source/pyapi/instance/devices.rst new file mode 100644 index 0000000..a5d6598 --- /dev/null +++ b/docs/source/pyapi/instance/devices.rst @@ -0,0 +1,5 @@ +``devices`` +=========== + +.. automodule:: compute.instance.devices + :members: diff --git a/docs/source/pyapi/instance/index.rst b/docs/source/pyapi/instance/index.rst index 659ffc2..cafd81f 100644 --- a/docs/source/pyapi/instance/index.rst +++ b/docs/source/pyapi/instance/index.rst @@ -7,4 +7,5 @@ instance guest_agent + devices schemas diff --git a/docs/source/pyapi/utils.rst b/docs/source/pyapi/utils.rst index b5ab60a..e5976f9 100644 --- a/docs/source/pyapi/utils.rst +++ b/docs/source/pyapi/utils.rst @@ -12,3 +12,10 @@ .. automodule:: compute.utils.ids :members: + + +``utils.dictutil`` +------------------ + +.. automodule:: compute.utils.dictutil + :members: diff --git a/packaging/Dockerfile b/packaging/Dockerfile index 4659618..d4092ad 100644 --- a/packaging/Dockerfile +++ b/packaging/Dockerfile @@ -14,6 +14,7 @@ RUN apt-get update; \ python3-setuptools \ python3-sphinx \ python3-sphinx-multiversion \ + python3-sphinx-argparse \ python3-libvirt \ python3-lxml \ python3-yaml \ diff --git a/packaging/files/compute.bash-completion b/packaging/files/compute.bash-completion index c97f241..2f49d77 100644 --- a/packaging/files/compute.bash-completion +++ b/packaging/files/compute.bash-completion @@ -18,8 +18,10 @@ _compute_root_cmd=" status setvcpus setmem - setpass" -_compute_init_opts="" + setpass + setcdrom + delete" +_compute_init_opts="--test --start" _compute_exec_opts="--timeout --executable --env --no-join-args" _compute_ls_opts="" _compute_start_opts="" @@ -33,6 +35,8 @@ _compute_status_opts="" _compute_setvcpus_opts="" _compute_setmem_opts="" _compute_setpass_opts="--encrypted" +_compute_setcdrom_opts="--detach" +_compute_delete_opts="--yes --save-volumes" _compute_complete_instances() { @@ -78,6 +82,8 @@ _compute_complete() setvcpus) _compute_compreply "$_compute_setvcpus_opts";; setmem) _compute_compreply "$_compute_setmem_opts";; setpass) _compute_compreply "$_compute_setpass_opts";; + setcdrom) _compute_compreply "$_compute_setcdrom_opts";; + delete) _compute_compreply "$_compute_delete_opts";; *) COMPREPLY=() esac ;; diff --git a/packaging/files/control b/packaging/files/control index 6b99835..b3dbfba 100644 --- a/packaging/files/control +++ b/packaging/files/control @@ -13,6 +13,7 @@ Build-Depends: python3-all, python3-sphinx, python3-sphinx-multiversion, + python3-sphinx-argparse, python3-libvirt, python3-lxml, python3-yaml, diff --git a/poetry.lock b/poetry.lock index 9aa3eb5..d727410 100644 --- a/poetry.lock +++ b/poetry.lock @@ -696,6 +696,24 @@ docs = ["sphinxcontrib-websupport"] lint = ["docutils-stubs", "flake8 (>=3.5.0)", "flake8-simplify", "isort", "mypy (>=0.990)", "ruff", "sphinx-lint", "types-requests"] test = ["cython (>=3.0)", "filelock", "html5lib", "pytest (>=4.6)", "setuptools (>=67.0)"] +[[package]] +name = "sphinx-argparse" +version = "0.4.0" +description = "A sphinx extension that automatically documents argparse commands and options" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sphinx_argparse-0.4.0-py3-none-any.whl", hash = "sha256:73bee01f7276fae2bf621ccfe4d167af7306e7288e3482005405d9f826f9b037"}, + {file = "sphinx_argparse-0.4.0.tar.gz", hash = "sha256:e0f34184eb56f12face774fbc87b880abdb9017a0998d1ec559b267e9697e449"}, +] + +[package.dependencies] +sphinx = ">=1.2.0" + +[package.extras] +markdown = ["CommonMark (>=0.5.6)"] + [[package]] name = "sphinx-autobuild" version = "2021.3.14" @@ -895,4 +913,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = '^3.11' -content-hash = "e5c07eebe683b92360ec12cada14fc5ccbe4e4add52549bf978f580e551abfb0" +content-hash = "cbded73a481e7c6d6e4c4d5de8e37ac5a53848ab774090a913f1846ef4d7421e" diff --git a/pyproject.toml b/pyproject.toml index b88af68..cc5891c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ isort = '^5.12.0' sphinx = '^7.2.6' sphinx-autobuild = '^2021.3.14' sphinx-multiversion = '^0.2.4' +sphinx-argparse = "^0.4.0" [build-system] requires = ['poetry-core']