diff --git a/.gitignore b/.gitignore index 0712286..eabf06c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ dist/ docs/build/ packaging/build/ +instance.yaml .ruff_cache/ __pycache__/ *.pyc diff --git a/Makefile b/Makefile index 249c194..abd63e3 100644 --- a/Makefile +++ b/Makefile @@ -49,4 +49,4 @@ test-build: build-deb upload-docs: docs-versions ssh root@hitomi 'rm -rf /srv/http/nixhacks.net/hstack/*' - scp -r $(DOCS_DUILDDIR) root@hitomi:/srv/http/nixhacks.net/hstack/ + scp -r $(DOCS_BUILDDIR)/* root@hitomi:/srv/http/nixhacks.net/hstack/ diff --git a/README.md b/README.md index 22c71ce..7d972af 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Compute -Compute instances management library and tools. +Compute instances management library. ## Docs @@ -10,14 +10,14 @@ Run `make serve-docs`. See [Development](#development) below. - [x] Create instances - [x] CDROM -- [ ] cloud-init for provisioning instances +- [x] cloud-init for provisioning instances - [x] Power management - [x] Pause and resume - [x] vCPU hotplug - [x] Memory hotplug - [x] Hot disk resize [not tested] - [x] CPU customization (emulation mode, model, vendor, features) -- [ ] CPU topology customization +- [x] CPU topology customization - [ ] BIOS/UEFI settings - [x] Device attaching - [x] Device detaching @@ -40,6 +40,7 @@ Run `make serve-docs`. See [Development](#development) below. - [ ] Backups - [ ] LXC - [ ] Attaching CDROM from sources: block, (http|https|ftp|ftps|tftp):// +- [ ] Instance clones (thin, fat) ## Development @@ -63,43 +64,7 @@ make build-deb # Installation -Packages can be installed via `dpkg` or `apt-get`: - -``` -# apt-get install ./compute*.deb -``` - -After installation prepare environment, run following command to start libvirtd and create required storage pools: - -``` -# systemctl enable --now libvirtd.service -# virsh net-start default -# virsh net-autostart default -# for pool in images volumes; do - virsh pool-define-as $pool dir - - - - "/$pool" - virsh pool-build $pool - virsh pool-start $pool - virsh pool-autostart $pool -done -``` - -Then set environment variables in your `~/.profile`, `~/.bashrc` or global in `/etc/profile.d/compute` or `/etc/bash.bashrc`: - -``` -export CMP_IMAGES_POOL=images -export CMP_VOLUMES_POOL=volumes -``` - -Configuration file is yet not supported. - -Make sure the variables are exported to the environment: - -``` -printenv | grep CMP_ -``` - -If the command didn't show anything _source_ your rc files or relogin. - +See [Installation](https://nixhacks.net/hstack/compute/master/installation.html). # Basic usage @@ -109,49 +74,17 @@ To get help run: compute --help ``` +See [CLI docs](https://nixhacks.net/hstack/compute/master/cli/index.html) for more info. + Also you can use `compute` as generic Python library. For example: ```python -from compute import Session +import compute -with Session() as session: +with compute.Session() as session: instance = session.get_instance('myinstance') if not instance.is_running(): instance.start() else: print('instance is already running') ``` - -# Create compute instances - -Place your qcow2 image in `/images` directory. For example `debian_12.qcow2`. - -Create `instance.yaml` file with following content: - -```yaml -name: myinstance -memory: 2048 # memory in MiB -vcpus: 2 -image: debian_12.qcow2 -volumes: - - type: file - is_system: true - target: vda - capacity: - value: 10 - unit: GiB -``` - -Refer to `Instance` class docs for more info. Full `instance.yaml` example will be provided later. - -To initialise instance run: - -``` -compute -l debug init instance.yaml -``` - -Start instance: - -``` -compute start myinstance -``` diff --git a/compute/__init__.py b/compute/__init__.py index e0722f1..e5395dc 100644 --- a/compute/__init__.py +++ b/compute/__init__.py @@ -15,8 +15,9 @@ """Compute instances management library.""" -__version__ = '0.1.0-dev2' +__version__ = '0.1.0-dev3' -from .instance import Instance, InstanceConfig, InstanceSchema +from .config import Config +from .instance import CloudInit, Instance, InstanceConfig, InstanceSchema from .session import Session from .storage import StoragePool, Volume, VolumeConfig diff --git a/compute/__main__.py b/compute/__main__.py index 346aaaf..9f277c2 100644 --- a/compute/__main__.py +++ b/compute/__main__.py @@ -15,7 +15,7 @@ """Command line interface for compute module.""" -from compute.cli import control +from compute.cli import parser -control.cli() +parser.run() diff --git a/compute/common.py b/compute/abstract.py similarity index 100% rename from compute/common.py rename to compute/abstract.py diff --git a/compute/cli/commands.py b/compute/cli/commands.py new file mode 100644 index 0000000..7c8a3ef --- /dev/null +++ b/compute/cli/commands.py @@ -0,0 +1,409 @@ +# This file is part of Compute +# +# Compute is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Compute is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Compute. If not, see . + +"""CLI commands.""" + +import argparse +import base64 +import json +import logging +import pathlib +import re +import shlex +import sys +import uuid + +import libvirt +import pydantic +import yaml + +from compute import Session +from compute.cli.term import Table, confirm +from compute.exceptions import GuestAgentTimeoutExpired +from compute.instance import CloudInit, GuestAgent, InstanceSchema +from compute.instance.devices import DiskConfig, DiskDriver +from compute.utils import dictutil, diskutils, ids + + +log = logging.getLogger(__name__) + +libvirt.registerErrorHandler( + lambda userdata, err: None, # noqa: ARG005 + ctx=None, +) + + +def init(session: Session, args: argparse.Namespace) -> None: + """Initialise compute instance using YAML config.""" + try: + data = yaml.load(args.file.read(), Loader=yaml.SafeLoader) + log.debug('Read from file: %s', data) + except yaml.YAMLError as e: + sys.exit(f'error: cannot parse YAML: {e}') + capabilities = session.get_capabilities() + node_info = session.get_node_info() + base_instance_config = { + 'name': str(uuid.uuid4()), + 'title': None, + 'description': None, + 'arch': capabilities.arch, + 'machine': capabilities.machine, + 'emulator': capabilities.emulator, + 'max_vcpus': node_info.cpus, + 'max_memory': node_info.memory, + 'cpu': { + 'emulation_mode': 'host-passthrough', + 'model': None, + 'vendor': None, + 'topology': None, + 'features': None, + }, + 'network_interfaces': [ + { + 'source': 'default', + 'mac': ids.random_mac(), + }, + ], + 'boot': {'order': ['cdrom', 'hd']}, + 'cloud_init': None, + } + data = dictutil.override(base_instance_config, data) + volumes = [] + targets = [] + for volume in data['volumes']: + base_disk_config = { + 'bus': 'virtio', + 'is_readonly': False, + 'driver': { + 'name': 'qemu', + 'type': 'qcow2', + 'cache': 'writethrough', + }, + } + base_cdrom_config = { + 'bus': 'ide', + 'is_readonly': True, + 'driver': { + 'name': 'qemu', + 'type': 'raw', + 'cache': 'writethrough', + }, + } + if volume.get('device') is None: + volume['device'] = 'disk' + if volume.get('target') is None: + prefix = 'hd' if volume['device'] == 'cdrom' else 'vd' + target = diskutils.get_disk_target(targets, prefix) + volume['target'] = target + targets.append(target) + else: + targets.append(volume['target']) + if volume['device'] == 'disk': + volumes.append(dictutil.override(base_disk_config, volume)) + if volume['device'] == 'cdrom': + volumes.append(dictutil.override(base_cdrom_config, volume)) + data['volumes'] = volumes + if data['cloud_init'] is not None: + cloud_init_config = { + 'user_data': None, + 'meta_data': None, + 'vendor_data': None, + 'network_config': None, + } + data['cloud_init'] = dictutil.override( + cloud_init_config, + data['cloud_init'], + ) + for item in data['cloud_init']: + cidata = data['cloud_init'][item] + if cidata is None: + pass + elif isinstance(cidata, str): + if cidata.startswith('base64:'): + data['cloud_init'][item] = base64.b64decode( + cidata.split(':')[1] + ).decode('utf-8') + elif re.fullmatch(r'^[^\n]{1,1024}$', cidata, re.I): + data_file = pathlib.Path(cidata) + if data_file.exists(): + with data_file.open('r') as f: + data['cloud_init'][item] = f.read() + else: + pass + else: + data['cloud_init'][item] = yaml.dump(cidata) + try: + log.debug('Input data: %s', data) + if args.test: + _ = InstanceSchema(**data) + print(json.dumps(dict(data), indent=4, sort_keys=True)) + sys.exit() + instance = session.create_instance(**data) + print(f'Initialised: {instance.name}') + if args.start: + instance.start() + print(f'Started: {instance.name}') + except pydantic.ValidationError as e: + for error in e.errors(): + fields = '.'.join([str(lc) for lc in error['loc']]) + print( + f"validation error: {fields}: {error['msg']}", + file=sys.stderr, + ) + + +def exec_(session: Session, args: argparse.Namespace) -> None: + """ + Execute command in guest via guest agent. + + NOTE: any argument after instance name will be passed into guest's shell + """ + instance = session.get_instance(args.instance) + ga = GuestAgent(instance.domain, timeout=args.timeout) + arguments = args.arguments.copy() + if len(arguments) > 1 and not args.no_join_args: + arguments = [shlex.join(arguments)] + if not args.no_join_args: + arguments.insert(0, '-c') + stdin = None + if not sys.stdin.isatty(): + stdin = sys.stdin.read() + try: + output = ga.guest_exec( + path=args.executable, + args=arguments, + env=args.env, + stdin=stdin, + capture_output=True, + decode_output=True, + poll=True, + ) + except GuestAgentTimeoutExpired as e: + sys.exit( + f'{e}. NOTE: command may still running in guest, ' + f'PID={ga.last_pid}' + ) + if output.stderr: + print(output.stderr.strip(), file=sys.stderr) + if output.stdout: + print(output.stdout.strip(), file=sys.stdout) + sys.exit(output.exitcode) + + +def ls(session: Session, args: argparse.Namespace) -> None: # noqa: ARG001 + """List compute instances.""" + table = Table() + table.header = ['NAME', 'STATE', 'NVCPUS', 'MEMORY'] + for instance in session.list_instances(): + info = instance.get_info() + table.add_row( + [ + instance.name, + instance.get_status() + ' ', + info.nproc, + f'{int(info.memory / 1024)} MiB', + ] + ) + print(table) + + +def lsdisks(session: Session, args: argparse.Namespace) -> None: + """List block devices attached to instance.""" + instance = session.get_instance(args.instance) + if args.persistent: + disks = instance.list_disks(persistent=True) + else: + disks = instance.list_disks() + table = Table() + table.header = ['TARGET', 'SOURCE'] + for disk in disks: + table.add_row([disk.target, disk.source]) + print(table) + + +def start(session: Session, args: argparse.Namespace) -> None: + """Start instance.""" + instance = session.get_instance(args.instance) + instance.start() + + +def shutdown(session: Session, args: argparse.Namespace) -> None: + """Shutdown instance.""" + instance = session.get_instance(args.instance) + if args.soft: + method = 'SOFT' + elif args.hard: + method = 'HARD' + elif args.unsafe: + method = 'UNSAFE' + else: + method = 'NORMAL' + instance.shutdown(method) + + +def reboot(session: Session, args: argparse.Namespace) -> None: + """Reboot instance.""" + instance = session.get_instance(args.instance) + instance.reboot() + + +def reset(session: Session, args: argparse.Namespace) -> None: + """Reset instance.""" + instance = session.get_instance(args.instance) + instance.reset() + + +def powrst(session: Session, args: argparse.Namespace) -> None: + """Power reset instance.""" + instance = session.get_instance(args.instance) + instance.power_reset() + + +def pause(session: Session, args: argparse.Namespace) -> None: + """Pause instance.""" + instance = session.get_instance(args.instance) + instance.pause() + + +def resume(session: Session, args: argparse.Namespace) -> None: + """Resume instance.""" + instance = session.get_instance(args.instance) + instance.resume() + + +def status(session: Session, args: argparse.Namespace) -> None: + """Display instance status.""" + instance = session.get_instance(args.instance) + print(instance.get_status()) + + +def setvcpus(session: Session, args: argparse.Namespace) -> None: + """Set instance vCPU number.""" + instance = session.get_instance(args.instance) + instance.set_vcpus(args.nvcpus, live=True) + + +def setmem(session: Session, args: argparse.Namespace) -> None: + """Set instance memory size.""" + instance = session.get_instance(args.instance) + instance.set_memory(args.memory, live=True) + + +def setpass(session: Session, args: argparse.Namespace) -> None: + """Set user password in guest.""" + instance = session.get_instance(args.instance) + instance.set_user_password( + args.username, + args.password, + encrypted=args.encrypted, + ) + + +def setcdrom(session: Session, args: argparse.Namespace) -> None: + """Manage CDROM devices.""" + instance = session.get_instance(args.instance) + if args.detach: + for disk in instance.list_disks(persistent=True): + if disk.device == 'cdrom' and disk.source == args.source: + instance.detach_disk(disk.target, live=False) + print( + f"disk '{disk.target}' detached, " + 'perform power reset to apply changes' + ) + return + disks_live = instance.list_disks(persistent=False) + disks_inactive = instance.list_disks(persistent=True) + disks = [d.target for d in disks_inactive if d not in disks_live] + target = diskutils.get_disk_target(disks, 'hd') + cdrom = DiskConfig( + type='file', + device='cdrom', + source=args.source, + target=target, + is_readonly=True, + bus='ide', + driver=DiskDriver('qemu', 'raw', 'writethrough'), + ) + instance.attach_device(cdrom, live=False) + print( + f"CDROM attached as disk '{target}', " + 'perform power reset to apply changes' + ) + + +def setcloudinit(session: Session, args: argparse.Namespace) -> None: + """ + Set cloud-init configuration. + + The cloud-init disk must not be mounted to the host system while making + changes using this command! In this case, data may be damaged when writing + to disk - if the new content of the file is longer than the old one, it + will be truncated. + """ + if ( + args.user_data is None + and args.vendor_data is None + and args.network_config is None + and args.meta_data is None + ): + sys.exit('nothing to do') + instance = session.get_instance(args.instance) + disks = instance.list_disks() + cloud_init_disk_path = None + cloud_init_disk_target = diskutils.get_disk_target( + [d.target for d in disks], prefix='vd' + ) + cloud_init = CloudInit() + if args.user_data: + cloud_init.user_data = args.user_data.read() + if args.vendor_data: + cloud_init.vendor_data = args.vendor_data.read() + if args.network_config: + cloud_init.network_config = args.network_config.read() + if args.meta_data: + cloud_init.meta_data = args.meta_data.read() + for disk in disks: + if disk.source.endswith('cloud-init.img'): + cloud_init_disk_path = disk.source + break + if cloud_init_disk_path is None: + volumes = session.get_storage_pool(session.VOLUMES_POOL) + cloud_init_disk_path = volumes.path.joinpath( + f'{instance.name}-cloud-init.img' + ) + cloud_init.create_disk(cloud_init_disk_path) + volumes.refresh() + cloud_init.attach_disk( + cloud_init_disk_path, + cloud_init_disk_target, + instance, + ) + else: + cloud_init.update_disk(cloud_init_disk_path) + + +def delete(session: Session, args: argparse.Namespace) -> None: + """Delete instance with local storage volumes.""" + if args.yes is True or confirm( + 'this action is irreversible, continue?', + default=False, + ): + instance = session.get_instance(args.instance) + if args.save_volumes is False: + instance.delete(with_volumes=True) + else: + instance.delete() + else: + print('aborted') diff --git a/compute/cli/control.py b/compute/cli/control.py deleted file mode 100644 index 2d85cd7..0000000 --- a/compute/cli/control.py +++ /dev/null @@ -1,624 +0,0 @@ -# This file is part of Compute -# -# Compute is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Compute is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Compute. If not, see . - -"""Command line interface.""" - -import argparse -import json -import logging -import os -import re -import shlex -import string -import sys -import uuid - -import libvirt -import yaml -from pydantic import ValidationError - -from compute import __version__ -from compute.exceptions import ComputeError, GuestAgentTimeoutError -from compute.instance import GuestAgent, Instance, InstanceSchema -from compute.instance.devices import DiskConfig, DiskDriver -from compute.session import Session -from compute.utils import dictutil, ids - - -log = logging.getLogger(__name__) -log_levels = [lv.lower() for lv in logging.getLevelNamesMapping()] - -libvirt.registerErrorHandler( - lambda userdata, err: None, # noqa: ARG005 - ctx=None, -) - - -class Table: - """Minimalistic text table constructor.""" - - def __init__(self, whitespace: str | None = None): - """Initialise Table.""" - self.whitespace = whitespace or '\t' - self.header = [] - self.rows = [] - self.table = '' - - def add_row(self, row: list) -> None: - """Add table row.""" - self.rows.append([str(col) for col in row]) - - def add_rows(self, rows: list[list]) -> None: - """Add multiple rows.""" - for row in rows: - self.add_row(row) - - def __str__(self) -> str: - """Build table and return.""" - widths = [max(map(len, col)) for col in zip(*self.rows, strict=True)] - self.rows.insert(0, [str(h).upper() for h in self.header]) - for row in self.rows: - self.table += self.whitespace.join( - ( - val.ljust(width) - for val, width in zip(row, widths, strict=True) - ) - ) - self.table += '\n' - return self.table.strip() - - -def _list_instances(session: Session) -> None: - table = Table() - table.header = ['NAME', 'STATE'] - for instance in session.list_instances(): - table.add_row( - [ - instance.name, - instance.get_status(), - ] - ) - print(table) - sys.exit() - - -def _exec_guest_agent_command( - session: Session, args: argparse.Namespace -) -> None: - instance = session.get_instance(args.instance) - ga = GuestAgent(instance.domain, timeout=args.timeout) - arguments = args.arguments.copy() - if len(arguments) > 1 and not args.no_join_args: - arguments = [shlex.join(arguments)] - if not args.no_join_args: - arguments.insert(0, '-c') - stdin = None - if not sys.stdin.isatty(): - stdin = sys.stdin.read() - try: - output = ga.guest_exec( - path=args.executable, - args=arguments, - env=args.env, - stdin=stdin, - capture_output=True, - decode_output=True, - poll=True, - ) - except GuestAgentTimeoutError as e: - sys.exit( - f'{e}. NOTE: command may still running in guest, ' - f'PID={ga.last_pid}' - ) - if output.stderr: - print(output.stderr.strip(), file=sys.stderr) - if output.stdout: - print(output.stdout.strip(), file=sys.stdout) - sys.exit(output.exitcode) - - -def _init_instance(session: Session, args: argparse.Namespace) -> None: - try: - data = yaml.load(args.file.read(), Loader=yaml.SafeLoader) - log.debug('Read from file: %s', data) - except yaml.YAMLError as e: - sys.exit(f'error: cannot parse YAML: {e}') - capabilities = session.get_capabilities() - node_info = session.get_node_info() - base_instance_config = { - 'name': str(uuid.uuid4()), - 'title': None, - 'description': None, - 'arch': capabilities.arch, - 'machine': capabilities.machine, - 'emulator': capabilities.emulator, - 'max_vcpus': node_info.cpus, - 'max_memory': node_info.memory, - 'cpu': { - 'emulation_mode': 'host-passthrough', - 'model': None, - 'vendor': None, - 'topology': None, - 'features': None, - }, - 'network_interfaces': [ - { - 'source': 'default', - 'mac': ids.random_mac(), - }, - ], - 'boot': {'order': ['cdrom', 'hd']}, - } - data = dictutil.override(base_instance_config, data) - volumes = [] - for volume in data['volumes']: - base_disk_config = { - 'bus': 'virtio', - 'is_readonly': False, - 'driver': { - 'name': 'qemu', - 'type': 'qcow2', - 'cache': 'writethrough', - }, - } - base_cdrom_config = { - 'bus': 'ide', - 'target': 'hda', - 'is_readonly': True, - 'driver': { - 'name': 'qemu', - 'type': 'raw', - 'cache': 'writethrough', - }, - } - if volume.get('device') is None: - volume['device'] = 'disk' - if volume['device'] == 'disk': - volumes.append(dictutil.override(base_disk_config, volume)) - if volume['device'] == 'cdrom': - volumes.append(dictutil.override(base_cdrom_config, volume)) - data['volumes'] = volumes - try: - log.debug('Input data: %s', data) - if args.test: - _ = InstanceSchema(**data) - print(json.dumps(dict(data), indent=4, sort_keys=True)) - sys.exit() - instance = session.create_instance(**data) - print(f'initialised: {instance.name}') - if args.start: - instance.start() - except ValidationError as e: - for error in e.errors(): - fields = '.'.join([str(lc) for lc in error['loc']]) - print( - f"validation error: {fields}: {error['msg']}", - file=sys.stderr, - ) - sys.exit() - - -def _shutdown_instance(session: Session, args: argparse.Namespace) -> None: - instance = session.get_instance(args.instance) - if args.soft: - method = 'SOFT' - elif args.hard: - method = 'HARD' - elif args.unsafe: - method = 'UNSAFE' - else: - method = 'NORMAL' - instance.shutdown(method) - - -def _confirm(message: str, *, default: bool | None = None) -> None: - while True: - match default: - case True: - prompt = 'default: yes' - case False: - prompt = 'default: no' - case _: - prompt = 'no default' - try: - answer = input(f'{message} ({prompt}) ') - except KeyboardInterrupt: - sys.exit('aborted') - if not answer and isinstance(default, bool): - return default - if re.match(r'^y(es)?$', answer, re.I): - return True - if re.match(r'^no?$', answer, re.I): - return False - print("Please respond 'yes' or 'no'") - - -def _delete_instance(session: Session, args: argparse.Namespace) -> None: - if args.yes is True or _confirm( - 'this action is irreversible, continue?', - default=False, - ): - instance = session.get_instance(args.instance) - if args.save_volumes is False: - instance.delete(with_volumes=True) - else: - instance.delete() - else: - print('aborted') - sys.exit() - - -def _get_disk_target(instance: Instance, prefix: str = 'hd') -> str: - disks_live = instance.list_disks(persistent=False) - disks_inactive = instance.list_disks(persistent=True) - disks = [d for d in disks_inactive if d not in disks_live] - devs = [d.target[-1] for d in disks if d.target.startswith(prefix)] - return prefix + [x for x in string.ascii_lowercase if x not in devs][0] # noqa: RUF015 - - -def _manage_cdrom(session: Session, args: argparse.Namespace) -> None: - instance = session.get_instance(args.instance) - if args.detach: - for disk in instance.list_disks(persistent=True): - if disk.device == 'cdrom' and disk.source == args.source: - instance.detach_disk(disk.target, live=False) - print( - f"disk '{disk.target}' detached, " - 'perform power reset to apply changes' - ) - return - target = _get_disk_target(instance, 'hd') - cdrom = DiskConfig( - type='file', - device='cdrom', - source=args.source, - target=target, - is_readonly=True, - bus='ide', - driver=DiskDriver('qemu', 'raw', 'writethrough'), - ) - instance.attach_device(cdrom, live=False) - print( - f"CDROM attached as disk '{target}', " - 'perform power reset to apply changes' - ) - - -def main(session: Session, args: argparse.Namespace) -> None: - """Perform actions.""" - match args.command: - case 'init': - _init_instance(session, args) - case 'exec': - _exec_guest_agent_command(session, args) - case 'ls': - _list_instances(session) - case 'start': - instance = session.get_instance(args.instance) - instance.start() - case 'shutdown': - _shutdown_instance(session, args) - case 'reboot': - instance = session.get_instance(args.instance) - instance.reboot() - case 'reset': - instance = session.get_instance(args.instance) - instance.reset() - case 'powrst': - instance = session.get_instance(args.instance) - instance.power_reset() - case 'pause': - instance = session.get_instance(args.instance) - instance.pause() - case 'resume': - instance = session.get_instance(args.instance) - instance.resume() - case 'status': - instance = session.get_instance(args.instance) - print(instance.status) - case 'setvcpus': - instance = session.get_instance(args.instance) - instance.set_vcpus(args.nvcpus, live=True) - case 'setmem': - instance = session.get_instance(args.instance) - instance.set_memory(args.memory, live=True) - case 'setpass': - instance = session.get_instance(args.instance) - instance.set_user_password( - args.username, - args.password, - encrypted=args.encrypted, - ) - case 'setcdrom': - _manage_cdrom(session, args) - case 'delete': - _delete_instance(session, args) - - -def get_parser() -> argparse.ArgumentParser: # noqa: PLR0915 - """Return command line arguments parser.""" - root = argparse.ArgumentParser( - prog='compute', - description='Manage compute instances.', - formatter_class=argparse.RawTextHelpFormatter, - ) - root.add_argument( - '-v', - '--verbose', - action='store_true', - default=False, - help='enable verbose mode', - ) - root.add_argument( - '-c', - '--connect', - metavar='URI', - help='libvirt connection URI', - ) - root.add_argument( - '-l', - '--log-level', - type=str.lower, - metavar='LEVEL', - choices=log_levels, - help='log level', - ) - root.add_argument( - '-V', - '--version', - action='version', - version=__version__, - ) - subparsers = root.add_subparsers(dest='command', metavar='COMMAND') - - # init command - init = subparsers.add_parser( - 'init', help='initialise instance using YAML config file' - ) - init.add_argument( - 'file', - type=argparse.FileType('r', encoding='UTF-8'), - nargs='?', - default='instance.yaml', - help='instance config [default: instance.yaml]', - ) - init.add_argument( - '-s', - '--start', - action='store_true', - default=False, - help='start instance after init', - ) - init.add_argument( - '-t', - '--test', - action='store_true', - default=False, - help='just print resulting instance config as JSON and exit', - ) - - # exec subcommand - execute = subparsers.add_parser( - 'exec', - help='execute command in guest via guest agent', - description=( - 'Execute command in guest via guest agent. ' - 'NOTE: any argument after instance name will be passed into ' - 'guest as shell command.' - ), - ) - execute.add_argument('instance') - execute.add_argument('arguments', nargs=argparse.REMAINDER) - execute.add_argument( - '-t', - '--timeout', - type=int, - default=60, - help=( - 'waiting time in seconds for a command to be executed ' - 'in guest [default: 60]' - ), - ) - execute.add_argument( - '-x', - '--executable', - default='/bin/sh', - help='path to executable in guest [default: /bin/sh]', - ) - execute.add_argument( - '-e', - '--env', - type=str, - nargs='?', - action='append', - help='environment variables to pass to executable in guest', - ) - execute.add_argument( - '-n', - '--no-join-args', - action='store_true', - default=False, - help=( - "do not join arguments list and add '-c' option, suitable " - 'for non-shell executables and other specific cases.' - ), - ) - - # ls subcommand - listall = subparsers.add_parser('ls', help='list instances') - listall.add_argument( - '-a', - '--all', - action='store_true', - default=False, - help='list all instances including inactive', - ) - - # start subcommand - start = subparsers.add_parser('start', help='start instance') - start.add_argument('instance') - - # shutdown subcommand - shutdown = subparsers.add_parser('shutdown', help='shutdown instance') - shutdown.add_argument('instance') - shutdown_opts = shutdown.add_mutually_exclusive_group() - shutdown_opts.add_argument( - '-s', - '--soft', - action='store_true', - help='normal guest OS shutdown, guest agent is used', - ) - shutdown_opts.add_argument( - '-n', - '--normal', - action='store_true', - help='shutdown with hypervisor selected method [default]', - ) - shutdown_opts.add_argument( - '-H', - '--hard', - action='store_true', - help=( - "gracefully destroy instance, it's like long " - 'pressing the power button' - ), - ) - shutdown_opts.add_argument( - '-u', - '--unsafe', - action='store_true', - help=( - 'destroy instance, this is similar to a power outage ' - 'and may result in data loss or corruption' - ), - ) - - # reboot subcommand - reboot = subparsers.add_parser('reboot', help='reboot instance') - reboot.add_argument('instance') - - # reset subcommand - reset = subparsers.add_parser('reset', help='reset instance') - reset.add_argument('instance') - - # powrst subcommand - powrst = subparsers.add_parser('powrst', help='power reset instance') - powrst.add_argument('instance') - - # pause subcommand - pause = subparsers.add_parser('pause', help='pause instance') - pause.add_argument('instance') - - # resume subcommand - resume = subparsers.add_parser('resume', help='resume paused instance') - resume.add_argument('instance') - - # status subcommand - status = subparsers.add_parser('status', help='display instance status') - status.add_argument('instance') - - # setvcpus subcommand - setvcpus = subparsers.add_parser('setvcpus', help='set vCPU number') - setvcpus.add_argument('instance') - setvcpus.add_argument('nvcpus', type=int) - - # setmem subcommand - setmem = subparsers.add_parser('setmem', help='set memory size') - setmem.add_argument('instance') - setmem.add_argument('memory', type=int, help='memory in MiB') - - # setpass subcommand - setpass = subparsers.add_parser( - 'setpass', - help='set user password in guest', - ) - setpass.add_argument('instance') - setpass.add_argument('username') - setpass.add_argument('password') - setpass.add_argument( - '-e', - '--encrypted', - action='store_true', - default=False, - help='set it if password is already encrypted', - ) - - # setcdrom subcommand - setcdrom = subparsers.add_parser('setcdrom', help='manage CDROM devices') - setcdrom.add_argument('instance') - setcdrom.add_argument('source', help='source for CDROM') - setcdrom.add_argument( - '-d', - '--detach', - action='store_true', - default=False, - help='detach CDROM device', - ) - - # delete subcommand - delete = subparsers.add_parser( - 'delete', - help='delete instance', - ) - delete.add_argument('instance') - delete.add_argument( - '-y', - '--yes', - action='store_true', - default=False, - help='automatic yes to prompt', - ) - delete.add_argument( - '--save-volumes', - action='store_true', - default=False, - help='do not delete local storage volumes', - ) - - return root - - -def cli() -> None: - """Run arguments parser.""" - root = get_parser() - args = root.parse_args() - if args.command is None: - root.print_help() - sys.exit() - log_level = args.log_level or os.getenv('CMP_LOG') - if isinstance(log_level, str) and log_level.lower() in log_levels: - logging.basicConfig( - level=logging.getLevelNamesMapping()[log_level.upper()] - ) - log.debug('CLI started with args: %s', args) - connect_uri = ( - args.connect - or os.getenv('CMP_LIBVIRT_URI') - or os.getenv('LIBVIRT_DEFAULT_URI') - or 'qemu:///system' - ) - try: - with Session(connect_uri) as session: - main(session, args) - except ComputeError as e: - sys.exit(f'error: {e}') - except KeyboardInterrupt: - sys.exit() - except SystemExit as e: - sys.exit(e) - - -if __name__ == '__main__': - cli() diff --git a/compute/cli/parser.py b/compute/cli/parser.py new file mode 100644 index 0000000..4517e4b --- /dev/null +++ b/compute/cli/parser.py @@ -0,0 +1,471 @@ +# This file is part of Compute +# +# Compute is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Compute is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Compute. If not, see . + +"""Command line argument parser.""" + +import argparse +import logging +import os +import sys +import textwrap +from collections.abc import Callable +from typing import NamedTuple + +from compute import Session, __version__ +from compute.cli import commands +from compute.exceptions import ComputeError + + +log = logging.getLogger(__name__) +log_levels = [lv.lower() for lv in logging.getLevelNamesMapping()] + + +class Doc(NamedTuple): + """Parsed docstring.""" + + help: str # noqa: A003 + desc: str + + +def get_doc(func: Callable) -> Doc: + """Extract help message and description from function docstring.""" + doc = func.__doc__ + if isinstance(doc, str): + doc = textwrap.dedent(doc).strip().split('\n\n') + return Doc(doc[0][0].lower() + doc[0][1:], '\n\n'.join(doc)) + return Doc('', '') + + +def get_parser() -> argparse.ArgumentParser: + """Return command line argument parser.""" + root = argparse.ArgumentParser( + prog='compute', + description='Manage compute instances.', + ) + root.add_argument( + '-V', + '--version', + action='version', + version=__version__, + ) + root.add_argument( + '-c', + '--connect', + dest='root_connect', + metavar='URI', + help='libvirt connection URI', + ) + root.add_argument( + '-l', + '--log-level', + dest='root_log_level', + type=str.lower, + choices=log_levels, + metavar='LEVEL', + help='log level', + ) + + # common options + common = argparse.ArgumentParser(add_help=False) + common.add_argument( + '-c', + '--connect', + metavar='URI', + help='libvirt connection URI', + ) + common.add_argument( + '-l', + '--log-level', + type=str.lower, + choices=log_levels, + metavar='LEVEL', + help='log level', + ) + + subparsers = root.add_subparsers(dest='command', metavar='COMMAND') + + # init command + init = subparsers.add_parser( + 'init', + parents=[common], + formatter_class=argparse.RawTextHelpFormatter, + help=get_doc(commands.init).help, + description=get_doc(commands.init).desc, + ) + init.add_argument( + 'file', + type=argparse.FileType('r', encoding='UTF-8'), + nargs='?', + default='instance.yaml', + help='instance config [default: instance.yaml]', + ) + init.add_argument( + '-s', + '--start', + action='store_true', + default=False, + help='start instance after init', + ) + init.add_argument( + '-t', + '--test', + action='store_true', + default=False, + help='just print resulting instance config as JSON and exit', + ) + init.set_defaults(func=commands.init) + + # exec subcommand + execute = subparsers.add_parser( + 'exec', + parents=[common], + formatter_class=argparse.RawTextHelpFormatter, + help=get_doc(commands.exec_).help, + description=get_doc(commands.exec_).desc, + ) + execute.add_argument('instance') + execute.add_argument('arguments', nargs=argparse.REMAINDER) + execute.add_argument( + '-t', + '--timeout', + type=int, + default=60, + help=( + 'waiting time in seconds for a command to be executed ' + 'in guest [default: 60]' + ), + ) + execute.add_argument( + '-x', + '--executable', + default='/bin/sh', + help='path to executable in guest [default: /bin/sh]', + ) + execute.add_argument( + '-e', + '--env', + type=str, + nargs='?', + action='append', + help='environment variables to pass to executable in guest', + ) + execute.add_argument( + '-n', + '--no-join-args', + action='store_true', + default=False, + help=( + "do not join arguments list and add '-c' option, suitable " + 'for non-shell executables and other specific cases.' + ), + ) + execute.set_defaults(func=commands.exec_) + + # ls subcommand + ls = subparsers.add_parser( + 'ls', + parents=[common], + formatter_class=argparse.RawTextHelpFormatter, + help=get_doc(commands.ls).help, + description=get_doc(commands.ls).desc, + ) + ls.set_defaults(func=commands.ls) + + # lsdisks subcommand + lsdisks = subparsers.add_parser( + 'lsdisks', + parents=[common], + formatter_class=argparse.RawTextHelpFormatter, + help=get_doc(commands.lsdisks).help, + description=get_doc(commands.lsdisks).desc, + ) + lsdisks.add_argument('instance') + lsdisks.add_argument( + '-p', + '--persistent', + action='store_true', + default=False, + help='display only persisnent devices', + ) + lsdisks.set_defaults(func=commands.lsdisks) + + # start subcommand + start = subparsers.add_parser( + 'start', + parents=[common], + formatter_class=argparse.RawTextHelpFormatter, + help=get_doc(commands.start).help, + description=get_doc(commands.start).desc, + ) + start.add_argument('instance') + start.set_defaults(func=commands.start) + + # shutdown subcommand + shutdown = subparsers.add_parser( + 'shutdown', + parents=[common], + formatter_class=argparse.RawTextHelpFormatter, + help=get_doc(commands.shutdown).help, + description=get_doc(commands.shutdown).desc, + ) + shutdown.add_argument('instance') + shutdown_opts = shutdown.add_mutually_exclusive_group() + shutdown_opts.add_argument( + '-s', + '--soft', + action='store_true', + help='normal guest OS shutdown, guest agent is used', + ) + shutdown_opts.add_argument( + '-n', + '--normal', + action='store_true', + help='shutdown with hypervisor selected method [default]', + ) + shutdown_opts.add_argument( + '-H', + '--hard', + action='store_true', + help=( + "gracefully destroy instance, it's like long " + 'pressing the power button' + ), + ) + shutdown_opts.add_argument( + '-u', + '--unsafe', + action='store_true', + help=( + 'destroy instance, this is similar to a power outage ' + 'and may result in data loss or corruption' + ), + ) + shutdown.set_defaults(func=commands.shutdown) + + # reboot subcommand + reboot = subparsers.add_parser( + 'reboot', + parents=[common], + formatter_class=argparse.RawTextHelpFormatter, + help=get_doc(commands.reboot).help, + description=get_doc(commands.reboot).desc, + ) + reboot.add_argument('instance') + reboot.set_defaults(func=commands.reboot) + + # reset subcommand + reset = subparsers.add_parser( + 'reset', + parents=[common], + formatter_class=argparse.RawTextHelpFormatter, + help=get_doc(commands.reset).help, + description=get_doc(commands.reset).desc, + ) + reset.add_argument('instance') + reset.set_defaults(func=commands.reset) + + # powrst subcommand + powrst = subparsers.add_parser( + 'powrst', + parents=[common], + formatter_class=argparse.RawTextHelpFormatter, + help=get_doc(commands.powrst).help, + description=get_doc(commands.powrst).desc, + ) + powrst.add_argument('instance') + powrst.set_defaults(func=commands.powrst) + + # pause subcommand + pause = subparsers.add_parser( + 'pause', + parents=[common], + formatter_class=argparse.RawTextHelpFormatter, + help=get_doc(commands.pause).help, + description=get_doc(commands.pause).desc, + ) + pause.add_argument('instance') + pause.set_defaults(func=commands.pause) + + # resume subcommand + resume = subparsers.add_parser( + 'resume', + parents=[common], + formatter_class=argparse.RawTextHelpFormatter, + help=get_doc(commands.resume).help, + description=get_doc(commands.resume).desc, + ) + resume.add_argument('instance') + resume.set_defaults(func=commands.resume) + + # status subcommand + status = subparsers.add_parser( + 'status', + parents=[common], + formatter_class=argparse.RawTextHelpFormatter, + help=get_doc(commands.status).help, + description=get_doc(commands.status).desc, + ) + status.add_argument('instance') + status.set_defaults(func=commands.status) + + # setvcpus subcommand + setvcpus = subparsers.add_parser( + 'setvcpus', + parents=[common], + formatter_class=argparse.RawTextHelpFormatter, + help=get_doc(commands.setvcpus).help, + description=get_doc(commands.setvcpus).desc, + ) + setvcpus.add_argument('instance') + setvcpus.add_argument('nvcpus', type=int) + setvcpus.set_defaults(func=commands.setvcpus) + + # setmem subcommand + setmem = subparsers.add_parser( + 'setmem', + parents=[common], + formatter_class=argparse.RawTextHelpFormatter, + help=get_doc(commands.setmem).help, + description=get_doc(commands.setmem).desc, + ) + setmem.add_argument('instance') + setmem.add_argument('memory', type=int, help='memory in MiB') + setmem.set_defaults(func=commands.setmem) + + # setpass subcommand + setpass = subparsers.add_parser( + 'setpass', + parents=[common], + formatter_class=argparse.RawTextHelpFormatter, + help=get_doc(commands.setpass).help, + description=get_doc(commands.setpass).desc, + ) + setpass.add_argument('instance') + setpass.add_argument('username') + setpass.add_argument('password') + setpass.add_argument( + '-e', + '--encrypted', + action='store_true', + default=False, + help='set it if password is already encrypted', + ) + setpass.set_defaults(func=commands.setpass) + + # setcdrom subcommand + setcdrom = subparsers.add_parser( + 'setcdrom', + parents=[common], + formatter_class=argparse.RawTextHelpFormatter, + help=get_doc(commands.setcdrom).help, + description=get_doc(commands.setcdrom).desc, + ) + setcdrom.add_argument('instance') + setcdrom.add_argument('source', help='source for CDROM') + setcdrom.add_argument( + '-d', + '--detach', + action='store_true', + default=False, + help='detach CDROM device', + ) + setcdrom.set_defaults(func=commands.setcdrom) + + # setcloudinit subcommand + setcloudinit = subparsers.add_parser( + 'setcloudinit', + parents=[common], + formatter_class=argparse.RawTextHelpFormatter, + help=get_doc(commands.setcloudinit).help, + description=get_doc(commands.setcloudinit).desc, + ) + setcloudinit.add_argument('instance') + setcloudinit.add_argument( + '--user-data', + type=argparse.FileType('r'), + help='user-data file', + ) + setcloudinit.add_argument( + '--vendor-data', + type=argparse.FileType('r'), + help='vendor-data file', + ) + setcloudinit.add_argument( + '--meta-data', + type=argparse.FileType('r'), + help='meta-data file', + ) + setcloudinit.add_argument( + '--network-config', + type=argparse.FileType('r'), + help='network-config file', + ) + setcloudinit.set_defaults(func=commands.setcloudinit) + + # delete subcommand + delete = subparsers.add_parser( + 'delete', + parents=[common], + formatter_class=argparse.RawTextHelpFormatter, + help=get_doc(commands.delete).help, + description=get_doc(commands.delete).desc, + ) + delete.add_argument('instance') + delete.add_argument( + '-y', + '--yes', + action='store_true', + default=False, + help='automatic yes to prompt', + ) + delete.add_argument( + '--save-volumes', + action='store_true', + default=False, + help='do not delete local storage volumes', + ) + delete.set_defaults(func=commands.delete) + + return root + + +def run() -> None: + """Run argument parser.""" + parser = get_parser() + args = parser.parse_args() + if args.command is None: + parser.print_help() + sys.exit() + log_level = args.root_log_level or args.log_level or os.getenv('CMP_LOG') + if isinstance(log_level, str) and log_level.lower() in log_levels: + logging.basicConfig( + level=logging.getLevelNamesMapping()[log_level.upper()] + ) + log.debug('CLI started with args: %s', args) + connect_uri = ( + args.root_connect + or args.connect + or os.getenv('CMP_LIBVIRT_URI') + or os.getenv('LIBVIRT_DEFAULT_URI') + or 'qemu:///system' + ) + try: + with Session(connect_uri) as session: + # Invoke command + args.func(session, args) + except ComputeError as e: + sys.exit(f'error: {e}') + except KeyboardInterrupt: + sys.exit() diff --git a/compute/cli/term.py b/compute/cli/term.py new file mode 100644 index 0000000..7bae76f --- /dev/null +++ b/compute/cli/term.py @@ -0,0 +1,77 @@ +# This file is part of Compute +# +# Compute is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Compute is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Compute. If not, see . + +"""Utils for creating terminal output and interface elements.""" + +import re +import sys + + +class Table: + """Minimalistic text table constructor.""" + + def __init__(self, whitespace: str | None = None): + """Initialise Table.""" + self.whitespace = whitespace or '\t' + self.header = [] + self.rows = [] + self.table = '' + + def add_row(self, row: list) -> None: + """Add table row.""" + self.rows.append([str(col) for col in row]) + + def add_rows(self, rows: list[list]) -> None: + """Add multiple rows.""" + for row in rows: + self.add_row(row) + + def __str__(self) -> str: + """Return table.""" + widths = [max(map(len, col)) for col in zip(*self.rows, strict=True)] + self.rows.insert(0, [str(h).upper() for h in self.header]) + for row in self.rows: + widths = widths or [len(i) for i in row] + self.table += self.whitespace.join( + ( + val.ljust(width) + for val, width in zip(row, widths, strict=True) + ) + ) + self.table += '\n' + return self.table.strip() + + +def confirm(message: str, *, default: bool | None = None) -> None: + """Start yes/no interactive dialog.""" + while True: + match default: + case True: + prompt = 'default: yes' + case False: + prompt = 'default: no' + case _: + prompt = 'no default' + try: + answer = input(f'{message} ({prompt}) ') + except KeyboardInterrupt: + sys.exit('aborted') + if not answer and isinstance(default, bool): + return default + if re.match(r'^y(es)?$', answer, re.I): + return True + if re.match(r'^no?$', answer, re.I): + return False + print("Please respond 'yes' or 'no'") diff --git a/compute/config.py b/compute/config.py new file mode 100644 index 0000000..a84c224 --- /dev/null +++ b/compute/config.py @@ -0,0 +1,121 @@ +# This file is part of Compute +# +# Compute is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Compute is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Compute. If not, see . + +"""Configuration loader.""" + +__all__ = ['Config', 'ConfigSchema'] + +import os +import tomllib +from collections import UserDict +from pathlib import Path +from typing import ClassVar + +from .abstract import EntityModel +from .exceptions import ConfigLoaderError +from .utils import dictutil + + +class LibvirtConfigSchema(EntityModel): + """Schema for libvirt config.""" + + uri: str + + +class LogConfigSchema(EntityModel): + """Logger congif schema.""" + + level: str | None = None + file: str | None = None + + +class StorageConfigSchema(EntityModel): + """Storage config schema.""" + + volumes: str + images: str + + +class ConfigSchema(EntityModel): + """Configuration file schema.""" + + libvirt: LibvirtConfigSchema | None + log: LogConfigSchema | None + storage: StorageConfigSchema + + +class Config(UserDict): + """ + UserDict for storing configuration. + + Environment variables prefix is ``CMP_``. Environment variables + have higher proirity then configuration file. + + :cvar str IMAGES_POOL: images storage pool name taken from env + :cvar str VOLUMES_POOL: volumes storage pool name taken from env + :cvar Path DEFAULT_CONFIG_FILE: :file:`/etc/computed/computed.toml` + :cvar dict DEFAULT_CONFIGURATION: + """ + + LIBVIRT_URI = os.getenv('CMP_LIBVIRT_URI') + IMAGES_POOL = os.getenv('CMP_IMAGES_POOL') + VOLUMES_POOL = os.getenv('CMP_VOLUMES_POOL') + + DEFAULT_CONFIG_FILE = Path('/etc/compute/computed.toml') + DEFAULT_CONFIGURATION: ClassVar[dict] = { + 'libvirt': { + 'uri': 'qemu:///system', + }, + 'log': { + 'level': None, + 'file': None, + }, + 'storage': { + 'images': 'images', + 'volumes': 'volumes', + }, + } + + def __init__(self, file: Path | None = None): + """ + Initialise Config. + + :param file: Path to configuration file. If `file` is None + use default path from :var:`Config.DEFAULT_CONFIG_FILE`. + """ + self.file = Path(file) if file else self.DEFAULT_CONFIG_FILE + try: + if self.file.exists(): + with self.file.open('rb') as configfile: + loaded = tomllib.load(configfile) + else: + loaded = {} + except tomllib.TOMLDecodeError as etoml: + raise ConfigLoaderError( + f'Bad TOML syntax: {self.file}: {etoml}' + ) from etoml + except (OSError, ValueError) as eread: + raise ConfigLoaderError( + f'Config read error: {self.file}: {eread}' + ) from eread + config = dictutil.override(self.DEFAULT_CONFIGURATION, loaded) + if self.LIBVIRT_URI: + config['libvirt']['uri'] = self.LIBVIRT_URI + if self.VOLUMES_POOL: + config['storage']['volumes'] = self.VOLUMES_POOL + if self.IMAGES_POOL: + config['storage']['images'] = self.IMAGES_POOL + ConfigSchema(**config) + super().__init__(config) diff --git a/compute/exceptions.py b/compute/exceptions.py index 0d8b9af..2a41da5 100644 --- a/compute/exceptions.py +++ b/compute/exceptions.py @@ -36,12 +36,12 @@ class GuestAgentUnavailableError(GuestAgentError): """Guest agent is not connected or is unavailable.""" -class GuestAgentTimeoutError(GuestAgentError): - """QEMU timeout exceeded.""" +class GuestAgentTimeoutExpired(GuestAgentError): # noqa: N818 + """QEMU timeout expired.""" def __init__(self, seconds: int): - """Initialise GuestAgentTimeoutExceededError.""" - super().__init__(f'QEMU timeout ({seconds} sec) exceeded') + """Initialise GuestAgentTimeoutExpired.""" + super().__init__(f'QEMU timeout ({seconds} sec) expired') class GuestAgentCommandNotSupportedError(GuestAgentError): diff --git a/compute/instance/__init__.py b/compute/instance/__init__.py index 42ddaf2..f569360 100644 --- a/compute/instance/__init__.py +++ b/compute/instance/__init__.py @@ -13,6 +13,7 @@ # You should have received a copy of the GNU General Public License # along with Compute. If not, see . +from .cloud_init import CloudInit from .guest_agent import GuestAgent from .instance import Instance, InstanceConfig from .schemas import InstanceSchema diff --git a/compute/instance/cloud_init.py b/compute/instance/cloud_init.py new file mode 100644 index 0000000..79bf679 --- /dev/null +++ b/compute/instance/cloud_init.py @@ -0,0 +1,221 @@ +# This file is part of Compute +# +# Compute is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Compute is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Compute. If not, see . + +# ruff: noqa: S603 + +""" +`Cloud-init`_ integration for bootstraping compute instances. + +.. _Cloud-init: https://cloudinit.readthedocs.io +""" + +import logging +import subprocess +import tempfile +from pathlib import Path + +from compute.exceptions import InstanceError + +from .devices import DiskConfig, DiskDriver +from .instance import Instance + + +log = logging.getLogger(__name__) + + +class CloudInit: + """ + Cloud-init integration. + + :ivar str user_data: user-data. + :ivar str vendor_data: vendor-data. + :ivar str network_config: network-config. + :ivar str meta_data: meta-data. + """ + + def __init__(self): + """Initialise :class:`CloudInit`.""" + self.user_data = None + self.vendor_data = None + self.network_config = None + self.meta_data = None + + def __repr__(self) -> str: + """Represent :class:`CloudInit` object.""" + return ( + self.__class__.__name__ + + '(' + + ', '.join( + [ + f'{self.user_data=}', + f'{self.vendor_data=}', + f'{self.network_config=}', + f'{self.meta_data=}', + ] + ) + + ')' + ).replace('self.', '') + + def _write_to_disk( + self, + disk: str, + filename: str, + data: str | None, + *, + force_file_create: bool = False, + delete_existing_file: bool = False, + default_data: str | None = None, + ) -> None: + data = data or default_data + log.debug('Input data %s: %r', filename, data) + if isinstance(data, str): + data = data.encode() + if data is None and force_file_create is False: + return + with tempfile.NamedTemporaryFile() as data_file: + if data is not None: + data_file.write(data) + data_file.flush() + if delete_existing_file: + log.debug('Deleting existing file') + filelist = subprocess.run( + ['/usr/bin/mdir', '-i', disk, '-b'], + capture_output=True, + check=True, + ) + files = [ + f.replace('::/', '') + for f in filelist.stdout.decode().splitlines() + ] + log.debug('Files on disk: %s', files) + log.debug("Removing '%s'", filename) + if filename in files: + subprocess.run( + ['/usr/bin/mdel', '-i', disk, f'::{filename}'], + check=True, + ) + log.debug("Writing file '%s'", filename) + subprocess.run( + [ + '/usr/bin/mcopy', + '-i', + disk, + data_file.name, + f'::{filename}', + ], + check=True, + ) + + def create_disk(self, disk: Path, *, force: bool = False) -> None: + """ + Create disk with cloud-init config files. + + :param path: Disk path. + :param force: Replace existing disk. + """ + if not isinstance(disk, Path): + disk = Path(disk) + if disk.exists(): + if disk.is_file() is False: + raise InstanceError('Cloud-init disk must be regular file') + if force: + disk.unlink() + else: + raise InstanceError('File already exists') + subprocess.run( + ['/usr/sbin/mkfs.vfat', '-n', 'CIDATA', '-C', str(disk), '1024'], + check=True, + stderr=subprocess.DEVNULL, + ) + self._write_to_disk( + disk=disk, + filename='user-data', + data=self.user_data, + force_file_create=True, + default_data='#cloud-config', + ) + self._write_to_disk( + disk=disk, + filename='vendor-data', + data=self.vendor_data, + ) + self._write_to_disk( + disk=disk, + filename='network-config', + data=self.network_config, + ) + self._write_to_disk( + disk=disk, + filename='meta-data', + data=self.meta_data, + force_file_create=True, + ) + + def update_disk(self, disk: Path) -> None: + """Update files on existing disk.""" + if not isinstance(disk, Path): + disk = Path(disk) + if not disk.exists(): + raise InstanceError(f"File '{disk}' does not exists") + if self.user_data: + self._write_to_disk( + disk=disk, + filename='user-data', + data=self.user_data, + force_file_create=True, + default_data='#cloud-config', + delete_existing_file=True, + ) + if self.vendor_data: + self._write_to_disk( + disk=disk, + filename='vendor-data', + data=self.vendor_data, + delete_existing_file=True, + ) + if self.network_config: + self._write_to_disk( + disk=disk, + filename='network-config', + data=self.network_config, + delete_existing_file=True, + ) + if self.meta_data: + self._write_to_disk( + disk=disk, + filename='meta-data', + data=self.meta_data, + force_file_create=True, + delete_existing_file=True, + ) + + def attach_disk(self, disk: Path, target: str, instance: Instance) -> None: + """ + Attach cloud-init disk to instance. + + :param disk: Path to disk. + :param instance: Compute instance object. + """ + instance.attach_device( + DiskConfig( + type='file', + device='disk', + source=disk, + target=target, + is_readonly=True, + bus='virtio', + driver=DiskDriver('qemu', 'raw'), + ) + ) diff --git a/compute/instance/devices.py b/compute/instance/devices.py index eba09f2..03673cc 100644 --- a/compute/instance/devices.py +++ b/compute/instance/devices.py @@ -24,7 +24,7 @@ from typing import Union from lxml import etree from lxml.builder import E -from compute.common import DeviceConfig +from compute.abstract import DeviceConfig from compute.exceptions import InvalidDeviceConfigError @@ -32,9 +32,9 @@ from compute.exceptions import InvalidDeviceConfigError class DiskDriver: """Disk driver description for libvirt.""" - name: str - type: str - cache: str + name: str = 'qemu' + type: str = 'qcow2' + cache: str = 'default' def __call__(self): """Return self.""" @@ -56,13 +56,7 @@ class DiskConfig(DeviceConfig): is_readonly: bool = False device: str = 'disk' bus: str = 'virtio' - driver: DiskDriver = field( - default_factory=DiskDriver( - name='qemu', - type='qcow2', - cache='writethrough', - ) - ) + driver: DiskDriver = field(default_factory=DiskDriver()) def to_xml(self) -> str: """Return XML config for libvirt.""" @@ -99,13 +93,14 @@ class DiskConfig(DeviceConfig): pretty_print=True, ).strip() driver = xml.find('driver') + cachetype = driver.get('cache') disk_params = { 'type': xml.get('type'), 'device': xml.get('device'), 'driver': DiskDriver( name=driver.get('name'), type=driver.get('type'), - cache=driver.get('cache'), + **({'cache': cachetype} if cachetype else {}), ), 'source': xml.find('source').get('file'), 'target': xml.find('target').get('dev'), @@ -122,7 +117,7 @@ class DiskConfig(DeviceConfig): if driver_param is None: msg = ( "'driver' tag must have " - "'name', 'type' and 'cache' attributes" + "'name' and 'type' attributes" ) raise InvalidDeviceConfigError(msg, xml_str) return cls(**disk_params) diff --git a/compute/instance/guest_agent.py b/compute/instance/guest_agent.py index a6f62da..d1b52cc 100644 --- a/compute/instance/guest_agent.py +++ b/compute/instance/guest_agent.py @@ -27,7 +27,7 @@ import libvirt_qemu from compute.exceptions import ( GuestAgentCommandNotSupportedError, GuestAgentError, - GuestAgentTimeoutError, + GuestAgentTimeoutExpired, GuestAgentUnavailableError, ) @@ -114,7 +114,7 @@ class GuestAgent: if command not in supported: raise GuestAgentCommandNotSupportedError(command) - def guest_exec( # noqa: PLR0913 + def guest_exec( self, path: str, args: list[str] | None = None, @@ -199,7 +199,7 @@ class GuestAgent: sleep(poll_interval) now = time() if now - start_time > self.timeout: - raise GuestAgentTimeoutError(self.timeout) + raise GuestAgentTimeoutExpired(self.timeout) log.debug( 'Polling command pid=%s finished, time taken: %s seconds', pid, diff --git a/compute/instance/instance.py b/compute/instance/instance.py index a626a3c..16fb960 100644 --- a/compute/instance/instance.py +++ b/compute/instance/instance.py @@ -25,7 +25,7 @@ import libvirt from lxml import etree from lxml.builder import E -from compute.common import DeviceConfig, EntityConfig +from compute.abstract import DeviceConfig, EntityConfig from compute.exceptions import ( GuestAgentCommandNotSupportedError, InstanceError, @@ -141,6 +141,14 @@ class InstanceConfig(EntityConfig): ) ) xml.append(self._gen_cpu_xml(self.cpu)) + xml.append( + E.clock( + E.timer(name='rtc', tickpolicy='catchup'), + E.timer(name='pit', tickpolicy='delay'), + E.timer(name='hpet', present='no'), + offset='utc', + ) + ) os = E.os(E.type('hvm', machine=self.machine, arch=self.arch)) for dev in self.boot.order: os.append(E.boot(dev=dev)) @@ -159,7 +167,7 @@ class InstanceConfig(EntityConfig): devices.append(E.emulator(str(self.emulator))) for interface in self.network_interfaces: devices.append(self._gen_network_interface_xml(interface)) - devices.append(E.graphics(type='vnc', port='-1', autoport='yes')) + devices.append(E.graphics(type='vnc', autoport='yes')) devices.append(E.input(type='tablet', bus='usb')) devices.append( E.channel( @@ -171,6 +179,7 @@ class InstanceConfig(EntityConfig): type='unix', ) ) + devices.append(E.serial(E.target(port='0'), type='pty')) devices.append( E.console(E.target(type='serial', port='0'), type='pty') ) @@ -212,10 +221,30 @@ class Instance: :param domain: libvirt domain object """ - self.domain = domain - self.connection = domain.connect() - self.name = domain.name() - self.guest_agent = GuestAgent(domain) + self._domain = domain + self._connection = domain.connect() + self._name = domain.name() + self._guest_agent = GuestAgent(domain) + + @property + def connection(self) -> libvirt.virConnect: + """Libvirt connection object.""" + return self._connection + + @property + def domain(self) -> libvirt.virDomain: + """Libvirt domain object.""" + return self._domain + + @property + def name(self) -> str: + """Instance name.""" + return self._name + + @property + def guest_agent(self) -> GuestAgent: + """:class:`GuestAgent` object.""" + return self._guest_agent def _expand_instance_state(self, state: int) -> str: states = { @@ -279,6 +308,9 @@ class Instance: def get_max_vcpus(self) -> int: """Maximum vCPUs number for domain.""" + if not self.is_running(): + xml = etree.fromstring(self.dump_xml(inactive=True)) + return int(xml.xpath('/domain/vcpu/text()')[0]) return self.domain.maxVcpus() def start(self) -> None: @@ -324,7 +356,6 @@ class Instance: :param method: Method used to shutdown instance """ if not self.is_running(): - log.warning('Instance is not running, nothing to do') return methods = { 'SOFT': libvirt.VIR_DOMAIN_SHUTDOWN_GUEST_AGENT, @@ -737,13 +768,24 @@ class Instance: :param with_volumes: If True delete local volumes with instance. """ + log.info("Shutdown instance '%s'", self.name) self.shutdown(method='HARD') disks = self.list_disks(persistent=True) log.debug('Disks list: %s', disks) for disk in disks: if with_volumes and disk.type == 'file': - volume = self.connection.storageVolLookupByPath(disk.source) - log.debug('Delete volume: %s', volume.path()) + try: + volume = self.connection.storageVolLookupByPath( + disk.source + ) + except libvirt.libvirtError as e: + if e.get_error_code() == libvirt.VIR_ERR_NO_STORAGE_VOL: + log.warning( + "Volume '%s' not found, skipped", + disk.source, + ) + continue + log.info('Delete volume: %s', volume.path()) volume.delete() - log.debug('Undefine instance') + log.info('Undefine instance') self.domain.undefine() diff --git a/compute/instance/schemas.py b/compute/instance/schemas.py index 1983826..1b61a6a 100644 --- a/compute/instance/schemas.py +++ b/compute/instance/schemas.py @@ -22,7 +22,7 @@ from pathlib import Path from pydantic import ValidationError, validator from pydantic.error_wrappers import ErrorWrapper -from compute.common import EntityModel +from compute.abstract import EntityModel from compute.utils.units import DataUnit @@ -109,6 +109,15 @@ class BootOptionsSchema(EntityModel): order: tuple +class CloudInitSchema(EntityModel): + """Cloud-init config model.""" + + user_data: str | None = None + meta_data: str | None = None + vendor_data: str | None = None + network_config: str | None = None + + class InstanceSchema(EntityModel): """Compute instance model.""" @@ -127,6 +136,7 @@ class InstanceSchema(EntityModel): volumes: list[VolumeSchema] network_interfaces: list[NetworkInterfaceSchema] image: str | None = None + cloud_init: CloudInitSchema | None = None @validator('name') def _check_name(cls, value: str) -> str: # noqa: N805 diff --git a/compute/session.py b/compute/session.py index 5fae003..a6b462a 100644 --- a/compute/session.py +++ b/compute/session.py @@ -16,7 +16,6 @@ """Hypervisor session manager.""" import logging -import os from contextlib import AbstractContextManager from types import TracebackType from typing import Any, NamedTuple @@ -25,19 +24,23 @@ from uuid import uuid4 import libvirt from lxml import etree +from .config import Config from .exceptions import ( InstanceNotFoundError, SessionError, StoragePoolNotFoundError, ) from .instance import Instance, InstanceConfig, InstanceSchema +from .instance.cloud_init import CloudInit from .instance.devices import DiskConfig, DiskDriver from .storage import StoragePool, VolumeConfig -from .utils import units +from .utils import diskutils, units log = logging.getLogger(__name__) +config = Config() + class Capabilities(NamedTuple): """Store domain capabilities info.""" @@ -72,27 +75,20 @@ class NodeInfo(NamedTuple): class Session(AbstractContextManager): - """ - Hypervisor session context manager. - - :cvar IMAGES_POOL: images storage pool name taken from env - :cvar VOLUMES_POOL: volumes storage pool name taken from env - """ - - IMAGES_POOL = os.getenv('CMP_IMAGES_POOL') - VOLUMES_POOL = os.getenv('CMP_VOLUMES_POOL') + """Hypervisor session context manager.""" def __init__(self, uri: str | None = None): """ Initialise session with hypervisor. - :ivar str uri: libvirt connection URI. - :ivar libvirt.virConnect connection: libvirt connection object. - :param uri: libvirt connection URI. """ - self.uri = uri or 'qemu:///system' - self.connection = libvirt.open(self.uri) + log.debug('Config=%s', config) + self.LIBVIRT_URI = config['libvirt']['uri'] + self.IMAGES_POOL = config['storage']['images'] + self.VOLUMES_POOL = config['storage']['volumes'] + self._uri = uri or self.LIBVIRT_URI + self._connection = libvirt.open(self._uri) def __enter__(self): """Return Session object.""" @@ -107,6 +103,16 @@ class Session(AbstractContextManager): """Close the connection when leaving the context.""" self.close() + @property + def uri(self) -> str: + """Libvirt connection URI.""" + return self._uri + + @property + def connection(self) -> libvirt.virConnect: + """Libvirt connection object.""" + return self._connection + def close(self) -> None: """Close connection to libvirt daemon.""" self.connection.close() @@ -207,38 +213,51 @@ class Session(AbstractContextManager): """ data = InstanceSchema(**kwargs) config = InstanceConfig(data) - log.info('Define XML...') - log.info(config.to_xml()) + log.info('Define instance XML') + log.debug(config.to_xml()) try: self.connection.defineXML(config.to_xml()) except libvirt.libvirtError as e: raise SessionError(f'Error defining instance: {e}') from e - log.info('Getting instance...') + log.info('Getting instance object...') instance = self.get_instance(config.name) log.info('Start processing volumes...') + log.info('Connecting to images pool...') + images_pool = self.get_storage_pool(self.IMAGES_POOL) + images_pool.refresh() + log.info('Connecting to volumes pool...') + volumes_pool = self.get_storage_pool(self.VOLUMES_POOL) + volumes_pool.refresh() + disk_targets = [] for volume in data.volumes: log.info('Processing volume=%s', volume) - log.info('Connecting to images pool...') - images_pool = self.get_storage_pool(self.IMAGES_POOL) - log.info('Connecting to volumes pool...') - volumes_pool = self.get_storage_pool(self.VOLUMES_POOL) log.info('Building volume configuration...') + capacity = None + disk_targets.append(volume.target) if not volume.source: - vol_name = f'{uuid4()}.qcow2' + volume_name = f'{uuid4()}.qcow2' else: - vol_name = volume.source + volume_name = volume.source if volume.device == 'cdrom': - log.debug('Volume %s is CDROM device', vol_name) + log.info('Volume %s is CDROM device', volume_name) + elif volume.source is not None: + log.info('Using volume %s as source', volume_name) + volume_source = volume.source + if volume.capacity: + capacity = units.to_bytes( + volume.capacity.value, volume.capacity.unit + ) else: capacity = units.to_bytes( volume.capacity.value, volume.capacity.unit ) - vol_conf = VolumeConfig( - name=vol_name, - path=str(volumes_pool.path.joinpath(vol_name)), + volume_config = VolumeConfig( + name=volume_name, + path=str(volumes_pool.path.joinpath(volume_name)), capacity=capacity, ) - log.info('Volume configuration is:\n %s', vol_conf.to_xml()) + volume_source = volume_config.path + log.debug('Volume config: %s', volume_config) if volume.is_system is True and data.image: log.info( "Volume is marked as 'system', start cloning image..." @@ -246,21 +265,22 @@ class Session(AbstractContextManager): log.info('Get image %s', data.image) image = images_pool.get_volume(data.image) log.info('Cloning image into volumes pool...') - vol = volumes_pool.clone_volume(image, vol_conf) - log.info( - 'Resize cloned volume to specified size: %s', - capacity, - ) - vol.resize(capacity, unit=units.DataUnit.BYTES) + vol = volumes_pool.clone_volume(image, volume_config) else: - log.info('Create volume...') - volumes_pool.create_volume(vol_conf) + log.info('Create volume %s', volume_config.name) + volumes_pool.create_volume(volume_config) + if capacity is not None: + log.info( + 'Resize cloned volume to specified size: %s', + capacity, + ) + vol.resize(capacity, unit=units.DataUnit.BYTES) log.info('Attaching volume to instance...') instance.attach_device( DiskConfig( type=volume.type, device=volume.device, - source=vol_conf.path, + source=volume_source, target=volume.target, is_readonly=volume.is_readonly, bus=volume.bus, @@ -271,6 +291,24 @@ class Session(AbstractContextManager): ), ) ) + if data.cloud_init: + log.info('Crating disk for cloud-init...') + cloud_init = CloudInit() + cloud_init.user_data = data.cloud_init.user_data + cloud_init.vendor_data = data.cloud_init.vendor_data + cloud_init.network_config = data.cloud_init.network_config + cloud_init.meta_data = data.cloud_init.meta_data + cloud_init_disk_path = volumes_pool.path.joinpath( + f'{instance.name}-cloud-init.img' + ) + cloud_init.create_disk(cloud_init_disk_path) + log.info('Attaching cloud-init disk to instance...') + volumes_pool.refresh() + cloud_init.attach_disk( + cloud_init_disk_path, + diskutils.get_disk_target(disk_targets, prefix='vd'), + instance, + ) return instance def get_instance(self, name: str) -> Instance: diff --git a/compute/storage/pool.py b/compute/storage/pool.py index 3f906bd..222e1bf 100644 --- a/compute/storage/pool.py +++ b/compute/storage/pool.py @@ -15,7 +15,11 @@ """Manage storage pools.""" +import datetime import logging +import time +from datetime import datetime as dt +from datetime import timedelta from pathlib import Path from typing import NamedTuple @@ -65,10 +69,32 @@ class StoragePool: """Return storage pool XML description as string.""" return self.pool.XMLDesc() - def refresh(self) -> None: - """Refresh storage pool.""" - # TODO @ge: handle libvirt asynchronous job related exceptions - self.pool.refresh() + def refresh(self, *, retry: bool = True, timeout: int = 30) -> None: + """ + Refresh storage pool. + + :param retry: If True retry pool refresh on :class:`libvirtError` + with running asynchronous jobs. + :param timeout: Retry timeout in secodns. Affets only if `retry` + is True. + """ + retry_timeout = dt.now(tz=datetime.UTC) + timedelta(seconds=timeout) + while dt.now(tz=datetime.UTC) < retry_timeout: + try: + self.pool.refresh() + except libvirt.libvirtError as e: + if 'asynchronous jobs running' in e.get_error_message(): + if retry is False: + raise StoragePoolError(e) from e + log.debug( + 'An error ocurred when refreshing storage pool ' + 'retrying after 1 sec...' + ) + time.sleep(1) + else: + raise StoragePoolError(e) from e + else: + return def create_volume(self, vol_conf: VolumeConfig) -> Volume: """Create storage volume and return Volume instance.""" diff --git a/compute/storage/volume.py b/compute/storage/volume.py index bc028d0..87739ff 100644 --- a/compute/storage/volume.py +++ b/compute/storage/volume.py @@ -23,7 +23,7 @@ import libvirt from lxml import etree from lxml.builder import E -from compute.common import EntityConfig +from compute.abstract import EntityConfig from compute.utils import units diff --git a/compute/utils/config_loader.py b/compute/utils/config_loader.py deleted file mode 100644 index f6fcbd7..0000000 --- a/compute/utils/config_loader.py +++ /dev/null @@ -1,56 +0,0 @@ -# This file is part of Compute -# -# Compute is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Compute is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Compute. If not, see . - -"""Configuration loader.""" - -import tomllib -from collections import UserDict -from pathlib import Path - -from compute.exceptions import ConfigLoaderError - - -DEFAULT_CONFIGURATION = {} -DEFAULT_CONFIG_FILE = '/etc/computed/computed.toml' - - -class ConfigLoader(UserDict): - """UserDict for storing configuration.""" - - def __init__(self, file: Path | None = None): - """ - Initialise ConfigLoader. - - :param file: Path to configuration file. If `file` is None - use default path from DEFAULT_CONFIG_FILE constant. - """ - # TODO @ge: load deafult configuration - self.file = Path(file) if file else Path(DEFAULT_CONFIG_FILE) - super().__init__(self.load()) - - def load(self) -> dict: - """Load confguration object from TOML file.""" - try: - with Path(self.file).open('rb') as configfile: - return tomllib.load(configfile) - # TODO @ge: add config schema validation - except tomllib.TOMLDecodeError as tomlerr: - raise ConfigLoaderError( - f'Bad TOML syntax in config file: {self.file}: {tomlerr}' - ) from tomlerr - except (OSError, ValueError) as readerr: - raise ConfigLoaderError( - f'Cannot read config file: {self.file}: {readerr}' - ) from readerr diff --git a/compute/utils/diskutils.py b/compute/utils/diskutils.py new file mode 100644 index 0000000..19a9a7f --- /dev/null +++ b/compute/utils/diskutils.py @@ -0,0 +1,45 @@ +# This file is part of Compute +# +# Compute is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Compute is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Compute. If not, see . + +"""Auxiliary functions for working with disks.""" + +import string + + +def get_disk_target( + disks: list[str], prefix: str, *, from_end: bool = False +) -> str: + """ + Return free disk name. + + .. code-block:: shell-session + + >>> get_disk_target(['vda', 'vdb'], 'vd') + 'vdc' + >>> get_disk_target(['vda', 'vdc'], 'vd') + 'vdb' + >>> get_disk_target(['vda', 'vdd'], 'vd', from_end=True) + 'vdz' + >>> get_disk_target(['vda', 'hda'], 'hd') + 'hdb' + + :param disks: List of attached disk names. + :param prefix: Disk name prefix. + :param from_end: If True select a drive letter starting from the + end of the alphabet. + """ + index = -1 if from_end else 0 + devs = [d[-1] for d in disks if d.startswith(prefix)] + return prefix + [x for x in string.ascii_lowercase if x not in devs][index] diff --git a/computed.toml b/computed.toml new file mode 100644 index 0000000..9e4c50c --- /dev/null +++ b/computed.toml @@ -0,0 +1,13 @@ +[libvirt] +# Libvirt connection URI. +# See https://libvirt.org/uri.html#qemu-qemu-and-kvm-uris +#uri = 'qemu:///system' + +[storage] +# Name of libvirt storage pool to store compute instance etalon images. +# compute takes images from here and creates disks for compute instances +# based on them. +#images = 'images' + +# Name of libvirt storage pool to store compute instance disks. +#volumes = 'volumes' diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css index f7a1566..4b2051d 100644 --- a/docs/source/_static/custom.css +++ b/docs/source/_static/custom.css @@ -1 +1,2 @@ div.code-block-caption {background: #d0d0d0;} +a:visited {color: #004B6B;} diff --git a/docs/source/cli/cloud_init.rst b/docs/source/cli/cloud_init.rst new file mode 100644 index 0000000..1aa0614 --- /dev/null +++ b/docs/source/cli/cloud_init.rst @@ -0,0 +1,127 @@ +Using Cloud-init +================ + +Cloud-init for new instances +---------------------------- + +Cloud-init configs may be set inplace into :file:`instance.yaml`. + +.. code-block:: yaml + :caption: Example with Debian generic QCOW2 image + :linenos: + + name: genericdebian + memory: 1024 + vcpus: 1 + image: debian-12-generic-amd64.qcow2 + volumes: + - type: file + is_system: true + capacity: + value: 5 + unit: GiB + cloud_init: + meta_data: + hostname: genericdebian + root_pass: secure_pass + user_data: | + ## template: jinja + #cloud-config + merge_how: + - name: list + settings: [append] + hostname: {{ ds.meta_data.hostname }} + fqdn: {{ ds.meta_data.hostname }}.instances.generic.cloud + manage_etc_hosts: true + chpasswd: + users: + - name: root + password: {{ ds.meta_data.root_pass }} + type: text + expire: False + ssh_pwauth: True + package_update: true + package_upgrade: true + packages: + - qemu-guest-agent + - vim + - psmisc + - htop + runcmd: + - [ systemctl, daemon-reload ] + - [ systemctl, enable, qemu-guest-agent.service ] + - [ systemctl, start, --no-block, qemu-guest-agent.service ] + +You can use separate file in this way: + +.. code-block:: yaml + :caption: user-data in separate file + :emphasize-lines: 11- + :linenos: + + name: genericdebian + memory: 1024 + vcpus: 1 + image: debian-12-generic-amd64.qcow2 + volumes: + - type: file + is_system: true + capacity: + value: 25 + unit: GiB + cloud_init: + user_data: user-data.yaml + +Base64 encoded string with data must be ``base64:`` prefixed: + +.. code-block:: yaml + :caption: user-data as base64 encoded string + :emphasize-lines: 11- + :linenos: + + name: genericdebian + memory: 1024 + vcpus: 1 + image: debian-12-generic-amd64.qcow2 + volumes: + - type: file + is_system: true + capacity: + value: 25 + unit: GiB + cloud_init: + user_data: base64:I2Nsb3VkLWNvbmZpZwpob3N0bmFtZTogY2xvdWRlYmlhbgpmcWRuOiBjbG91ZGViaWFuLmV4YW1wbGUuY29tCm1hbmFnZV9ldGNfaG9zdHM6IHRydWUK + +Also you can write config in YAML. Please note that in this case you will not be able to use the ``#cloud-config`` shebang. + +.. code-block:: yaml + :caption: meta-data as nested YAML + :emphasize-lines: 12-14 + :linenos: + + name: genericdebian + memory: 1024 + vcpus: 1 + image: debian-12-generic-amd64.qcow2 + volumes: + - type: file + is_system: true + capacity: + value: 25 + unit: GiB + cloud_init: + meta_data: + myvar: example + another_one: example_2 + user_data: | + #cloud-config + #something here + +Edit Cloud-init config files on existing instance +------------------------------------------------- + +Use ``setcloudinit`` subcommand:: + + compute setcloudinit myinstance --user-data user_data.yaml + +See `setcloudinit <../cli/reference.html#setcloudinit>`_ for details. diff --git a/docs/source/cli/usage.rst b/docs/source/cli/getting_started.rst similarity index 59% rename from docs/source/cli/usage.rst rename to docs/source/cli/getting_started.rst index 789feb6..1524f60 100644 --- a/docs/source/cli/usage.rst +++ b/docs/source/cli/getting_started.rst @@ -1,12 +1,21 @@ -Usage -===== +Getting started +=============== Creating compute instances -------------------------- -First place your image into images pool path. +Compute instances are created through a description in yaml format. The description may be partial, the configuration will be supplemented with default parameters. -Create :file:`inatance.yaml` config file with following content. Replace `debian_12.qcow2` with your actual image filename. +This page describes how to start up a basic instance, you'll probably want to use cloud-init to get the guest up and running, see the instructions at `Using cloud-init `_. + +The following examples contains minimal instance configuration. See also full example `here `_ + +Using prebuilt QCOW2 disk image +``````````````````````````````` + +First place your image into ``images`` pool path. + +Create :file:`instance.yaml` config file with following content. Replace `debian_12.qcow2` with your actual image filename. .. code-block:: yaml :caption: Using prebuilt QCOW2 disk image @@ -20,14 +29,13 @@ Create :file:`inatance.yaml` config file with following content. Replace `debian volumes: - type: file is_system: true - target: vda capacity: value: 10 unit: GiB Check out what configuration will be applied when ``init``:: - compute init -t + compute init --test Initialise instance with command:: @@ -50,7 +58,7 @@ Note that the ``image`` parameter is not used here. .. code-block:: yaml :caption: Using ISO image - :emphasize-lines: 11-13 + :emphasize-lines: 10-12 :linenos: name: myinstance @@ -59,7 +67,6 @@ Note that the ``image`` parameter is not used here. volumes: - type: file is_system: true - target: vda capacity: value: 10 unit: GiB @@ -80,9 +87,8 @@ Add ``address`` attribute to start listen on all host network interfaces. .. code-block:: xml :caption: libvirt XML config fragment :emphasize-lines: 2 - :linenos: - + @@ -96,7 +102,31 @@ Start instance and connect to VNC via any VNC client such as `Remmina `_ + You can use configuration file :file:`/etc/compute/computed.toml` or environment + variables. + + You can set environment variables in your :file:`~/.profile`, :file:`~/.bashrc` + or globally in :file:`/etc/profile.d/compute` or :file:`/etc/bash.bashrc`. For example: .. code-block:: sh + export CMP_LIBVIRT_URI=qemu:///system export CMP_IMAGES_POOL=images export CMP_VOLUMES_POOL=volumes - Configuration file is yet not supported. - Make sure the variables are exported to the environment:: printenv | grep CMP_ - If the command didn't show anything source your rc files or relogin. - 6. Prepare network:: virsh net-start default diff --git a/docs/source/pyapi/exceptions.rst b/docs/source/pyapi/exceptions.rst index 3912721..e5e3baa 100644 --- a/docs/source/pyapi/exceptions.rst +++ b/docs/source/pyapi/exceptions.rst @@ -1,5 +1,5 @@ -``exceptions`` -============== +``exceptions`` — Exceptions +=========================== .. automodule:: compute.exceptions :members: diff --git a/docs/source/pyapi/instance/cloud_init.rst b/docs/source/pyapi/instance/cloud_init.rst new file mode 100644 index 0000000..e23a4f3 --- /dev/null +++ b/docs/source/pyapi/instance/cloud_init.rst @@ -0,0 +1,5 @@ +``cloud_init`` +============== + +.. automodule:: compute.instance.cloud_init + :members: diff --git a/docs/source/pyapi/instance/guest_agent.rst b/docs/source/pyapi/instance/guest_agent.rst index 1305140..82d6b7d 100644 --- a/docs/source/pyapi/instance/guest_agent.rst +++ b/docs/source/pyapi/instance/guest_agent.rst @@ -3,4 +3,3 @@ .. automodule:: compute.instance.guest_agent :members: - :special-members: __init__ diff --git a/docs/source/pyapi/instance/index.rst b/docs/source/pyapi/instance/index.rst index cafd81f..d5266b3 100644 --- a/docs/source/pyapi/instance/index.rst +++ b/docs/source/pyapi/instance/index.rst @@ -1,11 +1,12 @@ -``instance`` -============ +``instance`` — Manage compute instances +======================================= .. toctree:: - :maxdepth: 1 + :maxdepth: 3 :caption: Contents: instance guest_agent devices + cloud_init schemas diff --git a/docs/source/pyapi/instance/instance.rst b/docs/source/pyapi/instance/instance.rst index 3c58f1f..945c1e6 100644 --- a/docs/source/pyapi/instance/instance.rst +++ b/docs/source/pyapi/instance/instance.rst @@ -3,4 +3,3 @@ .. automodule:: compute.instance.instance :members: - :special-members: __init__ diff --git a/docs/source/pyapi/session.rst b/docs/source/pyapi/session.rst index 2dec16e..968ad89 100644 --- a/docs/source/pyapi/session.rst +++ b/docs/source/pyapi/session.rst @@ -1,6 +1,5 @@ -``session`` -=========== +``session`` — Hypervisor session manager +======================================== .. automodule:: compute.session :members: - :special-members: __init__ diff --git a/docs/source/pyapi/storage/index.rst b/docs/source/pyapi/storage/index.rst index e9ea734..b6c38af 100644 --- a/docs/source/pyapi/storage/index.rst +++ b/docs/source/pyapi/storage/index.rst @@ -1,8 +1,8 @@ -``storage`` -============ +``storage`` — Manage storage pools and volumes +============================================== .. toctree:: - :maxdepth: 1 + :maxdepth: 3 :caption: Contents: pool diff --git a/docs/source/pyapi/storage/pool.rst b/docs/source/pyapi/storage/pool.rst index 398124e..d3b2a99 100644 --- a/docs/source/pyapi/storage/pool.rst +++ b/docs/source/pyapi/storage/pool.rst @@ -3,4 +3,3 @@ .. automodule:: compute.storage.pool :members: - :special-members: __init__ diff --git a/docs/source/pyapi/storage/volume.rst b/docs/source/pyapi/storage/volume.rst index e1ba8d0..0623324 100644 --- a/docs/source/pyapi/storage/volume.rst +++ b/docs/source/pyapi/storage/volume.rst @@ -3,4 +3,3 @@ .. automodule:: compute.storage.volume :members: - :special-members: __init__ diff --git a/docs/source/pyapi/utils.rst b/docs/source/pyapi/utils.rst index e5976f9..b5487ee 100644 --- a/docs/source/pyapi/utils.rst +++ b/docs/source/pyapi/utils.rst @@ -1,5 +1,5 @@ -``utils`` -========= +``utils`` — Common utils +======================== ``utils.units`` --------------- @@ -19,3 +19,10 @@ .. automodule:: compute.utils.dictutil :members: + + +``utils.diskutils`` +------------------- + +.. automodule:: compute.utils.diskutils + :members: diff --git a/packaging/build.sh b/packaging/build.sh index 575c5d5..50af8f0 100644 --- a/packaging/build.sh +++ b/packaging/build.sh @@ -11,5 +11,5 @@ sed -e "s%\.\./\.\.%$PWD%" -i ../docs/source/conf.py dh_make --copyright gpl3 --yes --python --file ../compute-*[.tar.gz] rm debian/*.ex debian/README.{Debian,source} debian/*.docs sed -e 's/\* Initial release.*/\* This is the development build, see commits in upstream repo for info./' -i debian/changelog -cp -v ../../files/{control,rules,copyright,docs,compute.bash-completion} debian/ +cp -v ../../files/{control,rules,copyright,docs,compute.bash-completion,install} debian/ dpkg-buildpackage -us -uc diff --git a/packaging/files/compute.bash-completion b/packaging/files/compute.bash-completion index 2f49d77..1a1a5bd 100644 --- a/packaging/files/compute.bash-completion +++ b/packaging/files/compute.bash-completion @@ -1,13 +1,13 @@ # compute bash completion script +_compute_global_opts="--connect --log-level" _compute_root_cmd=" + $_compute_global_opts --version - --verbose - --connect - --log-level init exec ls + lsdisks start shutdown reboot @@ -20,23 +20,34 @@ _compute_root_cmd=" setmem setpass setcdrom + setcloudinit delete" -_compute_init_opts="--test --start" -_compute_exec_opts="--timeout --executable --env --no-join-args" -_compute_ls_opts="" -_compute_start_opts="" -_compute_shutdown_opts="--soft --normal --hard --unsafe" -_compute_reboot_opts="" -_compute_reset_opts="" -_compute_powrst_opts="" -_compute_pause_opts="" -_compute_resume_opts="" -_compute_status_opts="" -_compute_setvcpus_opts="" -_compute_setmem_opts="" -_compute_setpass_opts="--encrypted" -_compute_setcdrom_opts="--detach" -_compute_delete_opts="--yes --save-volumes" +_compute_init_opts="$_compute_global_opts --test --start" +_compute_exec_opts="$_compute_global_opts + --timeout + --executable + --env + --no-join-args" +_compute_ls_opts="$_compute_global_opts" +_compute_lsdisks_opts="$_compute_global_opts --persistent" +_compute_start_opts="$_compute_global_opts" +_compute_shutdown_opts="$_compute_global_opts --soft --normal --hard --unsafe" +_compute_reboot_opts="$_compute_global_opts" +_compute_reset_opts="$_compute_global_opts" +_compute_powrst_opts="$_compute_global_opts" +_compute_pause_opts="$_compute_global_opts" +_compute_resume_opts="$_compute_global_opts" +_compute_status_opts="$_compute_global_opts" +_compute_setvcpus_opts="$_compute_global_opts" +_compute_setmem_opts="$_compute_global_opts" +_compute_setpass_opts="$_compute_global_opts --encrypted" +_compute_setcdrom_opts="$_compute_global_opts --detach" +_compute_setcloudinit_opts="$_compute_global_opts + --user-data + --vendor-data + --meta-data + --network-config" +_compute_delete_opts="$_compute_global_opts --yes --save-volumes" _compute_complete_instances() { @@ -49,12 +60,19 @@ _compute_complete_instances() _compute_compreply() { + local cgopts= + + if [[ "$1" == '-f' ]]; then + cgopts="-f" + shift + fi + if [[ "$current" = [a-z]* ]]; then _compute_compwords="$(_compute_complete_instances)" else _compute_compwords="$*" fi - COMPREPLY=($(compgen -W "$_compute_compwords" -- "$current")) + COMPREPLY=($(compgen $cgopts -W "$_compute_compwords" -- "$current")) } _compute_complete() @@ -68,9 +86,10 @@ _compute_complete() nshift=$((COMP_CWORD-1)) previous="${COMP_WORDS[COMP_CWORD-nshift]}" case "$previous" in - init) COMPREPLY=($(compgen -f -- "$current"));; + init) COMPREPLY=($(compgen -f -W "$_compute_init_opts" -- "$current"));; exec) _compute_compreply "$_compute_exec_opts";; ls) COMPREPLY=($(compgen -W "$_compute_ls_opts" -- "$current"));; + lsdisks) _compute_compreply "$_compute_lsdisks_opts";; start) _compute_compreply "$_compute_start_opts";; shutdown) _compute_compreply "$_compute_shutdown_opts";; reboot) _compute_compreply "$_compute_reboot_opts";; @@ -83,6 +102,7 @@ _compute_complete() setmem) _compute_compreply "$_compute_setmem_opts";; setpass) _compute_compreply "$_compute_setpass_opts";; setcdrom) _compute_compreply "$_compute_setcdrom_opts";; + setcloudinit) _compute_compreply -f "$_compute_setcloudinit_opts";; delete) _compute_compreply "$_compute_delete_opts";; *) COMPREPLY=() esac diff --git a/packaging/files/control b/packaging/files/control index b3dbfba..8f0cd99 100644 --- a/packaging/files/control +++ b/packaging/files/control @@ -33,12 +33,14 @@ Depends: python3-libvirt, python3-lxml, python3-yaml, - python3-pydantic + python3-pydantic, + mtools, + dosfstools Recommends: dnsmasq Suggests: compute-doc -Description: Compute instances management library and tools (Python 3) +Description: Compute instances management library (Python 3) Package: compute-doc Section: doc @@ -46,4 +48,4 @@ Architecture: all Depends: ${sphinxdoc:Depends}, ${misc:Depends}, -Description: Compute instances management library and tools (documentation) +Description: Compute instances management library (documentation) diff --git a/packaging/files/install b/packaging/files/install new file mode 100644 index 0000000..d31aa85 --- /dev/null +++ b/packaging/files/install @@ -0,0 +1 @@ +computed.toml etc/compute/ diff --git a/pyproject.toml b/pyproject.toml index cc5891c..29f3951 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,11 @@ [tool.poetry] name = 'compute' -version = '0.1.0-dev2' -description = 'Compute instances management library and tools' +version = '0.1.0-dev3' +description = 'Compute instances management library' +license = 'GPL-3.0-or-later' authors = ['ge '] readme = 'README.md' +include = ['computed.toml'] [tool.poetry.dependencies] python = '^3.11' @@ -13,7 +15,7 @@ pydantic = '1.10.4' pyyaml = "^6.0.1" [tool.poetry.scripts] -compute = 'compute.cli.control:cli' +compute = 'compute.cli.parser:run' [tool.poetry.group.dev.dependencies] ruff = '^0.1.3' @@ -56,6 +58,8 @@ ignore = [ 'EM101', 'TD003', 'TD006', 'FIX002', + 'C901', + 'PLR0912', 'PLR0913', 'PLR0915', ] exclude = ['__init__.py']