diff --git a/Makefile b/Makefile index abd63e3..1dbea2c 100644 --- a/Makefile +++ b/Makefile @@ -48,5 +48,5 @@ test-build: build-deb scp packaging/build/compute*.deb vm:~ upload-docs: docs-versions - ssh root@hitomi 'rm -rf /srv/http/nixhacks.net/hstack/*' - scp -r $(DOCS_BUILDDIR)/* root@hitomi:/srv/http/nixhacks.net/hstack/ + ssh root@hitomi 'rm -rf /srv/http/nixhacks.net/hstack/compute/*' + scp -r $(DOCS_BUILDDIR)/* root@hitomi:/srv/http/nixhacks.net/hstack/compute/ diff --git a/README.md b/README.md index 7d972af..e68ab66 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ Compute instances management library. ## Docs -Run `make serve-docs`. See [Development](#development) below. +Documantation is available [here](https://nixhacks.net/hstack/compute/master/index.html). +To build actual docs run `make serve-docs`. See [Development](#development) below. ## Roadmap @@ -41,6 +42,7 @@ Run `make serve-docs`. See [Development](#development) below. - [ ] LXC - [ ] Attaching CDROM from sources: block, (http|https|ftp|ftps|tftp):// - [ ] Instance clones (thin, fat) +- [ ] MicroVM ## Development @@ -52,7 +54,7 @@ Install [poetry](https://python-poetry.org/), clone this repository and run: poetry install --with dev --with docs ``` -# Build Debian package +## Build Debian package Install Docker first, then run: @@ -62,11 +64,11 @@ make build-deb `compute` and `compute-doc` packages will built. See packaging/build directory. -# Installation +## Installation See [Installation](https://nixhacks.net/hstack/compute/master/installation.html). -# Basic usage +## Basic usage To get help run: diff --git a/compute/__init__.py b/compute/__init__.py index e5395dc..e36e5ca 100644 --- a/compute/__init__.py +++ b/compute/__init__.py @@ -15,7 +15,7 @@ """Compute instances management library.""" -__version__ = '0.1.0-dev3' +__version__ = '0.1.0-dev4' from .config import Config from .instance import CloudInit, Instance, InstanceConfig, InstanceSchema diff --git a/compute/cli/commands.py b/compute/cli/commands.py index 7c8a3ef..6c277d1 100644 --- a/compute/cli/commands.py +++ b/compute/cli/commands.py @@ -55,7 +55,7 @@ def init(session: Session, args: argparse.Namespace) -> None: capabilities = session.get_capabilities() node_info = session.get_node_info() base_instance_config = { - 'name': str(uuid.uuid4()), + 'name': str(uuid.uuid4()).split('-')[0], 'title': None, 'description': None, 'arch': capabilities.arch, @@ -70,16 +70,28 @@ def init(session: Session, args: argparse.Namespace) -> None: 'topology': None, 'features': None, }, - 'network_interfaces': [ - { - 'source': 'default', - 'mac': ids.random_mac(), - }, - ], 'boot': {'order': ['cdrom', 'hd']}, 'cloud_init': None, } data = dictutil.override(base_instance_config, data) + net_default_interface = { + 'model': 'virtio', + 'source': 'default', + 'mac': ids.random_mac(), + } + net_config = data.get('network', 'DEFAULT') + if net_config == 'DEFAULT' or net_config is True: + data['network'] = {'interfaces': [net_default_interface]} + elif net_config is None or net_config is False: + pass # allow creating instance without network interfaces + else: + interfaces = data['network'].get('interfaces') + if interfaces: + interfaces_configs = [ + dictutil.override(net_default_interface, interface) + for interface in interfaces + ] + data['network']['interfaces'] = interfaces_configs volumes = [] targets = [] for volume in data['volumes']: @@ -246,8 +258,8 @@ def shutdown(session: Session, args: argparse.Namespace) -> None: method = 'SOFT' elif args.hard: method = 'HARD' - elif args.unsafe: - method = 'UNSAFE' + elif args.destroy: + method = 'DESTROY' else: method = 'NORMAL' instance.shutdown(method) diff --git a/compute/cli/parser.py b/compute/cli/parser.py index 4517e4b..ee935e5 100644 --- a/compute/cli/parser.py +++ b/compute/cli/parser.py @@ -226,7 +226,7 @@ def get_parser() -> argparse.ArgumentParser: '-s', '--soft', action='store_true', - help='normal guest OS shutdown, guest agent is used', + help='guest OS shutdown using guest agent', ) shutdown_opts.add_argument( '-n', @@ -244,12 +244,12 @@ def get_parser() -> argparse.ArgumentParser: ), ) shutdown_opts.add_argument( - '-u', - '--unsafe', + '-d', + '--destroy', action='store_true', help=( 'destroy instance, this is similar to a power outage ' - 'and may result in data loss or corruption' + 'and may result in data corruption' ), ) shutdown.set_defaults(func=commands.shutdown) diff --git a/compute/exceptions.py b/compute/exceptions.py index 2a41da5..c7f13f9 100644 --- a/compute/exceptions.py +++ b/compute/exceptions.py @@ -92,7 +92,7 @@ class InvalidDeviceConfigError(ComputeError): """Initialise InvalidDeviceConfigError.""" self.msg = f'Invalid device XML config: {msg}' self.loc = f' {xml}' - super().__init__(f'{self.msg}\n:{self.loc}') + super().__init__(f'{self.msg}:\n{self.loc}') class InvalidDataUnitError(ValueError, ComputeError): diff --git a/compute/instance/cloud_init.py b/compute/instance/cloud_init.py index a34e28a..2500bef 100644 --- a/compute/instance/cloud_init.py +++ b/compute/instance/cloud_init.py @@ -137,7 +137,7 @@ class CloudInit: subprocess.run( ['/usr/sbin/mkfs.vfat', '-n', 'CIDATA', '-C', str(disk), '1024'], check=True, - stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, ) self._write_to_disk( disk=disk, diff --git a/compute/instance/devices.py b/compute/instance/devices.py index 03673cc..52420ce 100644 --- a/compute/instance/devices.py +++ b/compute/instance/devices.py @@ -92,6 +92,8 @@ class DiskConfig(DeviceConfig): encoding='unicode', pretty_print=True, ).strip() + source = xml.find('source') + target = xml.find('target') driver = xml.find('driver') cachetype = driver.get('cache') disk_params = { @@ -102,14 +104,14 @@ class DiskConfig(DeviceConfig): type=driver.get('type'), **({'cache': cachetype} if cachetype else {}), ), - 'source': xml.find('source').get('file'), - 'target': xml.find('target').get('dev'), - 'bus': xml.find('target').get('bus'), + 'source': source.get('file') if source is not None else None, + 'target': target.get('dev') if target is not None else None, + 'bus': target.get('bus') if target is not None else None, 'is_readonly': False if xml.find('readonly') is None else True, } for param in disk_params: if disk_params[param] is None: - msg = f"missing XML tag '{param}'" + msg = f"missing tag '{param}'" raise InvalidDeviceConfigError(msg, xml_str) if param == 'driver': driver = disk_params[param] diff --git a/compute/instance/instance.py b/compute/instance/instance.py index 16fb960..a0f8bc9 100644 --- a/compute/instance/instance.py +++ b/compute/instance/instance.py @@ -20,6 +20,7 @@ __all__ = ['Instance', 'InstanceConfig', 'InstanceInfo'] import logging import time from typing import NamedTuple +from uuid import UUID import libvirt from lxml import etree @@ -66,7 +67,7 @@ class InstanceConfig(EntityConfig): self.emulator = schema.emulator self.arch = schema.arch self.boot = schema.boot - self.network_interfaces = schema.network_interfaces + self.network = schema.network def _gen_cpu_xml(self, cpu: CPUSchema) -> etree.Element: options = { @@ -119,6 +120,7 @@ class InstanceConfig(EntityConfig): return E.interface( E.source(network=interface.source), E.mac(address=interface.mac), + E.model(type=interface.model), type='network', ) @@ -165,8 +167,9 @@ class InstanceConfig(EntityConfig): ) devices = E.devices() devices.append(E.emulator(str(self.emulator))) - for interface in self.network_interfaces: - devices.append(self._gen_network_interface_xml(interface)) + if self.network: + for interface in self.network.interfaces: + devices.append(self._gen_network_interface_xml(interface)) devices.append(E.graphics(type='vnc', autoport='yes')) devices.append(E.input(type='tablet', bus='usb')) devices.append( @@ -212,18 +215,14 @@ class Instance: def __init__(self, domain: libvirt.virDomain): """ - Initialise Instance. - - :ivar libvirt.virDomain domain: domain object - :ivar libvirt.virConnect connection: connection object - :ivar str name: domain name - :ivar GuestAgent guest_agent: :class:`GuestAgent` object + Initialise Compute Instance object. :param domain: libvirt domain object """ self._domain = domain self._connection = domain.connect() self._name = domain.name() + self._uuid = domain.UUID() self._guest_agent = GuestAgent(domain) @property @@ -241,6 +240,11 @@ class Instance: """Instance name.""" return self._name + @property + def uuid(self) -> UUID: + """Instance UUID.""" + return UUID(bytes=self._uuid) + @property def guest_agent(self) -> GuestAgent: """:class:`GuestAgent` object.""" @@ -287,10 +291,9 @@ class Instance: def is_running(self) -> bool: """Return True if instance is running, else return False.""" - if self.domain.isActive() != 1: - # 0 - is inactive, -1 - is error - return False - return True + if self.domain.isActive() == 1: + return True + return False def is_autostart(self) -> bool: """Return True if instance autostart is enabled, else return False.""" @@ -318,7 +321,7 @@ class Instance: log.info("Starting instance '%s'", self.name) if self.is_running(): log.warning( - 'Already started, nothing to do instance=%s', self.name + "Instance '%s' is already started, nothing to do", self.name ) return try: @@ -347,8 +350,8 @@ class Instance: to unplugging machine from power. Internally send SIGTERM to instance process and destroy it gracefully. - UNSAFE - Force shutdown. Internally send SIGKILL to instance process. + DESTROY + Forced shutdown. Internally send SIGKILL to instance process. There is high data corruption risk! If method is None NORMAL method will used. @@ -361,7 +364,7 @@ class Instance: 'SOFT': libvirt.VIR_DOMAIN_SHUTDOWN_GUEST_AGENT, 'NORMAL': libvirt.VIR_DOMAIN_SHUTDOWN_DEFAULT, 'HARD': libvirt.VIR_DOMAIN_DESTROY_GRACEFUL, - 'UNSAFE': libvirt.VIR_DOMAIN_DESTROY_DEFAULT, + 'DESTROY': libvirt.VIR_DOMAIN_DESTROY_DEFAULT, } if method is None: method = 'NORMAL' @@ -372,11 +375,13 @@ class Instance: method = method.upper() if method not in methods: raise ValueError(f"Unsupported shutdown method: '{method}'") + if method == 'SOFT' and self.guest_agent.is_available() is False: + method = 'NORMAL' log.info("Performing instance shutdown with method '%s'", method) try: if method in ['SOFT', 'NORMAL']: self.domain.shutdownFlags(flags=methods[method]) - elif method in ['HARD', 'UNSAFE']: + elif method in ['HARD', 'DESTROY']: self.domain.destroyFlags(flags=methods[method]) except libvirt.libvirtError as e: raise InstanceError( @@ -443,8 +448,7 @@ class Instance: self.domain.setAutostart(autostart) except libvirt.libvirtError as e: raise InstanceError( - f'Cannot set autostart flag for instance={self.name} ' - f'{autostart=}: {e}' + f"Cannot set {autostart=} flag for instance '{self.name}': {e}" ) from e def set_vcpus(self, nvcpus: int, *, live: bool = False) -> None: @@ -466,7 +470,7 @@ class Instance: raise InstanceError('vCPUs count is greather than max_vcpus') if nvcpus == self.get_info().nproc: log.warning( - 'Instance instance=%s already have %s vCPUs, nothing to do', + "Instance '%s' already have %s vCPUs, nothing to do", self.name, nvcpus, ) @@ -492,18 +496,17 @@ class Instance: self.domain.setVcpusFlags(nvcpus, flags=flags) except GuestAgentCommandNotSupportedError: log.warning( - 'Cannot set vCPUs in guest via agent, you may ' - 'need to apply changes in guest manually.' + "'guest-set-vcpus' command is not supported, '" + 'you may need to enable CPUs in guest manually.' ) else: log.warning( - 'Cannot set vCPUs in guest OS on instance=%s. ' - 'You may need to apply CPUs in guest manually.', - self.name, + 'Guest agent is not installed or not connected, ' + 'you may need to enable CPUs in guest manually.' ) except libvirt.libvirtError as e: raise InstanceError( - f'Cannot set vCPUs for instance={self.name}: {e}' + f"Cannot set vCPUs for instance '{self.name}': {e}" ) from e def set_memory(self, memory: int, *, live: bool = False) -> None: diff --git a/compute/instance/schemas.py b/compute/instance/schemas.py index 1b61a6a..2326d5a 100644 --- a/compute/instance/schemas.py +++ b/compute/instance/schemas.py @@ -16,11 +16,11 @@ """Compute instance related objects schemas.""" import re +from collections import Counter from enum import StrEnum from pathlib import Path -from pydantic import ValidationError, validator -from pydantic.error_wrappers import ErrorWrapper +from pydantic import validator from compute.abstract import EntityModel from compute.utils.units import DataUnit @@ -74,12 +74,30 @@ class VolumeCapacitySchema(EntityModel): unit: DataUnit +class DiskCache(StrEnum): + """Possible disk cache mechanisms enumeration.""" + + NONE = 'none' + WRITETHROUGH = 'writethrough' + WRITEBACK = 'writeback' + DIRECTSYNC = 'directsync' + UNSAFE = 'unsafe' + + class DiskDriverSchema(EntityModel): """Virtual disk driver model.""" name: str type: str # noqa: A003 - cache: str = 'writethrough' + cache: DiskCache = DiskCache.WRITETHROUGH + + +class DiskBus(StrEnum): + """Possible disk buses enumeration.""" + + VIRTIO = 'virtio' + IDE = 'ide' + SATA = 'sata' class VolumeSchema(EntityModel): @@ -92,15 +110,30 @@ class VolumeSchema(EntityModel): source: str | None = None is_readonly: bool = False is_system: bool = False - bus: str = 'virtio' + bus: DiskBus = DiskBus.VIRTIO device: str = 'disk' +class NetworkAdapterModel(StrEnum): + """Network adapter models.""" + + VIRTIO = 'virtio' + E1000 = 'e1000' + RTL8139 = 'rtl8139' + + class NetworkInterfaceSchema(EntityModel): """Network inerface model.""" source: str mac: str + model: NetworkAdapterModel + + +class NetworkSchema(EntityModel): + """Network configuration schema.""" + + interfaces: list[NetworkInterfaceSchema] class BootOptionsSchema(EntityModel): @@ -134,7 +167,7 @@ class InstanceSchema(EntityModel): arch: str boot: BootOptionsSchema volumes: list[VolumeSchema] - network_interfaces: list[NetworkInterfaceSchema] + network: NetworkSchema | None | bool image: str | None = None cloud_init: CloudInitSchema | None = None @@ -142,7 +175,7 @@ class InstanceSchema(EntityModel): def _check_name(cls, value: str) -> str: # noqa: N805 if not re.match(r'^[a-z0-9_-]+$', value): msg = ( - 'Name can contain only lowercase letters, numbers, ' + 'Name must contain only lowercase letters, numbers, ' 'minus sign and underscore.' ) raise ValueError(msg) @@ -162,27 +195,33 @@ class InstanceSchema(EntityModel): if len([v for v in volumes if v.is_system is True]) != 1: msg = 'volumes list must contain one system volume' raise ValueError(msg) - for vol in volumes: - if vol.source is None and vol.capacity is None: - raise ValidationError( - [ - ErrorWrapper( - Exception( - "capacity is required if 'source' is unset" - ), - loc='X.capacity', - ) - ], - model=VolumeSchema, - ) - if vol.is_system is True and vol.is_readonly is True: + index = 0 + for volume in volumes: + index += 1 + if volume.source is None and volume.capacity is None: + msg = f"{index}: capacity is required if 'source' is unset" + raise ValueError(msg) + if volume.is_system is True and volume.is_readonly is True: msg = 'volume marked as system cannot be readonly' raise ValueError(msg) + sources = [v.source for v in volumes if v.source is not None] + targets = [v.target for v in volumes] + for item in [sources, targets]: + duplicates = Counter(item) - Counter(set(item)) + if duplicates: + msg = f'find duplicate values: {list(duplicates)}' + raise ValueError(msg) return volumes - @validator('network_interfaces') - def _check_network_interfaces(cls, value: list) -> list: # noqa: N805 - if not value: - msg = 'Network interfaces list must contain at least one element' + @validator('network') + def _check_network( + cls, # noqa: N805 + network: NetworkSchema | None | bool, + ) -> NetworkSchema | None | bool: + if network is True: + msg = ( + "'network' cannot be True, set it to False " + 'or provide network configuration' + ) raise ValueError(msg) - return value + return network diff --git a/compute/session.py b/compute/session.py index c872568..4978630 100644 --- a/compute/session.py +++ b/compute/session.py @@ -207,8 +207,8 @@ class Session(AbstractContextManager): :param volumes: List of storage volume configs. For more info see :class:`VolumeSchema`. :type volumes: list[dict] - :param network_interfaces: List of virtual network interfaces - configs. See :class:`NetworkInterfaceSchema` for more info. + :param network: List of virtual network interfaces configs. + See :class:`NetworkSchema` for more info. :type network_interfaces: list[dict] :param cloud_init: Cloud-init configuration. See :class:`CloudInitSchema` for info. diff --git a/compute/storage/pool.py b/compute/storage/pool.py index 222e1bf..cfc4409 100644 --- a/compute/storage/pool.py +++ b/compute/storage/pool.py @@ -73,9 +73,9 @@ class StoragePool: """ Refresh storage pool. - :param retry: If True retry pool refresh on :class:`libvirtError` - with running asynchronous jobs. - :param timeout: Retry timeout in secodns. Affets only if `retry` + :param retry: If True retry pool refresh on 'pool have running + asynchronous jobs' error. + :param timeout: Retry timeout in seconds. Affects only if `retry` is True. """ retry_timeout = dt.now(tz=datetime.UTC) + timedelta(seconds=timeout) diff --git a/compute/utils/units.py b/compute/utils/units.py index cec9a22..ea49f4e 100644 --- a/compute/utils/units.py +++ b/compute/utils/units.py @@ -15,6 +15,7 @@ """Tools for data units convertion.""" +from collections.abc import Callable from enum import StrEnum from compute.exceptions import InvalidDataUnitError @@ -28,6 +29,14 @@ class DataUnit(StrEnum): MIB = 'MiB' GIB = 'GiB' TIB = 'TiB' + KB = 'kb' + MB = 'Mb' + GB = 'Gb' + TB = 'Tb' + KBIT = 'kbit' + MBIT = 'Mbit' + GBIT = 'Gbit' + TBIT = 'Tbit' @classmethod def _missing_(cls, name: str) -> 'DataUnit': @@ -37,17 +46,74 @@ class DataUnit(StrEnum): return None -def to_bytes(value: int, unit: DataUnit = DataUnit.BYTES) -> int: - """Convert value to bytes. See :class:`DataUnit`.""" - try: - _ = DataUnit(unit) - except ValueError as e: - raise InvalidDataUnitError(e, list(DataUnit)) from e - powers = { +def validate_input(*args: str) -> Callable: + """Validate data units in functions input.""" + to_validate = args + + def decorator(func: Callable) -> Callable: + def wrapper(*args: float | str, **kwargs: str) -> Callable: + try: + if kwargs: + for arg in to_validate: + unit = kwargs[arg] + DataUnit(unit) + else: + for arg in args[1:]: + unit = arg + DataUnit(unit) + except ValueError as e: + raise InvalidDataUnitError(e, list(DataUnit)) from e + return func(*args, **kwargs) + + return wrapper + + return decorator + + +@validate_input('unit') +def to_bytes(value: float, unit: DataUnit = DataUnit.BYTES) -> float: + """Convert value to bytes.""" + unit = DataUnit(unit) + basis = 2 if unit.endswith('iB') else 10 + factor = 125 if unit.endswith('bit') else 1 + power = { DataUnit.BYTES: 0, - DataUnit.KIB: 1, - DataUnit.MIB: 2, - DataUnit.GIB: 3, - DataUnit.TIB: 4, + DataUnit.KIB: 10, + DataUnit.MIB: 20, + DataUnit.GIB: 30, + DataUnit.TIB: 40, + DataUnit.KB: 3, + DataUnit.MB: 6, + DataUnit.GB: 9, + DataUnit.TB: 12, + DataUnit.KBIT: 0, + DataUnit.MBIT: 3, + DataUnit.GBIT: 6, + DataUnit.TBIT: 9, } - return value * pow(1024, powers[unit]) + return value * factor * pow(basis, power[unit]) + + +@validate_input('from_unit', 'to_unit') +def convert(value: float, from_unit: DataUnit, to_unit: DataUnit) -> float: + """Convert units.""" + value_in_bits = to_bytes(value, from_unit) * 8 + to_unit = DataUnit(to_unit) + basis = 2 if to_unit.endswith('iB') else 10 + divisor = 1 if to_unit.endswith('bit') else 8 + power = { + DataUnit.BYTES: 0, + DataUnit.KIB: 10, + DataUnit.MIB: 20, + DataUnit.GIB: 30, + DataUnit.TIB: 40, + DataUnit.KB: 3, + DataUnit.MB: 6, + DataUnit.GB: 9, + DataUnit.TB: 12, + DataUnit.KBIT: 3, + DataUnit.MBIT: 6, + DataUnit.GBIT: 9, + DataUnit.TBIT: 12, + } + return value_in_bits / divisor / pow(basis, power[to_unit]) diff --git a/docs/source/conf.py b/docs/source/conf.py index 6505dd5..4bce759 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -6,7 +6,7 @@ sys.path.insert(0, os.path.abspath('../..')) project = 'Compute' copyright = '2023, Compute Authors' author = 'Compute Authors' -release = '0.1.0-dev3' +release = '0.1.0-dev4' # Sphinx general settings extensions = [ diff --git a/packaging/files/control b/packaging/files/control index 8f0cd99..5da48cb 100644 --- a/packaging/files/control +++ b/packaging/files/control @@ -28,16 +28,18 @@ Depends: ${misc:Depends}, qemu-system, qemu-utils, + libvirt-daemon, libvirt-daemon-system, + libvirt-daemon-driver-qemu, libvirt-clients, python3-libvirt, python3-lxml, python3-yaml, python3-pydantic, mtools, - dosfstools -Recommends: - dnsmasq + dosfstools, + dnsmasq, + dnsmasq-base Suggests: compute-doc Description: Compute instances management library (Python 3) diff --git a/pyproject.toml b/pyproject.toml index 90d8008..a70aed5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = 'compute' -version = '0.1.0-dev3' +version = '0.1.0-dev4' description = 'Compute instances management library' license = 'GPL-3.0-or-later' authors = ['ge ']