v0.1.0-dev4

Makefile

@@ -48,5 +48,5 @@ test-build: build-deb
 	scp packaging/build/compute*.deb vm:~
 
 upload-docs: docs-versions
-	ssh root@hitomi 'rm -rf /srv/http/nixhacks.net/hstack/*'
+	ssh root@hitomi 'rm -rf /srv/http/nixhacks.net/hstack/compute/*'
-	scp -r $(DOCS_BUILDDIR)/* root@hitomi:/srv/http/nixhacks.net/hstack/
+	scp -r $(DOCS_BUILDDIR)/* root@hitomi:/srv/http/nixhacks.net/hstack/compute/

README.md

@@ -4,7 +4,8 @@ Compute instances management library.
 
 ## Docs
 
-Run `make serve-docs`. See [Development](#development) below.
+Documantation is available [here](https://nixhacks.net/hstack/compute/master/index.html).
+To build actual docs run `make serve-docs`. See [Development](#development) below.
 
 ## Roadmap
 
@@ -41,6 +42,7 @@ Run `make serve-docs`. See [Development](#development) below.
 - [ ] LXC
 - [ ] Attaching CDROM from sources: block, (http|https|ftp|ftps|tftp)://
 - [ ] Instance clones (thin, fat)
+- [ ] MicroVM
 
 ## Development
 
@@ -52,7 +54,7 @@ Install [poetry](https://python-poetry.org/), clone this repository and run:
 poetry install --with dev --with docs
 ```
 
-# Build Debian package
+## Build Debian package
 
 Install Docker first, then run:
 
@@ -62,11 +64,11 @@ make build-deb
 
 `compute` and `compute-doc` packages will built. See packaging/build directory.
 
-# Installation
+## Installation
 
 See [Installation](https://nixhacks.net/hstack/compute/master/installation.html).
 
-# Basic usage
+## Basic usage
 
 To get help run:
 

compute/__init__.py

@@ -15,7 +15,7 @@
 
 """Compute instances management library."""
 
-__version__ = '0.1.0-dev3'
+__version__ = '0.1.0-dev4'
 
 from .config import Config
 from .instance import CloudInit, Instance, InstanceConfig, InstanceSchema

compute/cli/commands.py

@@ -55,7 +55,7 @@ def init(session: Session, args: argparse.Namespace) -> None:
     capabilities = session.get_capabilities()
     node_info = session.get_node_info()
     base_instance_config = {
-        'name': str(uuid.uuid4()),
+        'name': str(uuid.uuid4()).split('-')[0],
         'title': None,
         'description': None,
         'arch': capabilities.arch,
@@ -70,16 +70,28 @@ def init(session: Session, args: argparse.Namespace) -> None:
             'topology': None,
             'features': None,
         },
-        'network_interfaces': [
-            {
-                'source': 'default',
-                'mac': ids.random_mac(),
-            },
-        ],
         'boot': {'order': ['cdrom', 'hd']},
         'cloud_init': None,
     }
     data = dictutil.override(base_instance_config, data)
+    net_default_interface = {
+        'model': 'virtio',
+        'source': 'default',
+        'mac': ids.random_mac(),
+    }
+    net_config = data.get('network', 'DEFAULT')
+    if net_config == 'DEFAULT' or net_config is True:
+        data['network'] = {'interfaces': [net_default_interface]}
+    elif net_config is None or net_config is False:
+        pass  # allow creating instance without network interfaces
+    else:
+        interfaces = data['network'].get('interfaces')
+        if interfaces:
+            interfaces_configs = [
+                dictutil.override(net_default_interface, interface)
+                for interface in interfaces
+            ]
+            data['network']['interfaces'] = interfaces_configs
     volumes = []
     targets = []
     for volume in data['volumes']:
@@ -246,8 +258,8 @@ def shutdown(session: Session, args: argparse.Namespace) -> None:
         method = 'SOFT'
     elif args.hard:
         method = 'HARD'
-    elif args.unsafe:
+    elif args.destroy:
-        method = 'UNSAFE'
+        method = 'DESTROY'
     else:
         method = 'NORMAL'
     instance.shutdown(method)

compute/cli/parser.py

@@ -226,7 +226,7 @@ def get_parser() -> argparse.ArgumentParser:
         '-s',
         '--soft',
         action='store_true',
-        help='normal guest OS shutdown, guest agent is used',
+        help='guest OS shutdown using guest agent',
     )
     shutdown_opts.add_argument(
         '-n',
@@ -244,12 +244,12 @@ def get_parser() -> argparse.ArgumentParser:
         ),
     )
     shutdown_opts.add_argument(
-        '-u',
+        '-d',
-        '--unsafe',
+        '--destroy',
         action='store_true',
         help=(
             'destroy instance, this is similar to a power outage '
-            'and may result in data loss or corruption'
+            'and may result in data corruption'
         ),
     )
     shutdown.set_defaults(func=commands.shutdown)

compute/exceptions.py

@@ -92,7 +92,7 @@ class InvalidDeviceConfigError(ComputeError):
         """Initialise InvalidDeviceConfigError."""
         self.msg = f'Invalid device XML config: {msg}'
         self.loc = f'    {xml}'
-        super().__init__(f'{self.msg}\n:{self.loc}')
+        super().__init__(f'{self.msg}:\n{self.loc}')
 
 
 class InvalidDataUnitError(ValueError, ComputeError):

compute/instance/cloud_init.py

@@ -137,7 +137,7 @@ class CloudInit:
         subprocess.run(
             ['/usr/sbin/mkfs.vfat', '-n', 'CIDATA', '-C', str(disk), '1024'],
             check=True,
-            stderr=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL,
         )
         self._write_to_disk(
             disk=disk,

compute/instance/devices.py

@@ -92,6 +92,8 @@ class DiskConfig(DeviceConfig):
                 encoding='unicode',
                 pretty_print=True,
             ).strip()
+        source = xml.find('source')
+        target = xml.find('target')
         driver = xml.find('driver')
         cachetype = driver.get('cache')
         disk_params = {
@@ -102,14 +104,14 @@ class DiskConfig(DeviceConfig):
                 type=driver.get('type'),
                 **({'cache': cachetype} if cachetype else {}),
             ),
-            'source': xml.find('source').get('file'),
+            'source': source.get('file') if source is not None else None,
-            'target': xml.find('target').get('dev'),
+            'target': target.get('dev') if target is not None else None,
-            'bus': xml.find('target').get('bus'),
+            'bus': target.get('bus') if target is not None else None,
             'is_readonly': False if xml.find('readonly') is None else True,
         }
         for param in disk_params:
             if disk_params[param] is None:
-                msg = f"missing XML tag '{param}'"
+                msg = f"missing tag '{param}'"
                 raise InvalidDeviceConfigError(msg, xml_str)
             if param == 'driver':
                 driver = disk_params[param]

compute/instance/instance.py

@@ -20,6 +20,7 @@ __all__ = ['Instance', 'InstanceConfig', 'InstanceInfo']
 import logging
 import time
 from typing import NamedTuple
+from uuid import UUID
 
 import libvirt
 from lxml import etree
@@ -66,7 +67,7 @@ class InstanceConfig(EntityConfig):
         self.emulator = schema.emulator
         self.arch = schema.arch
         self.boot = schema.boot
-        self.network_interfaces = schema.network_interfaces
+        self.network = schema.network
 
     def _gen_cpu_xml(self, cpu: CPUSchema) -> etree.Element:
         options = {
@@ -119,6 +120,7 @@ class InstanceConfig(EntityConfig):
         return E.interface(
             E.source(network=interface.source),
             E.mac(address=interface.mac),
+            E.model(type=interface.model),
             type='network',
         )
 
@@ -165,7 +167,8 @@ class InstanceConfig(EntityConfig):
         )
         devices = E.devices()
         devices.append(E.emulator(str(self.emulator)))
-        for interface in self.network_interfaces:
+        if self.network:
+            for interface in self.network.interfaces:
                 devices.append(self._gen_network_interface_xml(interface))
         devices.append(E.graphics(type='vnc', autoport='yes'))
         devices.append(E.input(type='tablet', bus='usb'))
@@ -212,18 +215,14 @@ class Instance:
 
     def __init__(self, domain: libvirt.virDomain):
         """
-        Initialise Instance.
+        Initialise Compute Instance object.
-
-        :ivar libvirt.virDomain domain: domain object
-        :ivar libvirt.virConnect connection: connection object
-        :ivar str name: domain name
-        :ivar GuestAgent guest_agent: :class:`GuestAgent` object
 
         :param domain: libvirt domain object
         """
         self._domain = domain
         self._connection = domain.connect()
         self._name = domain.name()
+        self._uuid = domain.UUID()
         self._guest_agent = GuestAgent(domain)
 
     @property
@@ -241,6 +240,11 @@ class Instance:
         """Instance name."""
         return self._name
 
+    @property
+    def uuid(self) -> UUID:
+        """Instance UUID."""
+        return UUID(bytes=self._uuid)
+
     @property
     def guest_agent(self) -> GuestAgent:
         """:class:`GuestAgent` object."""
@@ -287,10 +291,9 @@ class Instance:
 
     def is_running(self) -> bool:
         """Return True if instance is running, else return False."""
-        if self.domain.isActive() != 1:
+        if self.domain.isActive() == 1:
-            # 0 - is inactive, -1 - is error
-            return False
             return True
+        return False
 
     def is_autostart(self) -> bool:
         """Return True if instance autostart is enabled, else return False."""
@@ -318,7 +321,7 @@ class Instance:
         log.info("Starting instance '%s'", self.name)
         if self.is_running():
             log.warning(
-                'Already started, nothing to do instance=%s', self.name
+                "Instance '%s' is already started, nothing to do", self.name
             )
             return
         try:
@@ -347,8 +350,8 @@ class Instance:
             to unplugging machine from power. Internally send SIGTERM to
             instance process and destroy it gracefully.
 
-        UNSAFE
+        DESTROY
-            Force shutdown. Internally send SIGKILL to instance process.
+            Forced shutdown. Internally send SIGKILL to instance process.
             There is high data corruption risk!
 
         If method is None NORMAL method will used.
@@ -361,7 +364,7 @@ class Instance:
             'SOFT': libvirt.VIR_DOMAIN_SHUTDOWN_GUEST_AGENT,
             'NORMAL': libvirt.VIR_DOMAIN_SHUTDOWN_DEFAULT,
             'HARD': libvirt.VIR_DOMAIN_DESTROY_GRACEFUL,
-            'UNSAFE': libvirt.VIR_DOMAIN_DESTROY_DEFAULT,
+            'DESTROY': libvirt.VIR_DOMAIN_DESTROY_DEFAULT,
         }
         if method is None:
             method = 'NORMAL'
@@ -372,11 +375,13 @@ class Instance:
         method = method.upper()
         if method not in methods:
             raise ValueError(f"Unsupported shutdown method: '{method}'")
+        if method == 'SOFT' and self.guest_agent.is_available() is False:
+            method = 'NORMAL'
         log.info("Performing instance shutdown with method '%s'", method)
         try:
             if method in ['SOFT', 'NORMAL']:
                 self.domain.shutdownFlags(flags=methods[method])
-            elif method in ['HARD', 'UNSAFE']:
+            elif method in ['HARD', 'DESTROY']:
                 self.domain.destroyFlags(flags=methods[method])
         except libvirt.libvirtError as e:
             raise InstanceError(
@@ -443,8 +448,7 @@ class Instance:
             self.domain.setAutostart(autostart)
         except libvirt.libvirtError as e:
             raise InstanceError(
-                f'Cannot set autostart flag for instance={self.name} '
+                f"Cannot set {autostart=} flag for instance '{self.name}': {e}"
-                f'{autostart=}: {e}'
             ) from e
 
     def set_vcpus(self, nvcpus: int, *, live: bool = False) -> None:
@@ -466,7 +470,7 @@ class Instance:
             raise InstanceError('vCPUs count is greather than max_vcpus')
         if nvcpus == self.get_info().nproc:
             log.warning(
-                'Instance instance=%s already have %s vCPUs, nothing to do',
+                "Instance '%s' already have %s vCPUs, nothing to do",
                 self.name,
                 nvcpus,
             )
@@ -492,18 +496,17 @@ class Instance:
                         self.domain.setVcpusFlags(nvcpus, flags=flags)
                     except GuestAgentCommandNotSupportedError:
                         log.warning(
-                            'Cannot set vCPUs in guest via agent, you may '
+                            "'guest-set-vcpus' command is not supported, '"
-                            'need to apply changes in guest manually.'
+                            'you may need to enable CPUs in guest manually.'
                         )
                 else:
                     log.warning(
-                        'Cannot set vCPUs in guest OS on instance=%s. '
+                        'Guest agent is not installed or not connected, '
-                        'You may need to apply CPUs in guest manually.',
+                        'you may need to enable CPUs in guest manually.'
-                        self.name,
                     )
         except libvirt.libvirtError as e:
             raise InstanceError(
-                f'Cannot set vCPUs for instance={self.name}: {e}'
+                f"Cannot set vCPUs for instance '{self.name}': {e}"
             ) from e
 
     def set_memory(self, memory: int, *, live: bool = False) -> None:

compute/instance/schemas.py

@@ -16,11 +16,11 @@
 """Compute instance related objects schemas."""
 
 import re
+from collections import Counter
 from enum import StrEnum
 from pathlib import Path
 
-from pydantic import ValidationError, validator
+from pydantic import validator
-from pydantic.error_wrappers import ErrorWrapper
 
 from compute.abstract import EntityModel
 from compute.utils.units import DataUnit
@@ -74,12 +74,30 @@ class VolumeCapacitySchema(EntityModel):
     unit: DataUnit
 
 
+class DiskCache(StrEnum):
+    """Possible disk cache mechanisms enumeration."""
+
+    NONE = 'none'
+    WRITETHROUGH = 'writethrough'
+    WRITEBACK = 'writeback'
+    DIRECTSYNC = 'directsync'
+    UNSAFE = 'unsafe'
+
+
 class DiskDriverSchema(EntityModel):
     """Virtual disk driver model."""
 
     name: str
     type: str  # noqa: A003
-    cache: str = 'writethrough'
+    cache: DiskCache = DiskCache.WRITETHROUGH
+
+
+class DiskBus(StrEnum):
+    """Possible disk buses enumeration."""
+
+    VIRTIO = 'virtio'
+    IDE = 'ide'
+    SATA = 'sata'
 
 
 class VolumeSchema(EntityModel):
@@ -92,15 +110,30 @@ class VolumeSchema(EntityModel):
     source: str | None = None
     is_readonly: bool = False
     is_system: bool = False
-    bus: str = 'virtio'
+    bus: DiskBus = DiskBus.VIRTIO
     device: str = 'disk'
 
 
+class NetworkAdapterModel(StrEnum):
+    """Network adapter models."""
+
+    VIRTIO = 'virtio'
+    E1000 = 'e1000'
+    RTL8139 = 'rtl8139'
+
+
 class NetworkInterfaceSchema(EntityModel):
     """Network inerface model."""
 
     source: str
     mac: str
+    model: NetworkAdapterModel
+
+
+class NetworkSchema(EntityModel):
+    """Network configuration schema."""
+
+    interfaces: list[NetworkInterfaceSchema]
 
 
 class BootOptionsSchema(EntityModel):
@@ -134,7 +167,7 @@ class InstanceSchema(EntityModel):
     arch: str
     boot: BootOptionsSchema
     volumes: list[VolumeSchema]
-    network_interfaces: list[NetworkInterfaceSchema]
+    network: NetworkSchema | None | bool
     image: str | None = None
     cloud_init: CloudInitSchema | None = None
 
@@ -142,7 +175,7 @@ class InstanceSchema(EntityModel):
     def _check_name(cls, value: str) -> str:  # noqa: N805
         if not re.match(r'^[a-z0-9_-]+$', value):
             msg = (
-                'Name can contain only lowercase letters, numbers, '
+                'Name must contain only lowercase letters, numbers, '
                 'minus sign and underscore.'
             )
             raise ValueError(msg)
@@ -162,27 +195,33 @@ class InstanceSchema(EntityModel):
         if len([v for v in volumes if v.is_system is True]) != 1:
             msg = 'volumes list must contain one system volume'
             raise ValueError(msg)
-        for vol in volumes:
+        index = 0
-            if vol.source is None and vol.capacity is None:
+        for volume in volumes:
-                raise ValidationError(
+            index += 1
-                    [
+            if volume.source is None and volume.capacity is None:
-                        ErrorWrapper(
+                msg = f"{index}: capacity is required if 'source' is unset"
-                            Exception(
+                raise ValueError(msg)
-                                "capacity is required if 'source' is unset"
+            if volume.is_system is True and volume.is_readonly is True:
-                            ),
-                            loc='X.capacity',
-                        )
-                    ],
-                    model=VolumeSchema,
-                )
-            if vol.is_system is True and vol.is_readonly is True:
                 msg = 'volume marked as system cannot be readonly'
                 raise ValueError(msg)
+        sources = [v.source for v in volumes if v.source is not None]
+        targets = [v.target for v in volumes]
+        for item in [sources, targets]:
+            duplicates = Counter(item) - Counter(set(item))
+            if duplicates:
+                msg = f'find duplicate values: {list(duplicates)}'
+                raise ValueError(msg)
         return volumes
 
-    @validator('network_interfaces')
+    @validator('network')
-    def _check_network_interfaces(cls, value: list) -> list:  # noqa: N805
+    def _check_network(
-        if not value:
+        cls,  # noqa: N805
-            msg = 'Network interfaces list must contain at least one element'
+        network: NetworkSchema | None | bool,
+    ) -> NetworkSchema | None | bool:
+        if network is True:
+            msg = (
+                "'network' cannot be True, set it to False "
+                'or provide network configuration'
+            )
             raise ValueError(msg)
-        return value
+        return network

compute/session.py

@@ -207,8 +207,8 @@ class Session(AbstractContextManager):
         :param volumes: List of storage volume configs. For more info
             see :class:`VolumeSchema`.
         :type volumes: list[dict]
-        :param network_interfaces: List of virtual network interfaces
+        :param network: List of virtual network interfaces configs.
-            configs. See :class:`NetworkInterfaceSchema` for more info.
+            See :class:`NetworkSchema` for more info.
         :type network_interfaces: list[dict]
         :param cloud_init: Cloud-init configuration. See
             :class:`CloudInitSchema` for info.

compute/storage/pool.py

@@ -73,9 +73,9 @@ class StoragePool:
         """
         Refresh storage pool.
 
-        :param retry: If True retry pool refresh on :class:`libvirtError`
+        :param retry: If True retry pool refresh on 'pool have running
-            with running asynchronous jobs.
+            asynchronous jobs' error.
-        :param timeout: Retry timeout in secodns. Affets only if `retry`
+        :param timeout: Retry timeout in seconds. Affects only if `retry`
             is True.
         """
         retry_timeout = dt.now(tz=datetime.UTC) + timedelta(seconds=timeout)

compute/utils/units.py

@@ -15,6 +15,7 @@
 
 """Tools for data units convertion."""
 
+from collections.abc import Callable
 from enum import StrEnum
 
 from compute.exceptions import InvalidDataUnitError
@@ -28,6 +29,14 @@ class DataUnit(StrEnum):
     MIB = 'MiB'
     GIB = 'GiB'
     TIB = 'TiB'
+    KB = 'kb'
+    MB = 'Mb'
+    GB = 'Gb'
+    TB = 'Tb'
+    KBIT = 'kbit'
+    MBIT = 'Mbit'
+    GBIT = 'Gbit'
+    TBIT = 'Tbit'
 
     @classmethod
     def _missing_(cls, name: str) -> 'DataUnit':
@@ -37,17 +46,74 @@ class DataUnit(StrEnum):
         return None
 
 
-def to_bytes(value: int, unit: DataUnit = DataUnit.BYTES) -> int:
+def validate_input(*args: str) -> Callable:
-    """Convert value to bytes. See :class:`DataUnit`."""
+    """Validate data units in functions input."""
+    to_validate = args
+
+    def decorator(func: Callable) -> Callable:
+        def wrapper(*args: float | str, **kwargs: str) -> Callable:
             try:
-        _ = DataUnit(unit)
+                if kwargs:
+                    for arg in to_validate:
+                        unit = kwargs[arg]
+                        DataUnit(unit)
+                else:
+                    for arg in args[1:]:
+                        unit = arg
+                        DataUnit(unit)
             except ValueError as e:
                 raise InvalidDataUnitError(e, list(DataUnit)) from e
-    powers = {
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+@validate_input('unit')
+def to_bytes(value: float, unit: DataUnit = DataUnit.BYTES) -> float:
+    """Convert value to bytes."""
+    unit = DataUnit(unit)
+    basis = 2 if unit.endswith('iB') else 10
+    factor = 125 if unit.endswith('bit') else 1
+    power = {
         DataUnit.BYTES: 0,
-        DataUnit.KIB: 1,
+        DataUnit.KIB: 10,
-        DataUnit.MIB: 2,
+        DataUnit.MIB: 20,
-        DataUnit.GIB: 3,
+        DataUnit.GIB: 30,
-        DataUnit.TIB: 4,
+        DataUnit.TIB: 40,
+        DataUnit.KB: 3,
+        DataUnit.MB: 6,
+        DataUnit.GB: 9,
+        DataUnit.TB: 12,
+        DataUnit.KBIT: 0,
+        DataUnit.MBIT: 3,
+        DataUnit.GBIT: 6,
+        DataUnit.TBIT: 9,
     }
-    return value * pow(1024, powers[unit])
+    return value * factor * pow(basis, power[unit])
+
+
+@validate_input('from_unit', 'to_unit')
+def convert(value: float, from_unit: DataUnit, to_unit: DataUnit) -> float:
+    """Convert units."""
+    value_in_bits = to_bytes(value, from_unit) * 8
+    to_unit = DataUnit(to_unit)
+    basis = 2 if to_unit.endswith('iB') else 10
+    divisor = 1 if to_unit.endswith('bit') else 8
+    power = {
+        DataUnit.BYTES: 0,
+        DataUnit.KIB: 10,
+        DataUnit.MIB: 20,
+        DataUnit.GIB: 30,
+        DataUnit.TIB: 40,
+        DataUnit.KB: 3,
+        DataUnit.MB: 6,
+        DataUnit.GB: 9,
+        DataUnit.TB: 12,
+        DataUnit.KBIT: 3,
+        DataUnit.MBIT: 6,
+        DataUnit.GBIT: 9,
+        DataUnit.TBIT: 12,
+    }
+    return value_in_bits / divisor / pow(basis, power[to_unit])

docs/source/conf.py

@@ -6,7 +6,7 @@ sys.path.insert(0, os.path.abspath('../..'))
 project = 'Compute'
 copyright = '2023, Compute Authors'
 author = 'Compute Authors'
-release = '0.1.0-dev3'
+release = '0.1.0-dev4'
 
 # Sphinx general settings
 extensions = [

packaging/files/control

@@ -28,16 +28,18 @@ Depends:
  ${misc:Depends},
  qemu-system,
  qemu-utils,
+ libvirt-daemon,
  libvirt-daemon-system,
+ libvirt-daemon-driver-qemu,
  libvirt-clients,
  python3-libvirt,
  python3-lxml,
  python3-yaml,
  python3-pydantic,
  mtools,
- dosfstools
+ dosfstools,
-Recommends:
+ dnsmasq,
- dnsmasq
+ dnsmasq-base
 Suggests:
  compute-doc
 Description: Compute instances management library (Python 3)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = 'compute'
-version = '0.1.0-dev3'
+version = '0.1.0-dev4'
 description = 'Compute instances management library'
 license = 'GPL-3.0-or-later'
 authors = ['ge <ge@nixhacks.net>']