ge/python-compute
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

v0.1.0-dev4

Browse Source
This commit is contained in:
ge 2024-01-13 00:45:30 +03:00
parent bdff33759c
commit d2515cace8
16 changed files with 226 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -48,5 +48,5 @@ test-build: build-deb
scp packaging/build/compute*.deb vm:~
upload-docs: docs-versions
ssh root@hitomi 'rm -rf /srv/http/nixhacks.net/hstack/*'
scp -r $(DOCS_BUILDDIR)/* root@hitomi:/srv/http/nixhacks.net/hstack/
ssh root@hitomi 'rm -rf /srv/http/nixhacks.net/hstack/compute/*'
scp -r $(DOCS_BUILDDIR)/* root@hitomi:/srv/http/nixhacks.net/hstack/compute/

10
README.md
View File

@ -4,7 +4,8 @@ Compute instances management library.
## Docs
Run `make serve-docs`. See [Development](#development) below.
Documantation is available [here](https://nixhacks.net/hstack/compute/master/index.html).
To build actual docs run `make serve-docs`. See [Development](#development) below.
## Roadmap
@ -41,6 +42,7 @@ Run `make serve-docs`. See [Development](#development) below.
- [ ] LXC
- [ ] Attaching CDROM from sources: block, (http|https|ftp|ftps|tftp)://
- [ ] Instance clones (thin, fat)
- [ ] MicroVM
## Development
@ -52,7 +54,7 @@ Install [poetry](https://python-poetry.org/), clone this repository and run:
poetry install --with dev --with docs
```
# Build Debian package
## Build Debian package
Install Docker first, then run:
@ -62,11 +64,11 @@ make build-deb
`compute` and `compute-doc` packages will built. See packaging/build directory.
# Installation
## Installation
See [Installation](https://nixhacks.net/hstack/compute/master/installation.html).
# Basic usage
## Basic usage
To get help run:

2
compute/__init__.py
View File

@ -15,7 +15,7 @@
"""Compute instances management library."""
__version__ = '0.1.0-dev3'
__version__ = '0.1.0-dev4'
from .config import Config
from .instance import CloudInit, Instance, InstanceConfig, InstanceSchema

30
compute/cli/commands.py
View File

@ -55,7 +55,7 @@ def init(session: Session, args: argparse.Namespace) -> None:
capabilities = session.get_capabilities()
node_info = session.get_node_info()
base_instance_config = {
'name': str(uuid.uuid4()),
'name': str(uuid.uuid4()).split('-')[0],
'title': None,
'description': None,
'arch': capabilities.arch,
@ -70,16 +70,28 @@ def init(session: Session, args: argparse.Namespace) -> None:
'topology': None,
'features': None,
},
'network_interfaces': [
{
'source': 'default',
'mac': ids.random_mac(),
},
],
'boot': {'order': ['cdrom', 'hd']},
'cloud_init': None,
}
data = dictutil.override(base_instance_config, data)
net_default_interface = {
'model': 'virtio',
'source': 'default',
'mac': ids.random_mac(),
}
net_config = data.get('network', 'DEFAULT')
if net_config == 'DEFAULT' or net_config is True:
data['network'] = {'interfaces': [net_default_interface]}
elif net_config is None or net_config is False:
pass # allow creating instance without network interfaces
else:
interfaces = data['network'].get('interfaces')
if interfaces:
interfaces_configs = [
dictutil.override(net_default_interface, interface)
for interface in interfaces
]
data['network']['interfaces'] = interfaces_configs
volumes = []
targets = []
for volume in data['volumes']:
@ -246,8 +258,8 @@ def shutdown(session: Session, args: argparse.Namespace) -> None:
method = 'SOFT'
elif args.hard:
method = 'HARD'
elif args.unsafe:
method = 'UNSAFE'
elif args.destroy:
method = 'DESTROY'
else:
method = 'NORMAL'
instance.shutdown(method)

8
compute/cli/parser.py
View File

@ -226,7 +226,7 @@ def get_parser() -> argparse.ArgumentParser:
'-s',
'--soft',
action='store_true',
help='normal guest OS shutdown, guest agent is used',
help='guest OS shutdown using guest agent',
)
shutdown_opts.add_argument(
'-n',
@ -244,12 +244,12 @@ def get_parser() -> argparse.ArgumentParser:
),
)
shutdown_opts.add_argument(
'-u',
'--unsafe',
'-d',
'--destroy',
action='store_true',
help=(
'destroy instance, this is similar to a power outage '
'and may result in data loss or corruption'
'and may result in data corruption'
),
)
shutdown.set_defaults(func=commands.shutdown)

2
compute/exceptions.py
View File

@ -92,7 +92,7 @@ class InvalidDeviceConfigError(ComputeError):
"""Initialise InvalidDeviceConfigError."""
self.msg = f'Invalid device XML config: {msg}'
self.loc = f' {xml}'
super().__init__(f'{self.msg}\n:{self.loc}')
super().__init__(f'{self.msg}:\n{self.loc}')
class InvalidDataUnitError(ValueError, ComputeError):

2
compute/instance/cloud_init.py
View File

@ -137,7 +137,7 @@ class CloudInit:
subprocess.run(
['/usr/sbin/mkfs.vfat', '-n', 'CIDATA', '-C', str(disk), '1024'],
check=True,
stderr=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
)
self._write_to_disk(
disk=disk,

10
compute/instance/devices.py
View File

@ -92,6 +92,8 @@ class DiskConfig(DeviceConfig):
encoding='unicode',
pretty_print=True,
).strip()
source = xml.find('source')
target = xml.find('target')
driver = xml.find('driver')
cachetype = driver.get('cache')
disk_params = {
@ -102,14 +104,14 @@ class DiskConfig(DeviceConfig):
type=driver.get('type'),
**({'cache': cachetype} if cachetype else {}),
),
'source': xml.find('source').get('file'),
'target': xml.find('target').get('dev'),
'bus': xml.find('target').get('bus'),
'source': source.get('file') if source is not None else None,
'target': target.get('dev') if target is not None else None,
'bus': target.get('bus') if target is not None else None,
'is_readonly': False if xml.find('readonly') is None else True,
}
for param in disk_params:
if disk_params[param] is None:
msg = f"missing XML tag '{param}'"
msg = f"missing tag '{param}'"
raise InvalidDeviceConfigError(msg, xml_str)
if param == 'driver':
driver = disk_params[param]

53
compute/instance/instance.py
View File

@ -20,6 +20,7 @@ __all__ = ['Instance', 'InstanceConfig', 'InstanceInfo']
import logging
import time
from typing import NamedTuple
from uuid import UUID
import libvirt
from lxml import etree
@ -66,7 +67,7 @@ class InstanceConfig(EntityConfig):
self.emulator = schema.emulator
self.arch = schema.arch
self.boot = schema.boot
self.network_interfaces = schema.network_interfaces
self.network = schema.network
def _gen_cpu_xml(self, cpu: CPUSchema) -> etree.Element:
options = {
@ -119,6 +120,7 @@ class InstanceConfig(EntityConfig):
return E.interface(
E.source(network=interface.source),
E.mac(address=interface.mac),
E.model(type=interface.model),
type='network',
)
@ -165,7 +167,8 @@ class InstanceConfig(EntityConfig):
)
devices = E.devices()
devices.append(E.emulator(str(self.emulator)))
for interface in self.network_interfaces:
if self.network:
for interface in self.network.interfaces:
devices.append(self._gen_network_interface_xml(interface))
devices.append(E.graphics(type='vnc', autoport='yes'))
devices.append(E.input(type='tablet', bus='usb'))
@ -212,18 +215,14 @@ class Instance:
def __init__(self, domain: libvirt.virDomain):
"""
Initialise Instance.
:ivar libvirt.virDomain domain: domain object
:ivar libvirt.virConnect connection: connection object
:ivar str name: domain name
:ivar GuestAgent guest_agent: :class:`GuestAgent` object
Initialise Compute Instance object.
:param domain: libvirt domain object
"""
self._domain = domain
self._connection = domain.connect()
self._name = domain.name()
self._uuid = domain.UUID()
self._guest_agent = GuestAgent(domain)
@property
@ -241,6 +240,11 @@ class Instance:
"""Instance name."""
return self._name
@property
def uuid(self) -> UUID:
"""Instance UUID."""
return UUID(bytes=self._uuid)
@property
def guest_agent(self) -> GuestAgent:
""":class:`GuestAgent` object."""
@ -287,10 +291,9 @@ class Instance:
def is_running(self) -> bool:
"""Return True if instance is running, else return False."""
if self.domain.isActive() != 1:
# 0 - is inactive, -1 - is error
return False
if self.domain.isActive() == 1:
return True
return False
def is_autostart(self) -> bool:
"""Return True if instance autostart is enabled, else return False."""
@ -318,7 +321,7 @@ class Instance:
log.info("Starting instance '%s'", self.name)
if self.is_running():
log.warning(
'Already started, nothing to do instance=%s', self.name
"Instance '%s' is already started, nothing to do", self.name
)
return
try:
@ -347,8 +350,8 @@ class Instance:
to unplugging machine from power. Internally send SIGTERM to
instance process and destroy it gracefully.
UNSAFE
Force shutdown. Internally send SIGKILL to instance process.
DESTROY
Forced shutdown. Internally send SIGKILL to instance process.
There is high data corruption risk!
If method is None NORMAL method will used.
@ -361,7 +364,7 @@ class Instance:
'SOFT': libvirt.VIR_DOMAIN_SHUTDOWN_GUEST_AGENT,
'NORMAL': libvirt.VIR_DOMAIN_SHUTDOWN_DEFAULT,
'HARD': libvirt.VIR_DOMAIN_DESTROY_GRACEFUL,
'UNSAFE': libvirt.VIR_DOMAIN_DESTROY_DEFAULT,
'DESTROY': libvirt.VIR_DOMAIN_DESTROY_DEFAULT,
}
if method is None:
method = 'NORMAL'
@ -372,11 +375,13 @@ class Instance:
method = method.upper()
if method not in methods:
raise ValueError(f"Unsupported shutdown method: '{method}'")
if method == 'SOFT' and self.guest_agent.is_available() is False:
method = 'NORMAL'
log.info("Performing instance shutdown with method '%s'", method)
try:
if method in ['SOFT', 'NORMAL']:
self.domain.shutdownFlags(flags=methods[method])
elif method in ['HARD', 'UNSAFE']:
elif method in ['HARD', 'DESTROY']:
self.domain.destroyFlags(flags=methods[method])
except libvirt.libvirtError as e:
raise InstanceError(
@ -443,8 +448,7 @@ class Instance:
self.domain.setAutostart(autostart)
except libvirt.libvirtError as e:
raise InstanceError(
f'Cannot set autostart flag for instance={self.name} '
f'{autostart=}: {e}'
f"Cannot set {autostart=} flag for instance '{self.name}': {e}"
) from e
def set_vcpus(self, nvcpus: int, *, live: bool = False) -> None:
@ -466,7 +470,7 @@ class Instance:
raise InstanceError('vCPUs count is greather than max_vcpus')
if nvcpus == self.get_info().nproc:
log.warning(
'Instance instance=%s already have %s vCPUs, nothing to do',
"Instance '%s' already have %s vCPUs, nothing to do",
self.name,
nvcpus,
)
@ -492,18 +496,17 @@ class Instance:
self.domain.setVcpusFlags(nvcpus, flags=flags)
except GuestAgentCommandNotSupportedError:
log.warning(
'Cannot set vCPUs in guest via agent, you may '
'need to apply changes in guest manually.'
"'guest-set-vcpus' command is not supported, '"
'you may need to enable CPUs in guest manually.'
)
else:
log.warning(
'Cannot set vCPUs in guest OS on instance=%s. '
'You may need to apply CPUs in guest manually.',
self.name,
'Guest agent is not installed or not connected, '
'you may need to enable CPUs in guest manually.'
)
except libvirt.libvirtError as e:
raise InstanceError(
f'Cannot set vCPUs for instance={self.name}: {e}'
f"Cannot set vCPUs for instance '{self.name}': {e}"
) from e
def set_memory(self, memory: int, *, live: bool = False) -> None:

89
compute/instance/schemas.py
View File

@ -16,11 +16,11 @@
"""Compute instance related objects schemas."""
import re
from collections import Counter
from enum import StrEnum
from pathlib import Path
from pydantic import ValidationError, validator
from pydantic.error_wrappers import ErrorWrapper
from pydantic import validator
from compute.abstract import EntityModel
from compute.utils.units import DataUnit
@ -74,12 +74,30 @@ class VolumeCapacitySchema(EntityModel):
unit: DataUnit
class DiskCache(StrEnum):
"""Possible disk cache mechanisms enumeration."""
NONE = 'none'
WRITETHROUGH = 'writethrough'
WRITEBACK = 'writeback'
DIRECTSYNC = 'directsync'
UNSAFE = 'unsafe'
class DiskDriverSchema(EntityModel):
"""Virtual disk driver model."""
name: str
type: str # noqa: A003
cache: str = 'writethrough'
cache: DiskCache = DiskCache.WRITETHROUGH
class DiskBus(StrEnum):
"""Possible disk buses enumeration."""
VIRTIO = 'virtio'
IDE = 'ide'
SATA = 'sata'
class VolumeSchema(EntityModel):
@ -92,15 +110,30 @@ class VolumeSchema(EntityModel):
source: str | None = None
is_readonly: bool = False
is_system: bool = False
bus: str = 'virtio'
bus: DiskBus = DiskBus.VIRTIO
device: str = 'disk'
class NetworkAdapterModel(StrEnum):
"""Network adapter models."""
VIRTIO = 'virtio'
E1000 = 'e1000'
RTL8139 = 'rtl8139'
class NetworkInterfaceSchema(EntityModel):
"""Network inerface model."""
source: str
mac: str
model: NetworkAdapterModel
class NetworkSchema(EntityModel):
"""Network configuration schema."""
interfaces: list[NetworkInterfaceSchema]
class BootOptionsSchema(EntityModel):
@ -134,7 +167,7 @@ class InstanceSchema(EntityModel):
arch: str
boot: BootOptionsSchema
volumes: list[VolumeSchema]
network_interfaces: list[NetworkInterfaceSchema]
network: NetworkSchema | None | bool
image: str | None = None
cloud_init: CloudInitSchema | None = None
@ -142,7 +175,7 @@ class InstanceSchema(EntityModel):
def _check_name(cls, value: str) -> str: # noqa: N805
if not re.match(r'^[a-z0-9_-]+$', value):
msg = (
'Name can contain only lowercase letters, numbers, '
'Name must contain only lowercase letters, numbers, '
'minus sign and underscore.'
)
raise ValueError(msg)
@ -162,27 +195,33 @@ class InstanceSchema(EntityModel):
if len([v for v in volumes if v.is_system is True]) != 1:
msg = 'volumes list must contain one system volume'
raise ValueError(msg)
for vol in volumes:
if vol.source is None and vol.capacity is None:
raise ValidationError(
[
ErrorWrapper(
Exception(
"capacity is required if 'source' is unset"
),
loc='X.capacity',
)
],
model=VolumeSchema,
)
if vol.is_system is True and vol.is_readonly is True:
index = 0
for volume in volumes:
index += 1
if volume.source is None and volume.capacity is None:
msg = f"{index}: capacity is required if 'source' is unset"
raise ValueError(msg)
if volume.is_system is True and volume.is_readonly is True:
msg = 'volume marked as system cannot be readonly'
raise ValueError(msg)
sources = [v.source for v in volumes if v.source is not None]
targets = [v.target for v in volumes]
for item in [sources, targets]:
duplicates = Counter(item) - Counter(set(item))
if duplicates:
msg = f'find duplicate values: {list(duplicates)}'
raise ValueError(msg)
return volumes
@validator('network_interfaces')
def _check_network_interfaces(cls, value: list) -> list: # noqa: N805
if not value:
msg = 'Network interfaces list must contain at least one element'
@validator('network')
def _check_network(
cls, # noqa: N805
network: NetworkSchema | None | bool,
) -> NetworkSchema | None | bool:
if network is True:
msg = (
"'network' cannot be True, set it to False "
'or provide network configuration'
)
raise ValueError(msg)
return value
return network

4
compute/session.py
View File

@ -207,8 +207,8 @@ class Session(AbstractContextManager):
:param volumes: List of storage volume configs. For more info
see :class:`VolumeSchema`.
:type volumes: list[dict]
:param network_interfaces: List of virtual network interfaces
configs. See :class:`NetworkInterfaceSchema` for more info.
:param network: List of virtual network interfaces configs.
See :class:`NetworkSchema` for more info.
:type network_interfaces: list[dict]
:param cloud_init: Cloud-init configuration. See
:class:`CloudInitSchema` for info.

6
compute/storage/pool.py
View File

@ -73,9 +73,9 @@ class StoragePool:
"""
Refresh storage pool.
:param retry: If True retry pool refresh on :class:`libvirtError`
with running asynchronous jobs.
:param timeout: Retry timeout in secodns. Affets only if `retry`
:param retry: If True retry pool refresh on 'pool have running
asynchronous jobs' error.
:param timeout: Retry timeout in seconds. Affects only if `retry`
is True.
"""
retry_timeout = dt.now(tz=datetime.UTC) + timedelta(seconds=timeout)

84
compute/utils/units.py
View File

@ -15,6 +15,7 @@
"""Tools for data units convertion."""
from collections.abc import Callable
from enum import StrEnum
from compute.exceptions import InvalidDataUnitError
@ -28,6 +29,14 @@ class DataUnit(StrEnum):
MIB = 'MiB'
GIB = 'GiB'
TIB = 'TiB'
KB = 'kb'
MB = 'Mb'
GB = 'Gb'
TB = 'Tb'
KBIT = 'kbit'
MBIT = 'Mbit'
GBIT = 'Gbit'
TBIT = 'Tbit'
@classmethod
def _missing_(cls, name: str) -> 'DataUnit':
@ -37,17 +46,74 @@ class DataUnit(StrEnum):
return None
def to_bytes(value: int, unit: DataUnit = DataUnit.BYTES) -> int:
"""Convert value to bytes. See :class:`DataUnit`."""
def validate_input(*args: str) -> Callable:
"""Validate data units in functions input."""
to_validate = args
def decorator(func: Callable) -> Callable:
def wrapper(*args: float | str, **kwargs: str) -> Callable:
try:
_ = DataUnit(unit)
if kwargs:
for arg in to_validate:
unit = kwargs[arg]
DataUnit(unit)
else:
for arg in args[1:]:
unit = arg
DataUnit(unit)
except ValueError as e:
raise InvalidDataUnitError(e, list(DataUnit)) from e
powers = {
return func(*args, **kwargs)
return wrapper
return decorator
@validate_input('unit')
def to_bytes(value: float, unit: DataUnit = DataUnit.BYTES) -> float:
"""Convert value to bytes."""
unit = DataUnit(unit)
basis = 2 if unit.endswith('iB') else 10
factor = 125 if unit.endswith('bit') else 1
power = {
DataUnit.BYTES: 0,
DataUnit.KIB: 1,
DataUnit.MIB: 2,
DataUnit.GIB: 3,
DataUnit.TIB: 4,
DataUnit.KIB: 10,
DataUnit.MIB: 20,
DataUnit.GIB: 30,
DataUnit.TIB: 40,
DataUnit.KB: 3,
DataUnit.MB: 6,
DataUnit.GB: 9,
DataUnit.TB: 12,
DataUnit.KBIT: 0,
DataUnit.MBIT: 3,
DataUnit.GBIT: 6,
DataUnit.TBIT: 9,
}
return value * pow(1024, powers[unit])
return value * factor * pow(basis, power[unit])
@validate_input('from_unit', 'to_unit')
def convert(value: float, from_unit: DataUnit, to_unit: DataUnit) -> float:
"""Convert units."""
value_in_bits = to_bytes(value, from_unit) * 8
to_unit = DataUnit(to_unit)
basis = 2 if to_unit.endswith('iB') else 10
divisor = 1 if to_unit.endswith('bit') else 8
power = {
DataUnit.BYTES: 0,
DataUnit.KIB: 10,
DataUnit.MIB: 20,
DataUnit.GIB: 30,
DataUnit.TIB: 40,
DataUnit.KB: 3,
DataUnit.MB: 6,
DataUnit.GB: 9,
DataUnit.TB: 12,
DataUnit.KBIT: 3,
DataUnit.MBIT: 6,
DataUnit.GBIT: 9,
DataUnit.TBIT: 12,
}
return value_in_bits / divisor / pow(basis, power[to_unit])

2
docs/source/conf.py
View File

@ -6,7 +6,7 @@ sys.path.insert(0, os.path.abspath('../..'))
project = 'Compute'
copyright = '2023, Compute Authors'
author = 'Compute Authors'
release = '0.1.0-dev3'
release = '0.1.0-dev4'
# Sphinx general settings
extensions = [

8
packaging/files/control
View File

@ -28,16 +28,18 @@ Depends:
${misc:Depends},
qemu-system,
qemu-utils,
libvirt-daemon,
libvirt-daemon-system,
libvirt-daemon-driver-qemu,
libvirt-clients,
python3-libvirt,
python3-lxml,
python3-yaml,
python3-pydantic,
mtools,
dosfstools
Recommends:
dnsmasq
dosfstools,
dnsmasq,
dnsmasq-base
Suggests:
compute-doc
Description: Compute instances management library (Python 3)

2
pyproject.toml
View File

@ -1,6 +1,6 @@
[tool.poetry]
name = 'compute'
version = '0.1.0-dev3'
version = '0.1.0-dev4'
description = 'Compute instances management library'
license = 'GPL-3.0-or-later'
authors = ['ge <ge@nixhacks.net>']