ge/python-compute
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

various improvements

Browse Source
This commit is contained in:
ge 2023-11-09 01:17:50 +03:00
parent 8e7f185fc6
commit 1dd3e9a720
28 changed files with 557 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -11,7 +11,7 @@ build: format lint
poetry build
format:
poetry run isort --lai 2 $(SRC)
poetry run isort $(SRC)
poetry run ruff format $(SRC)
lint:
@ -23,7 +23,7 @@ docs:
docs-versions:
poetry run sphinx-multiversion $(DOCS_SRC) $(DOCS_BUILD)
serve-docs: docs-versions
serve-docs:
poetry run sphinx-autobuild $(DOCS_SRC) $(DOCS_BUILD)
clean:

10
README.md
View File

@ -6,7 +6,7 @@ Currently supports only QEMU/KVM based virtual machines.
## Docs
Run `make serve-docs`.
Run `make serve-docs`. See [Development](#development) below.
## Roadmap
@ -39,3 +39,11 @@ Run `make serve-docs`.
- [ ] Instance migrations
- [ ] HTTP API
- [ ] Full functional CLI [in progress]
## Development
Install [poetry](https://python-poetry.org/), clone this repository and run:
```
poetry install --with dev --with docs
```

2
compute/__init__.py
View File

@ -1,4 +1,4 @@
"""Compute Service library."""
"""Compute instances management library."""
__version__ = '0.1.0'

26
compute/cli/_create.py
View File

@ -1,26 +0,0 @@
import argparse
from compute import Session
from compute.utils import identifiers
def _create_instance(session: Session, args: argparse.Namespace) -> None:
"""
Умолчания (достать информацию из либвирта):
- arch
- machine
- emulator
- CPU
- cpu_vendor
- cpu_model
- фичи
- max_memory
- max_vcpus
(сегнерировать):
- MAC адрес
- boot_order = ('cdrom', 'hd')
- title = ''
- name = uuid.uuid4().hex
"""
print(args)

155
compute/cli/control.py
View File

@ -1,12 +1,18 @@
"""Command line interface."""
import argparse
import io
import logging
import os
import shlex
import sys
from collections import UserDict
from typing import Any
from uuid import uuid4
import libvirt
import yaml
from pydantic import ValidationError
from compute import __version__
from compute.exceptions import (
@ -15,8 +21,7 @@ from compute.exceptions import (
)
from compute.instance import GuestAgent
from compute.session import Session
from ._create import _create_instance
from compute.utils import ids
log = logging.getLogger(__name__)
@ -37,41 +42,41 @@ class Table:
"""Initialise Table."""
self.whitespace = whitespace or '\t'
self.header = []
self._rows = []
self._table = ''
self.rows = []
self.table = ''
def row(self, row: list) -> None:
def add_row(self, row: list) -> None:
"""Add table row."""
self._rows.append([str(col) for col in row])
self.rows.append([str(col) for col in row])
def rows(self, rows: list[list]) -> None:
def add_rows(self, rows: list[list]) -> None:
"""Add multiple rows."""
for row in rows:
self.row(row)
self.add_row(row)
def __str__(self) -> str:
"""Build table and return."""
widths = [max(map(len, col)) for col in zip(*self._rows, strict=True)]
self._rows.insert(0, [str(h).upper() for h in self.header])
for row in self._rows:
self._table += self.whitespace.join(
widths = [max(map(len, col)) for col in zip(*self.rows, strict=True)]
self.rows.insert(0, [str(h).upper() for h in self.header])
for row in self.rows:
self.table += self.whitespace.join(
(
val.ljust(width)
for val, width in zip(row, widths, strict=True)
)
)
self._table += '\n'
return self._table.strip()
self.table += '\n'
return self.table.strip()
def _list_instances(session: Session) -> None:
table = Table()
table.header = ['NAME', 'STATE']
for instance in session.list_instances():
table.row(
table.add_row(
[
instance.name,
instance.status,
instance.get_status(),
]
)
print(table)
@ -113,11 +118,93 @@ def _exec_guest_agent_command(
sys.exit(output.exitcode)
class _NotPresent:
"""
Type for representing non-existent dictionary keys.
See :class:`_FillableDict`.
"""
class _FillableDict(UserDict):
"""Use :method:`fill` to add key if not present."""
def __init__(self, data: dict):
self.data = data
def fill(self, key: str, value: Any) -> None: # noqa: ANN401
if self.data.get(key, _NotPresent) is _NotPresent:
self.data[key] = value
def _merge_dicts(a: dict, b: dict, path: list[str] | None = None) -> dict:
"""Merge `b` into `a`. Return modified `a`."""
if path is None:
path = []
for key in b:
if key in a:
if isinstance(a[key], dict) and isinstance(b[key], dict):
_merge_dicts(a[key], b[key], [path + str(key)])
elif a[key] == b[key]:
pass # same leaf value
else:
a[key] = b[key] # replace existing key's values
else:
a[key] = b[key]
return a
def _create_instance(session: Session, file: io.TextIOWrapper) -> None:
try:
data = _FillableDict(yaml.load(file.read(), Loader=yaml.SafeLoader))
log.debug('Read from file: %s', data)
except yaml.YAMLError as e:
sys.exit(f'error: cannot parse YAML: {e}')
capabilities = session.get_capabilities()
node_info = session.get_node_info()
data.fill('name', uuid4().hex)
data.fill('title', None)
data.fill('description', None)
data.fill('arch', capabilities.arch)
data.fill('machine', capabilities.machine)
data.fill('emulator', capabilities.emulator)
data.fill('max_vcpus', node_info.cpus)
data.fill('max_memory', node_info.memory)
data.fill('cpu', {})
cpu = {
'emulation_mode': 'host-passthrough',
'model': None,
'vendor': None,
'topology': None,
'features': None,
}
data['cpu'] = _merge_dicts(data['cpu'], cpu)
data.fill(
'network_interfaces',
[{'source': 'default', 'mac': ids.random_mac()}],
)
data.fill('boot', {'order': ['cdrom', 'hd']})
try:
log.debug('Input data: %s', data)
session.create_instance(**data)
except ValidationError as e:
for error in e.errors():
fields = '.'.join([str(lc) for lc in error['loc']])
print(
f"validation error: {fields}: {error['msg']}",
file=sys.stderr,
)
sys.exit()
def main(session: Session, args: argparse.Namespace) -> None:
"""Perform actions."""
match args.command:
case 'create':
_create_instance(session, args)
_create_instance(session, args.file)
case 'exec':
_exec_guest_agent_command(session, args)
case 'ls':
@ -179,30 +266,13 @@ def cli() -> None: # noqa: PLR0915
subparsers = root.add_subparsers(dest='command', metavar='COMMAND')
# create command
create = subparsers.add_parser('create', help='create compute instance')
create.add_argument('image', nargs='?')
create.add_argument('--name', help='instance name, used as ID')
create.add_argument('--title', help='human-understandable instance title')
create.add_argument('--desc', default='', help='instance description')
create.add_argument('--memory', type=int, help='memory in MiB')
create.add_argument('--max-memory', type=int, help='max memory in MiB')
create.add_argument('--vcpus', type=int)
create.add_argument('--max-vcpus', type=int)
create.add_argument('--cpu-vendor')
create.add_argument('--cpu-model')
create.add_argument(
'--cpu-emulation-mode',
choices=['host-passthrough', 'host-model', 'custom'],
default='host-passthrough',
create = subparsers.add_parser(
'create', help='create new instance from YAML config file'
)
create.add_argument(
'file',
type=argparse.FileType('r', encoding='UTF-8'),
)
create.add_argument('--cpu-features')
create.add_argument('--cpu-topology')
create.add_argument('--mahine')
create.add_argument('--emulator')
create.add_argument('--arch')
create.add_argument('--boot-order')
create.add_argument('--volume')
create.add_argument('-f', '--file', help='create instance from YAML')
# exec subcommand
execute = subparsers.add_parser(
@ -303,14 +373,17 @@ def cli() -> None: # noqa: PLR0915
if log_level in log_levels:
logging.basicConfig(level=log_levels[log_level])
log.debug('CLI started with args: %s', args)
# Perform actions
try:
with Session(args.connect) as session:
main(session, args)
except ComputeServiceError as e:
sys.exit(f'error: {e}')
except (KeyboardInterrupt, SystemExit):
except KeyboardInterrupt:
sys.exit()
except SystemExit as e:
sys.exit(e)
except Exception as e: # noqa: BLE001
sys.exit(f'unexpected error {type(e)}: {e}')

2
compute/exceptions.py
View File

@ -1,4 +1,4 @@
"""Compute Service exceptions."""
"""Exceptions."""
class ComputeServiceError(Exception):

27
compute/instance/guest_agent.py
View File

@ -1,4 +1,4 @@
"""Manage QEMU guest agent."""
"""Interacting with the QEMU Guest Agent."""
import json
import logging
@ -19,9 +19,6 @@ from compute.exceptions import (
log = logging.getLogger(__name__)
QEMU_TIMEOUT = 60
POLL_INTERVAL = 0.3
class GuestExecOutput(NamedTuple):
"""QEMU guest-exec command output."""
@ -35,7 +32,7 @@ class GuestExecOutput(NamedTuple):
class GuestAgent:
"""Class for interacting with QEMU guest agent."""
def __init__(self, domain: libvirt.virDomain, timeout: int | None = None):
def __init__(self, domain: libvirt.virDomain, timeout: int = 60):
"""
Initialise GuestAgent.
@ -43,7 +40,7 @@ class GuestAgent:
:param timeout: QEMU timeout
"""
self.domain = domain
self.timeout = timeout or QEMU_TIMEOUT
self.timeout = timeout
self.flags = libvirt_qemu.VIR_DOMAIN_QEMU_MONITOR_COMMAND_DEFAULT
self.last_pid = None
@ -65,9 +62,6 @@ class GuestAgent:
return json.loads(output)
except libvirt.libvirtError as e:
if e.get_error_code() == libvirt.VIR_ERR_AGENT_UNRESPONSIVE:
log.exception(
'Guest agent is unavailable on instanse=%s', self.name
)
raise GuestAgentUnavailableError(e) from e
raise GuestAgentError(e) from e
@ -95,9 +89,7 @@ class GuestAgent:
def raise_for_commands(self, commands: list[str]) -> None:
"""
Check QEMU guest agent command availability.
Raise exception if command is not available.
Raise exception if QEMU GA command is not available.
:param commands: List of required commands
:raise: GuestAgentCommandNotSupportedError
@ -164,12 +156,15 @@ class GuestAgent:
stderr = b64decode(stderr or '').decode('utf-8')
return GuestExecOutput(exited, exitcode, stdout, stderr)
def guest_exec_status(self, pid: int, *, poll: bool = False) -> dict:
def guest_exec_status(
self, pid: int, *, poll: bool = False, poll_interval: float = 0.3
) -> dict:
"""
Execute guest-exec-status and return output.
:param pid: PID in guest
:param poll: If True poll command status with POLL_INTERVAL
:param pid: PID in guest.
:param poll: If True poll command status.
:param poll_interval: Time between attempts to obtain command status.
:return: Command output
:rtype: dict
"""
@ -185,7 +180,7 @@ class GuestAgent:
command_status = self.execute(command)
if command_status['return']['exited']:
break
sleep(POLL_INTERVAL)
sleep(poll_interval)
now = time()
if now - start_time > self.timeout:
raise GuestAgentTimeoutExceededError(self.timeout)

141
compute/instance/instance.py
View File

@ -1,9 +1,9 @@
"""Manage compute instances."""
__all__ = ['Instance', 'InstanceConfig']
__all__ = ['Instance', 'InstanceConfig', 'InstanceInfo']
import logging
from dataclasses import dataclass
from typing import NamedTuple
import libvirt
from lxml import etree
@ -16,14 +16,19 @@ from compute.exceptions import (
from compute.utils import units
from .guest_agent import GuestAgent
from .schemas import CPUSchema, InstanceSchema, NetworkInterfaceSchema
from .schemas import (
CPUEmulationMode,
CPUSchema,
InstanceSchema,
NetworkInterfaceSchema,
)
log = logging.getLogger(__name__)
class InstanceConfig:
"""Compute instance description for libvirt."""
"""Compute instance config builder."""
def __init__(self, schema: InstanceSchema):
"""
@ -46,21 +51,33 @@ class InstanceConfig:
self.network_interfaces = schema.network_interfaces
def _gen_cpu_xml(self, cpu: CPUSchema) -> etree.Element:
xml = E.cpu(match='exact', mode=cpu.emulation_mode)
xml.append(E.model(cpu.model, fallback='forbid'))
xml.append(E.vendor(cpu.vendor))
xml.append(
E.topology(
sockets=str(cpu.topology.sockets),
dies=str(cpu.topology.dies),
cores=str(cpu.topology.cores),
threads=str(cpu.topology.threads),
options = {
'mode': cpu.emulation_mode,
'match': 'exact',
'check': 'partial',
}
if cpu.emulation_mode == CPUEmulationMode.HOST_PASSTHROUGH:
options['check'] = 'none'
options['migratable'] = 'on'
xml = E.cpu(**options)
if cpu.model:
xml.append(E.model(cpu.model, fallback='forbid'))
if cpu.vendor:
xml.append(E.vendor(cpu.vendor))
if cpu.topology:
xml.append(
E.topology(
sockets=str(cpu.topology.sockets),
dies=str(cpu.topology.dies),
cores=str(cpu.topology.cores),
threads=str(cpu.topology.threads),
)
)
)
for feature in cpu.features.require:
xml.append(E.feature(policy='require', name=feature))
for feature in cpu.features.disable:
xml.append(E.feature(policy='disable', name=feature))
if cpu.features:
for feature in cpu.features.require:
xml.append(E.feature(policy='require', name=feature))
for feature in cpu.features.disable:
xml.append(E.feature(policy='disable', name=feature))
return xml
def _gen_vcpus_xml(self, vcpus: int, max_vcpus: int) -> etree.Element:
@ -89,15 +106,15 @@ class InstanceConfig:
def to_xml(self) -> str:
"""Return XML config for libvirt."""
xml = E.domain(
E.name(self.name),
E.title(self.title),
E.description(self.description),
E.metadata(),
E.memory(str(self.memory * 1024), unit='KiB'),
E.currentMemory(str(self.memory * 1024), unit='KiB'),
type='kvm',
)
xml = E.domain(type='kvm')
xml.append(E.name(self.name))
if self.title:
xml.append(E.title(self.title))
if self.description:
xml.append(E.description(self.description))
xml.append(E.metadata())
xml.append(E.memory(str(self.max_memory * 1024), unit='KiB'))
xml.append(E.currentMemory(str(self.memory * 1024), unit='KiB'))
xml.append(
E.vcpu(
str(self.max_vcpus),
@ -148,8 +165,14 @@ class InstanceConfig:
return etree.tostring(xml, encoding='unicode', pretty_print=True)
@dataclass
class InstanceInfo:
class InstanceInfo(NamedTuple):
"""
Store compute instance info.
Reference:
https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainInfo
"""
state: str
max_memory: int
memory: int
@ -193,13 +216,8 @@ class Instance:
}
return states[state]
@property
def info(self) -> InstanceInfo:
"""
Return instance info.
https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainInfo
"""
def get_info(self) -> InstanceInfo:
"""Return instance info."""
_info = self.domain.info()
return InstanceInfo(
state=self._expand_instance_state(_info[0]),
@ -209,8 +227,7 @@ class Instance:
cputime=_info[4],
)
@property
def status(self) -> str:
def get_status(self) -> str:
"""
Return instance state: 'running', 'shutoff', etc.
@ -225,7 +242,6 @@ class Instance:
) from e
return self._expand_instance_state(state)
@property
def is_running(self) -> bool:
"""Return True if instance is running, else return False."""
if self.domain.isActive() != 1:
@ -233,7 +249,6 @@ class Instance:
return False
return True
@property
def is_autostart(self) -> bool:
"""Return True if instance autostart is enabled, else return False."""
try:
@ -244,10 +259,18 @@ class Instance:
f'instance={self.name}: {e}'
) from e
def get_max_memory(self) -> int:
"""Maximum memory value for domain in KiB."""
return self.domain.maxMemory()
def get_max_vcpus(self) -> int:
"""Maximum vCPUs number for domain."""
return self.domain.maxVcpus()
def start(self) -> None:
"""Start defined instance."""
log.info('Starting instnce=%s', self.name)
if self.is_running:
if self.is_running():
log.warning(
'Already started, nothing to do instance=%s', self.name
)
@ -311,6 +334,15 @@ class Instance:
f'Cannot shutdown instance={self.name} ' f'{method=}: {e}'
) from e
def reboot(self) -> None:
"""Send ACPI signal to guest OS to reboot. OS may ignore this."""
try:
self.domain.reboot()
except libvirt.libvirtError as e:
raise InstanceError(
f'Cannot reboot instance={self.name}: {e}'
) from e
def reset(self) -> None:
"""
Reset instance.
@ -331,14 +363,19 @@ class Instance:
f'Cannot reset instance={self.name}: {e}'
) from e
def reboot(self) -> None:
"""Send ACPI signal to guest OS to reboot. OS may ignore this."""
try:
self.domain.reboot()
except libvirt.libvirtError as e:
raise InstanceError(
f'Cannot reboot instance={self.name}: {e}'
) from e
def power_reset(self) -> None:
"""
Shutdown instance and start.
By analogy with real hardware, this is a normal server shutdown,
and then turning off from the power supply and turning it on again.
This method is applicable in cases where there has been a
configuration change in libvirt and you need to restart the
instance to apply the new configuration.
"""
self.shutdown(method='NORMAL')
self.start()
def set_autostart(self, *, enabled: bool) -> None:
"""
@ -383,7 +420,7 @@ class Instance:
flags = libvirt.VIR_DOMAIN_AFFECT_CONFIG
self.domain.setVcpusFlags(nvcpus, flags=flags)
if live is True:
if not self.is_running:
if not self.is_running():
log.warning(
'Instance is not running, changes applied in '
'instance config.'
@ -453,7 +490,7 @@ class Instance:
:param device: Object with device description e.g. DiskConfig
:param live: Affect a running instance
"""
if live and self.is_running:
if live and self.is_running():
flags = (
libvirt.VIR_DOMAIN_AFFECT_LIVE
| libvirt.VIR_DOMAIN_AFFECT_CONFIG
@ -471,7 +508,7 @@ class Instance:
:param device: Object with device description e.g. DiskConfig
:param live: Affect a running instance
"""
if live and self.is_running:
if live and self.is_running():
flags = (
libvirt.VIR_DOMAIN_AFFECT_LIVE
| libvirt.VIR_DOMAIN_AFFECT_CONFIG

72
compute/instance/schemas.py
View File

@ -4,11 +4,20 @@ import re
from enum import StrEnum
from pathlib import Path
from pydantic import BaseModel, validator
from pydantic import BaseModel, Extra, validator
from compute.utils.units import DataUnit
class EntityModel(BaseModel):
"""Basic entity model."""
class Config:
"""Do not allow extra fields."""
extra = Extra.forbid
class CPUEmulationMode(StrEnum):
"""CPU emulation mode enumerated."""
@ -18,7 +27,7 @@ class CPUEmulationMode(StrEnum):
MAXIMUM = 'maximum'
class CPUTopologySchema(BaseModel):
class CPUTopologySchema(EntityModel):
"""CPU topology model."""
sockets: int
@ -27,67 +36,66 @@ class CPUTopologySchema(BaseModel):
dies: int = 1
class CPUFeaturesSchema(BaseModel):
class CPUFeaturesSchema(EntityModel):
"""CPU features model."""
require: list[str]
disable: list[str]
class CPUSchema(BaseModel):
class CPUSchema(EntityModel):
"""CPU model."""
emulation_mode: CPUEmulationMode
model: str
vendor: str
topology: CPUTopologySchema
features: CPUFeaturesSchema
model: str | None
vendor: str | None
topology: CPUTopologySchema | None
features: CPUFeaturesSchema | None
class VolumeType(StrEnum):
"""Storage volume types enumeration."""
FILE = 'file'
NETWORK = 'network'
class VolumeCapacitySchema(BaseModel):
class VolumeCapacitySchema(EntityModel):
"""Storage volume capacity field model."""
value: int
unit: DataUnit
class VolumeSchema(BaseModel):
class VolumeSchema(EntityModel):
"""Storage volume model."""
type: VolumeType # noqa: A003
source: Path
target: str
capacity: VolumeCapacitySchema
readonly: bool = False
source: str | None = None
is_readonly: bool = False
is_system: bool = False
class NetworkInterfaceSchema(BaseModel):
class NetworkInterfaceSchema(EntityModel):
"""Network inerface model."""
source: str
mac: str
class BootOptionsSchema(BaseModel):
class BootOptionsSchema(EntityModel):
"""Instance boot settings."""
order: tuple
class InstanceSchema(BaseModel):
class InstanceSchema(EntityModel):
"""Compute instance model."""
name: str
title: str
description: str
title: str | None
description: str | None
memory: int
max_memory: int
vcpus: int
@ -96,10 +104,10 @@ class InstanceSchema(BaseModel):
machine: str
emulator: Path
arch: str
image: str
boot: BootOptionsSchema
volumes: list[VolumeSchema]
network_interfaces: list[NetworkInterfaceSchema]
image: str | None = None
@validator('name')
def _check_name(cls, value: str) -> str: # noqa: N805
@ -111,12 +119,28 @@ class InstanceSchema(BaseModel):
raise ValueError(msg)
return value
@validator('volumes')
def _check_volumes(cls, value: list) -> list: # noqa: N805
if len([v for v in value if v.is_system is True]) != 1:
msg = 'Volumes list must contain one system volume'
@validator('cpu')
def _check_topology(cls, cpu: int, values: dict) -> CPUSchema: # noqa: N805
topo = cpu.topology
max_vcpus = values['max_vcpus']
if topo and topo.sockets * topo.cores * topo.threads != max_vcpus:
msg = f'CPU topology does not match with {max_vcpus=}'
raise ValueError(msg)
return value
return cpu
@validator('volumes')
def _check_volumes(cls, volumes: list) -> list: # noqa: N805
if len([v for v in volumes if v.is_system is True]) != 1:
msg = 'volumes list must contain one system volume'
raise ValueError(msg)
vol_with_source = 0
for vol in volumes:
if vol.is_system is True and vol.is_readonly is True:
msg = 'volume marked as system cannot be readonly'
raise ValueError(msg)
if vol.source is not None:
vol_with_source += 1
return volumes
@validator('network_interfaces')
def _check_network_interfaces(cls, value: list) -> list: # noqa: N805

73
compute/session.py
View File

@ -26,6 +26,25 @@ class Capabilities(NamedTuple):
virt: str
emulator: str
machine: str
max_vcpus: int
class NodeInfo(NamedTuple):
"""
Store compute node info.
See https://libvirt.org/html/libvirt-libvirt-host.html#virNodeInfo
NOTE: memory unit in libvirt docs is wrong! Actual unit is MiB.
"""
arch: str
memory: int
cpus: int
mhz: int
nodes: int
sockets: int
cores: int
threads: int
class Session(AbstractContextManager):
@ -68,7 +87,21 @@ class Session(AbstractContextManager):
"""Close connection to libvirt daemon."""
self.connection.close()
def capabilities(self) -> Capabilities:
def get_node_info(self) -> NodeInfo:
"""Return information about compute node."""
info = self.connection.getInfo()
return NodeInfo(
arch=info[0],
memory=info[1],
cpus=info[2],
mhz=info[3],
nodes=info[4],
sockets=info[5],
cores=info[6],
threads=info[7],
)
def get_capabilities(self) -> Capabilities:
"""Return capabilities e.g. arch, virt, emulator, etc."""
prefix = '/domainCapabilities'
caps = etree.fromstring(self.connection.getDomainCapabilities()) # noqa: S320
@ -77,6 +110,7 @@ class Session(AbstractContextManager):
virt=caps.xpath(f'{prefix}/domain/text()')[0],
emulator=caps.xpath(f'{prefix}/path/text()')[0],
machine=caps.xpath(f'{prefix}/machine/text()')[0],
max_vcpus=int(caps.xpath(f'{prefix}/vcpu/@max')[0]),
)
def create_instance(self, **kwargs: Any) -> Instance:
@ -87,14 +121,35 @@ class Session(AbstractContextManager):
:type name: str
:param title: Instance title for humans.
:type title: str
:param description: Some information about instance
:param description: Some information about instance.
:type description: str
:param memory: Memory in MiB.
:type memory: int
:param max_memory: Maximum memory in MiB.
:type max_memory: int
:param vcpus: Number of vCPUs.
:type vcpus: int
:param max_vcpus: Maximum vCPUs.
:type max_vcpus: int
:param cpu: CPU configuration. See :class:`CPUSchema` for info.
:type cpu: dict
:param machine: QEMU emulated machine.
:type machine: str
:param emulator: Path to emulator.
:type emulator: str
:param arch: CPU architecture to virtualization.
:type arch: str
:param boot: Boot settings. See :class:`BootOptionsSchema`.
:type boot: dict
:param image: Source disk image name for system disk.
:type image: str
:param volumes: List of storage volume configs. For more info
see :class:`VolumeSchema`.
:type volumes: list[dict]
:param network_interfaces: List of virtual network interfaces
configs. See :class:`NetworkInterfaceSchema` for more info.
:type network_interfaces: list[dict]
"""
# TODO @ge: create instances in transaction
data = InstanceSchema(**kwargs)
config = InstanceConfig(data)
log.info('Define XML...')
@ -113,19 +168,17 @@ class Session(AbstractContextManager):
log.info('Connecting to volumes pool...')
volumes_pool = self.get_storage_pool(self.VOLUMES_POOL)
log.info('Building volume configuration...')
# if not volume.source:
# В случае если пользователь передаёт source для волюма, следует
# в либвирте делать поиск волюма по пути, а не по имени
# gen_vol_name
# TODO @ge: come up with something else
vol_name = f'{config.name}-{volume.target}-{uuid4()}.qcow2'
if not volume.source:
vol_name = f'{config.name}-{volume.target}-{uuid4()}.qcow2'
else:
vol_name = volume.source
vol_conf = VolumeConfig(
name=vol_name,
path=str(volumes_pool.path.joinpath(vol_name)),
capacity=capacity,
)
log.info('Volume configuration is:\n %s', vol_conf.to_xml())
if volume.is_system is True:
if volume.is_system is True and data.image:
log.info(
"Volume is marked as 'system', start cloning image..."
)

14
compute/storage/pool.py
View File

@ -15,8 +15,8 @@ from .volume import Volume, VolumeConfig
log = logging.getLogger(__name__)
class StoragePoolUsage(NamedTuple):
"""Storage pool usage info schema."""
class StoragePoolUsageInfo(NamedTuple):
"""Storage pool usage info."""
capacity: int
allocation: int
@ -30,17 +30,17 @@ class StoragePool:
"""Initislise StoragePool."""
self.pool = pool
self.name = pool.name()
self.path = self._get_path()
@property
def path(self) -> Path:
def _get_path(self) -> Path:
"""Return storage pool path."""
xml = etree.fromstring(self.pool.XMLDesc()) # noqa: S320
return Path(xml.xpath('/pool/target/path/text()')[0])
def usage(self) -> StoragePoolUsage:
def get_usage_info(self) -> StoragePoolUsageInfo:
"""Return info about storage pool usage."""
xml = etree.fromstring(self.pool.XMLDesc()) # noqa: S320
return StoragePoolUsage(
return StoragePoolUsageInfo(
capacity=int(xml.xpath('/pool/capacity/text()')[0]),
allocation=int(xml.xpath('/pool/allocation/text()')[0]),
available=int(xml.xpath('/pool/available/text()')[0]),
@ -58,7 +58,7 @@ class StoragePool:
def create_volume(self, vol_conf: VolumeConfig) -> Volume:
"""Create storage volume and return Volume instance."""
log.info(
'Create storage volume vol=%s in pool=%s', vol_conf.name, self.pool
'Create storage volume vol=%s in pool=%s', vol_conf.name, self.name
)
vol = self.pool.createXML(
vol_conf.to_xml(),

6
compute/storage/volume.py
View File

@ -87,11 +87,7 @@ class Volume:
self.pool_name = pool.name()
self.vol = vol
self.name = vol.name()
@property
def path(self) -> Path:
"""Return path to volume."""
return Path(self.vol.path())
self.path = Path(vol.path())
def dump_xml(self) -> str:
"""Return volume XML description as string."""

0
compute/utils/identifiers.py → compute/utils/ids.py
View File

1
docs/source/conf.py
View File

@ -1,3 +1,4 @@
# Add ../../compute to path for autodoc
import os
import sys
sys.path.insert(0, os.path.abspath('../../compute'))

5
docs/source/python-api/exceptions.rst Normal file
View File

@ -0,0 +1,5 @@
``exceptions``
==============
.. automodule:: compute.exceptions
:members:

2
docs/source/python-api/index.rst
View File

@ -45,3 +45,5 @@ Modules documentation
session
instance/index
storage/index
utils
exceptions

1
docs/source/python-api/instance/guest_agent.rst
View File

@ -3,3 +3,4 @@
.. automodule:: compute.instance.guest_agent
:members:
:special-members: __init__

2
docs/source/python-api/instance/instance.rst
View File

@ -3,4 +3,4 @@
.. automodule:: compute.instance.instance
:members:
:special-members:
:special-members: __init__

7
docs/source/python-api/session.rst
View File

@ -1,9 +1,6 @@
``session``
===========
.. autoclass:: compute.Session
:members:
:special-members:
.. autoclass:: compute.session.Capabilities
.. automodule:: compute.session
:members:
:special-members: __init__

1
docs/source/python-api/storage/pool.rst
View File

@ -3,3 +3,4 @@
.. automodule:: compute.storage.pool
:members:
:special-members: __init__

1
docs/source/python-api/storage/volume.rst
View File

@ -3,3 +3,4 @@
.. automodule:: compute.storage.volume
:members:
:special-members: __init__

14
docs/source/python-api/utils.rst Normal file
View File

@ -0,0 +1,14 @@
``utils``
=========
``utils.units``
---------------
.. automodule:: compute.utils.units
:members:
``utils.ids``
-------------
.. automodule:: compute.utils.ids
:members:

25
fdict.py Normal file
View File

@ -0,0 +1,25 @@
from collections import UserDict
from typing import Any
class _NotPresent:
"""
Type for representing non-existent dictionary keys.
See :class:`_FillableDict`
"""
class _FillableDict(UserDict):
"""Use :method:`fill` to add key if not present."""
def __init__(self, data: dict):
self.data = data
def fill(self, key: str, value: Any) -> None:
if self.data.get(key, _NotPresent) is _NotPresent:
self.data[key] = value
d = _FillableDict({'a': None, 'b': 'BBBB'})
d.fill('c', 'CCCCCCCCC')
d.fill('a', 'CCCCCCCCC')
d['a'].fill('gg', 'AAAAAAAA')
print(d)

10
instance.yaml Normal file
View File

@ -0,0 +1,10 @@
title: dev-1
vcpus: 4
memory: 4096
volumes:
- is_system: true
type: file
target: vda
capacity:
value: 5
unit: GiB

34
pars.py Normal file
View File

@ -0,0 +1,34 @@
import re
def _split_unit(val: str) -> dict | None:
match = re.match(r'([0-9]+)([a-z]+)', val, re.I)
if match:
return {
'value': match.groups()[0],
'unit': match.groups()[1],
}
return None
def _parse_complex_arg(arg: str) -> dict:
# key=value --> {'key': 'value'}
if re.match(r'.+=.+', arg):
key, val = arg.split('=')
# system --> {'is_system': True}
# ro --> {'is_readonly': True}
elif re.match(r'^[a-z0-9_\.\-]+$', arg, re.I):
key = 'is_' + arg.replace('ro', 'readonly')
val = True
else:
raise ValueError('Invalid argument pattern')
# key=15GiB --> {'key': {'value': 15, 'unit': 'GiB'}}
if not isinstance(val, bool):
val = _split_unit(val) or val
return {key: val}
print(_parse_complex_arg('source=/volumes/50c4410b-2ef0-4ffd-a2e5-04f0212772d4.qcow2'))
print(_parse_complex_arg('capacity=15GiB'))
print(_parse_complex_arg('system'))
print(_parse_complex_arg('cpu.cores=8'))

20
pd.py Normal file
View File

@ -0,0 +1,20 @@
from pydantic import BaseModel, Extra
class EntityModel(BaseModel):
"""Basic entity model."""
aaa: int
bbb: int
class Config:
extra = Extra.forbid
class Some(EntityModel):
ooo: str
www: str
a = Some(ooo='dsda', www='wcd', sds=1)
print(a)

64
poetry.lock generated
View File

@ -511,6 +511,66 @@ files = [
[package.extras]
plugins = ["importlib-metadata"]
[[package]]
name = "pyyaml"
version = "6.0.1"
description = "YAML parser and emitter for Python"
category = "main"
optional = false
python-versions = ">=3.6"
files = [
{file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"},
{file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"},
{file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"},
{file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"},
{file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"},
{file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"},
{file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"},
{file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"},
{file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"},
{file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"},
{file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"},
{file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"},
{file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"},
{file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"},
{file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"},
{file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
{file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
{file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
{file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
{file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
{file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
{file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"},
{file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"},
{file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"},
{file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"},
{file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"},
{file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"},
{file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"},
{file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"},
{file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"},
{file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"},
{file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"},
{file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"},
{file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"},
{file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"},
{file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"},
{file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"},
{file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"},
{file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"},
{file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"},
{file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"},
{file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"},
{file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"},
{file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"},
{file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"},
{file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"},
{file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"},
{file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"},
{file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"},
{file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"},
]
[[package]]
name = "requests"
version = "2.31.0"
@ -834,5 +894,5 @@ zstd = ["zstandard (>=0.18.0)"]
[metadata]
lock-version = "2.0"
python-versions = "^3.11"
content-hash = "413ca8b2e0d37bf9e2835dd9050a3cc98e4a37186c78b780a65d62d05adce8c1"
python-versions = '^3.11'
content-hash = "e5c07eebe683b92360ec12cada14fc5ccbe4e4add52549bf978f580e551abfb0"

57
pyproject.toml
View File

@ -1,53 +1,60 @@
[tool.poetry]
name = "compute"
version = "0.1.0"
description = "Library built on top of libvirt for Compute Service"
authors = ["ge <ge@nixhacks.net>"]
readme = "README.md"
name = 'compute'
version = '0.1.0'
description = 'Compute instances management library'
authors = ['ge <ge@nixhacks.net>']
readme = 'README.md'
[tool.poetry.dependencies]
python = "^3.11"
libvirt-python = "9.0.0"
lxml = "^4.9.2"
pydantic = "1.10.4"
python = '^3.11'
libvirt-python = '9.0.0'
lxml = '^4.9.2'
pydantic = '1.10.4'
pyyaml = "^6.0.1"
[tool.poetry.scripts]
compute = "compute.cli.control:cli"
compute = 'compute.cli.control:cli'
[tool.poetry.group.dev.dependencies]
ruff = "^0.1.3"
isort = "^5.12.0"
ruff = '^0.1.3'
isort = '^5.12.0'
[tool.poetry.group.docs.dependencies]
sphinx = "^7.2.6"
sphinx-autobuild = "^2021.3.14"
sphinx-multiversion = "^0.2.4"
sphinx = '^7.2.6'
sphinx-autobuild = '^2021.3.14'
sphinx-multiversion = '^0.2.4'
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
requires = ['poetry-core']
build-backend = 'poetry.core.masonry.api'
[tool.isort]
skip = ['.gitignore']
lines_after_imports = 2
include_trailing_comma = true
split_on_trailing_comma = true
[tool.ruff]
line-length = 79
indent-width = 4
target-version = "py311"
target-version = 'py311'
[tool.ruff.lint]
select = ["ALL"]
select = ['ALL']
ignore = [
"Q000", "Q003", "D211", "D212", "ANN101", "ISC001", "COM812",
"D203", "ANN204", "T201",
"EM102", "TRY003", # maybe not ignore?
"TD003", "TD006", "FIX002", # todo strings linting
'Q000', 'Q003', 'D211', 'D212', 'ANN101', 'ISC001', 'COM812',
'D203', 'ANN204', 'T201',
'EM102', 'TRY003', # maybe not ignore?
'TD003', 'TD006', 'FIX002', # todo strings linting
]
exclude = ["__init__.py"]
exclude = ['__init__.py']
[tool.ruff.lint.flake8-annotations]
mypy-init-return = true
allow-star-arg-any = true
[tool.ruff.format]
quote-style = "single"
quote-style = 'single'
[tool.ruff.isort]
lines-after-imports = 2