ge/python-compute
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

various improvements

Browse Source
This commit is contained in:
ge
2023-12-03 23:25:34 +03:00
parent 0d5246e95e
commit b0fa1b7b25
36 changed files with 842 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
compute/__init__.py
View File

@@ -5,17 +5,17 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Compute instances management library."""
__version__ = '0.1.0-dev1'
__version__ = '0.1.0-dev2'
from .instance import Instance, InstanceConfig, InstanceSchema
from .session import Session

4
compute/__main__.py
View File

@@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Command line interface for compute module."""

285
compute/cli/control.py
View File

@@ -5,25 +5,25 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Command line interface."""
import argparse
import io
import json
import logging
import os
import re
import shlex
import string
import sys
from collections import UserDict
from typing import Any
from uuid import uuid4
import uuid
import libvirt
import yaml
@@ -31,9 +31,10 @@ from pydantic import ValidationError
from compute import __version__
from compute.exceptions import ComputeError, GuestAgentTimeoutError
from compute.instance import GuestAgent
from compute.instance import GuestAgent, Instance, InstanceSchema
from compute.instance.devices import DiskConfig, DiskDriver
from compute.session import Session
from compute.utils import ids
from compute.utils import dictutil, ids
log = logging.getLogger(__name__)
@@ -128,78 +129,77 @@ def _exec_guest_agent_command(
sys.exit(output.exitcode)
class _NotPresent:
"""
Type for representing non-existent dictionary keys.
See :class:`_FillableDict`.
"""
class _FillableDict(UserDict):
"""Use :method:`fill` to add key if not present."""
def __init__(self, data: dict):
self.data = data
def fill(self, key: str, value: Any) -> None: # noqa: ANN401
if self.data.get(key, _NotPresent) is _NotPresent:
self.data[key] = value
def _merge_dicts(a: dict, b: dict, path: list[str] | None = None) -> dict:
"""Merge `b` into `a`. Return modified `a`."""
if path is None:
path = []
for key in b:
if key in a:
if isinstance(a[key], dict) and isinstance(b[key], dict):
_merge_dicts(a[key], b[key], [path + str(key)])
elif a[key] == b[key]:
pass # same leaf value
else:
a[key] = b[key] # replace existing key's values
else:
a[key] = b[key]
return a
def _create_instance(session: Session, file: io.TextIOWrapper) -> None:
def _init_instance(session: Session, args: argparse.Namespace) -> None:
try:
data = _FillableDict(yaml.load(file.read(), Loader=yaml.SafeLoader))
data = yaml.load(args.file.read(), Loader=yaml.SafeLoader)
log.debug('Read from file: %s', data)
except yaml.YAMLError as e:
sys.exit(f'error: cannot parse YAML: {e}')
capabilities = session.get_capabilities()
node_info = session.get_node_info()
data.fill('name', uuid4().hex)
data.fill('title', None)
data.fill('description', None)
data.fill('arch', capabilities.arch)
data.fill('machine', capabilities.machine)
data.fill('emulator', capabilities.emulator)
data.fill('max_vcpus', node_info.cpus)
data.fill('max_memory', node_info.memory)
data.fill('cpu', {})
cpu = {
'emulation_mode': 'host-passthrough',
'model': None,
'vendor': None,
'topology': None,
'features': None,
base_instance_config = {
'name': str(uuid.uuid4()),
'title': None,
'description': None,
'arch': capabilities.arch,
'machine': capabilities.machine,
'emulator': capabilities.emulator,
'max_vcpus': node_info.cpus,
'max_memory': node_info.memory,
'cpu': {
'emulation_mode': 'host-passthrough',
'model': None,
'vendor': None,
'topology': None,
'features': None,
},
'network_interfaces': [
{
'source': 'default',
'mac': ids.random_mac(),
},
],
'boot': {'order': ['cdrom', 'hd']},
}
data['cpu'] = _merge_dicts(data['cpu'], cpu)
data.fill(
'network_interfaces',
[{'source': 'default', 'mac': ids.random_mac()}],
)
data.fill('boot', {'order': ['cdrom', 'hd']})
data = dictutil.override(base_instance_config, data)
volumes = []
for volume in data['volumes']:
base_disk_config = {
'bus': 'virtio',
'is_readonly': False,
'driver': {
'name': 'qemu',
'type': 'qcow2',
'cache': 'writethrough',
},
}
base_cdrom_config = {
'bus': 'ide',
'target': 'hda',
'is_readonly': True,
'driver': {
'name': 'qemu',
'type': 'raw',
'cache': 'writethrough',
},
}
if volume.get('device') is None:
volume['device'] = 'disk'
if volume['device'] == 'disk':
volumes.append(dictutil.override(base_disk_config, volume))
if volume['device'] == 'cdrom':
volumes.append(dictutil.override(base_cdrom_config, volume))
data['volumes'] = volumes
try:
log.debug('Input data: %s', data)
session.create_instance(**data)
if args.test:
_ = InstanceSchema(**data)
print(json.dumps(dict(data), indent=4, sort_keys=True))
sys.exit()
instance = session.create_instance(**data)
print(f'initialised: {instance.name}')
if args.start:
instance.start()
except ValidationError as e:
for error in e.errors():
fields = '.'.join([str(lc) for lc in error['loc']])
@@ -223,11 +223,84 @@ def _shutdown_instance(session: Session, args: argparse.Namespace) -> None:
instance.shutdown(method)
def _confirm(message: str, *, default: bool | None = None) -> None:
while True:
match default:
case True:
prompt = 'default: yes'
case False:
prompt = 'default: no'
case _:
prompt = 'no default'
try:
answer = input(f'{message} ({prompt}) ')
except KeyboardInterrupt:
sys.exit('aborted')
if not answer and isinstance(default, bool):
return default
if re.match(r'^y(es)?$', answer, re.I):
return True
if re.match(r'^no?$', answer, re.I):
return False
print("Please respond 'yes' or 'no'")
def _delete_instance(session: Session, args: argparse.Namespace) -> None:
if args.yes is True or _confirm(
'this action is irreversible, continue?',
default=False,
):
instance = session.get_instance(args.instance)
if args.save_volumes is False:
instance.delete(with_volumes=True)
else:
instance.delete()
else:
print('aborted')
sys.exit()
def _get_disk_target(instance: Instance, prefix: str = 'hd') -> str:
disks_live = instance.list_disks(persistent=False)
disks_inactive = instance.list_disks(persistent=True)
disks = [d for d in disks_inactive if d not in disks_live]
devs = [d.target[-1] for d in disks if d.target.startswith(prefix)]
return prefix + [x for x in string.ascii_lowercase if x not in devs][0] # noqa: RUF015
def _manage_cdrom(session: Session, args: argparse.Namespace) -> None:
instance = session.get_instance(args.instance)
if args.detach:
for disk in instance.list_disks(persistent=True):
if disk.device == 'cdrom' and disk.source == args.source:
instance.detach_disk(disk.target, live=False)
print(
f"disk '{disk.target}' detached, "
'perform power reset to apply changes'
)
return
target = _get_disk_target(instance, 'hd')
cdrom = DiskConfig(
type='file',
device='cdrom',
source=args.source,
target=target,
is_readonly=True,
bus='ide',
driver=DiskDriver('qemu', 'raw', 'writethrough'),
)
instance.attach_device(cdrom, live=False)
print(
f"CDROM attached as disk '{target}', "
'perform power reset to apply changes'
)
def main(session: Session, args: argparse.Namespace) -> None:
"""Perform actions."""
match args.command:
case 'init':
_create_instance(session, args.file)
_init_instance(session, args)
case 'exec':
_exec_guest_agent_command(session, args)
case 'ls':
@@ -268,13 +341,17 @@ def main(session: Session, args: argparse.Namespace) -> None:
args.password,
encrypted=args.encrypted,
)
case 'setcdrom':
_manage_cdrom(session, args)
case 'delete':
_delete_instance(session, args)
def cli() -> None: # noqa: PLR0915
def get_parser() -> argparse.ArgumentParser: # noqa: PLR0915
"""Return command line arguments parser."""
root = argparse.ArgumentParser(
prog='compute',
description='manage compute instances',
description='Manage compute instances.',
formatter_class=argparse.RawTextHelpFormatter,
)
root.add_argument(
@@ -317,12 +394,27 @@ def cli() -> None: # noqa: PLR0915
default='instance.yaml',
help='instance config [default: instance.yaml]',
)
init.add_argument(
'-s',
'--start',
action='store_true',
default=False,
help='start instance after init',
)
init.add_argument(
'-t',
'--test',
action='store_true',
default=False,
help='just print resulting instance config as JSON and exit',
)
# exec subcommand
execute = subparsers.add_parser(
'exec',
help='execute command in guest via guest agent',
description=(
'Execute command in guest via guest agent. '
'NOTE: any argument after instance name will be passed into '
'guest as shell command.'
),
@@ -463,27 +555,60 @@ def cli() -> None: # noqa: PLR0915
help='set it if password is already encrypted',
)
# setcdrom subcommand
setcdrom = subparsers.add_parser('setcdrom', help='manage CDROM devices')
setcdrom.add_argument('instance')
setcdrom.add_argument('source', help='source for CDROM')
setcdrom.add_argument(
'-d',
'--detach',
action='store_true',
default=False,
help='detach CDROM device',
)
# delete subcommand
delete = subparsers.add_parser(
'delete',
help='delete instance',
)
delete.add_argument('instance')
delete.add_argument(
'-y',
'--yes',
action='store_true',
default=False,
help='automatic yes to prompt',
)
delete.add_argument(
'--save-volumes',
action='store_true',
default=False,
help='do not delete local storage volumes',
)
return root
def cli() -> None:
"""Run arguments parser."""
root = get_parser()
args = root.parse_args()
if args.command is None:
root.print_help()
sys.exit()
log_level = args.log_level or os.getenv('CMP_LOG')
if isinstance(log_level, str) and log_level.lower() in log_levels:
logging.basicConfig(
level=logging.getLevelNamesMapping()[log_level.upper()]
)
log.debug('CLI started with args: %s', args)
connect_uri = (
args.connect
or os.getenv('CMP_LIBVIRT_URI')
or os.getenv('LIBVIRT_DEFAULT_URI')
or 'qemu:///system'
)
try:
with Session(connect_uri) as session:
main(session, args)
@@ -493,8 +618,6 @@ def cli() -> None: # noqa: PLR0915
sys.exit()
except SystemExit as e:
sys.exit(e)
except Exception as e: # noqa: BLE001
sys.exit(f'unexpected error {type(e)}: {e}')
if __name__ == '__main__':

4
compute/common.py
View File

@@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Common symbols."""

27
compute/exceptions.py
View File

@@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Exceptions."""
@@ -80,9 +80,32 @@ class InstanceNotFoundError(InstanceError):
super().__init__(f"compute instance '{msg}' not found")
class InvalidDeviceConfigError(ComputeError):
"""
Invalid device XML description.
:class:`DeviceCoonfig` instance cannot be created because
device config in libvirt XML config is not valid.
"""
def __init__(self, msg: str, xml: str):
"""Initialise InvalidDeviceConfigError."""
self.msg = f'Invalid device XML config: {msg}'
self.loc = f' {xml}'
super().__init__(f'{self.msg}\n:{self.loc}')
class InvalidDataUnitError(ValueError, ComputeError):
"""Data unit is not valid."""
def __init__(self, msg: str, units: list):
"""Initialise InvalidDataUnitError."""
super().__init__(f'{msg}, valid units are: {", ".join(units)}')
class DictMergeConflictError(ComputeError):
"""Conflict when merging dicts."""
def __init__(self, key: str):
"""Initialise DictMergeConflictError."""
super().__init__(f'Conflicting key: {key}')

4
compute/instance/__init__.py
View File

@@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
from .guest_agent import GuestAgent
from .instance import Instance, InstanceConfig

128
compute/instance/devices.py Normal file
View File

@@ -0,0 +1,128 @@
# This file is part of Compute
#
# Compute is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
# ruff: noqa: SIM211, UP007, A003
"""Virtual devices configs."""
from dataclasses import dataclass, field
from pathlib import Path
from typing import Union
from lxml import etree
from lxml.builder import E
from compute.common import DeviceConfig
from compute.exceptions import InvalidDeviceConfigError
@dataclass
class DiskDriver:
"""Disk driver description for libvirt."""
name: str
type: str
cache: str
def __call__(self):
"""Return self."""
return self
@dataclass
class DiskConfig(DeviceConfig):
"""
Disk config builder.
Generate XML config for attaching or detaching storage volumes
to compute instances.
"""
type: str
source: str | Path
target: str
is_readonly: bool = False
device: str = 'disk'
bus: str = 'virtio'
driver: DiskDriver = field(
default_factory=DiskDriver(
name='qemu',
type='qcow2',
cache='writethrough',
)
)
def to_xml(self) -> str:
"""Return XML config for libvirt."""
xml = E.disk(type=self.type, device=self.device)
xml.append(
E.driver(
name=self.driver.name,
type=self.driver.type,
cache=self.driver.cache,
)
)
if self.source and self.type == 'file':
xml.append(E.source(file=str(self.source)))
xml.append(E.target(dev=self.target, bus=self.bus))
if self.is_readonly:
xml.append(E.readonly())
return etree.tostring(xml, encoding='unicode', pretty_print=True)
@classmethod
def from_xml(cls, xml: Union[str, etree.Element]) -> 'DiskConfig':
"""
Create :class:`DiskConfig` instance from XML config.
:param xml: Disk device XML configuration as :class:`str` or lxml
:class:`etree.Element` object.
"""
if isinstance(xml, str):
xml_str = xml
xml = etree.fromstring(xml)
else:
xml_str = etree.tostring(
xml,
encoding='unicode',
pretty_print=True,
).strip()
driver = xml.find('driver')
disk_params = {
'type': xml.get('type'),
'device': xml.get('device'),
'driver': DiskDriver(
name=driver.get('name'),
type=driver.get('type'),
cache=driver.get('cache'),
),
'source': xml.find('source').get('file'),
'target': xml.find('target').get('dev'),
'bus': xml.find('target').get('bus'),
'is_readonly': False if xml.find('readonly') is None else True,
}
for param in disk_params:
if disk_params[param] is None:
msg = f"missing XML tag '{param}'"
raise InvalidDeviceConfigError(msg, xml_str)
if param == 'driver':
driver = disk_params[param]
for driver_param in [driver.name, driver.type, driver.cache]:
if driver_param is None:
msg = (
"'driver' tag must have "
"'name', 'type' and 'cache' attributes"
)
raise InvalidDeviceConfigError(msg, xml_str)
return cls(**disk_params)

4
compute/instance/guest_agent.py
View File

@@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Interacting with the QEMU Guest Agent."""

78
compute/instance/instance.py
View File

@@ -5,19 +5,20 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Manage compute instances."""
__all__ = ['Instance', 'InstanceConfig', 'InstanceInfo']
import logging
import time
from typing import NamedTuple
import libvirt
@@ -29,9 +30,9 @@ from compute.exceptions import (
GuestAgentCommandNotSupportedError,
InstanceError,
)
from compute.storage import DiskConfig
from compute.utils import units
from .devices import DiskConfig
from .guest_agent import GuestAgent
from .schemas import (
CPUEmulationMode,
@@ -282,7 +283,7 @@ class Instance:
def start(self) -> None:
"""Start defined instance."""
log.info('Starting instnce=%s', self.name)
log.info("Starting instance '%s'", self.name)
if self.is_running():
log.warning(
'Already started, nothing to do instance=%s', self.name
@@ -292,7 +293,7 @@ class Instance:
self.domain.create()
except libvirt.libvirtError as e:
raise InstanceError(
f'Cannot start instance={self.name}: {e}'
f"Cannot start instance '{self.name}': {e}"
) from e
def shutdown(self, method: str | None = None) -> None:
@@ -323,6 +324,7 @@ class Instance:
:param method: Method used to shutdown instance
"""
if not self.is_running():
log.warning('Instance is not running, nothing to do')
return
methods = {
'SOFT': libvirt.VIR_DOMAIN_SHUTDOWN_GUEST_AGENT,
@@ -339,6 +341,7 @@ class Instance:
method = method.upper()
if method not in methods:
raise ValueError(f"Unsupported shutdown method: '{method}'")
log.info("Performing instance shutdown with method '%s'", method)
try:
if method in ['SOFT', 'NORMAL']:
self.domain.shutdownFlags(flags=methods[method])
@@ -346,7 +349,7 @@ class Instance:
self.domain.destroyFlags(flags=methods[method])
except libvirt.libvirtError as e:
raise InstanceError(
f'Cannot shutdown instance={self.name} ' f'{method=}: {e}'
f"Cannot shutdown instance '{self.name}' with '{method=}': {e}"
) from e
def reboot(self) -> None:
@@ -375,7 +378,7 @@ class Instance:
self.domain.reset()
except libvirt.libvirtError as e:
raise InstanceError(
f'Cannot reset instance={self.name}: {e}'
f"Cannot reset instance '{self.name}': {e}"
) from e
def power_reset(self) -> None:
@@ -389,7 +392,13 @@ class Instance:
configuration change in libvirt and you need to restart the
instance to apply the new configuration.
"""
self.shutdown(method='NORMAL')
log.debug("Performing power reset for instance '%s'", self.name)
self.shutdown('NORMAL')
time.sleep(3)
# TODO @ge: do safe shutdown insted of this shit
if self.is_running():
self.shutdown('HARD')
time.sleep(1)
self.start()
def set_autostart(self, *, enabled: bool) -> None:
@@ -550,7 +559,9 @@ class Instance:
return
self.domain.detachDeviceFlags(device.to_xml(), flags=flags)
def get_disk(self, name: str) -> DiskConfig | None:
def get_disk(
self, name: str, *, persistent: bool = False
) -> DiskConfig | None:
"""
Return :class:`DiskConfig` by disk target name.
@@ -558,20 +569,27 @@ class Instance:
:param name: Disk name e.g. `vda`, `sda`, etc. This name may
not match the name of the disk inside the guest OS.
:param persistent: If True get only persistent volumes described
in instance XML config.
"""
xml = etree.fromstring(self.dump_xml())
xml = etree.fromstring(self.dump_xml(inactive=persistent))
child = xml.xpath(f'/domain/devices/disk/target[@dev="{name}"]')
if len(child) == 0:
return None
return DiskConfig.from_xml(child[0].getparent())
def list_disks(self) -> list[DiskConfig]:
"""Return list of attached disk devices."""
xml = etree.fromstring(self.dump_xml())
def list_disks(self, *, persistent: bool = False) -> list[DiskConfig]:
"""
Return list of attached disk devices.
:param persistent: If True list only persistent volumes described
in instance XML config.
"""
xml = etree.fromstring(self.dump_xml(inactive=persistent))
disks = xml.xpath('/domain/devices/disk')
return [DiskConfig.from_xml(disk) for disk in disks]
def detach_disk(self, name: str) -> None:
def detach_disk(self, name: str, *, live: bool = False) -> None:
"""
Detach disk device by target name.
@@ -580,15 +598,17 @@ class Instance:
:param name: Disk name e.g. `vda`, `sda`, etc. This name may
not match the name of the disk inside the guest OS.
:param live: Affect a running instance. Not supported for CDROM
devices.
"""
disk = self.get_disk(name)
disk = self.get_disk(name, persistent=live)
if disk is None:
log.warning(
"Volume with target '%s' is already detached",
name,
)
return
self.detach_device(disk, live=True)
self.detach_device(disk, live=live)
def resize_disk(
self, name: str, capacity: int, unit: units.DataUnit
@@ -601,6 +621,7 @@ class Instance:
:param capacity: New capacity.
:param unit: Capacity unit.
"""
# TODO @ge: check actual size before making changes
self.domain.blockResize(
name,
units.to_bytes(capacity, unit=unit),
@@ -619,7 +640,7 @@ class Instance:
def list_ssh_keys(self, user: str) -> list[str]:
"""
Return list of SSH keys on guest for specific user.
Return list of authorized SSH keys in guest for specific user.
:param user: Username.
"""
@@ -655,7 +676,7 @@ class Instance:
append: bool = False,
) -> None:
"""
Add SSH keys to guest for specific user.
Add authorized SSH keys to guest for specific user.
:param user: Username.
:param keys: List of authorized SSH keys.
@@ -666,7 +687,7 @@ class Instance:
qemu_ga_commands = ['guest-ssh-add-authorized-keys']
if remove and append:
raise InstanceError(
"'append' and 'remove' parameters is mutually exclusive"
"'append' and 'remove' parameters are mutually exclusive"
)
if not self.is_running():
raise InstanceError(
@@ -693,7 +714,7 @@ class Instance:
"""
Set new user password in guest OS.
This action performs by guest agent inside the guest.
This action is performed by guest agent inside the guest.
:param user: Username.
:param password: Password.
@@ -702,6 +723,7 @@ class Instance:
"""
self.guest_agent.raise_for_commands(['guest-set-user-password'])
flags = libvirt.VIR_DOMAIN_PASSWORD_ENCRYPTED if encrypted else 0
log.debug("Setting up password for user '%s'", user)
self.domain.setUserPassword(user, password, flags=flags)
def dump_xml(self, *, inactive: bool = False) -> str:
@@ -709,11 +731,19 @@ class Instance:
flags = libvirt.VIR_DOMAIN_XML_INACTIVE if inactive else 0
return self.domain.XMLDesc(flags)
def delete(self) -> None:
"""Delete instance with local disks."""
def delete(self, *, with_volumes: bool = False) -> None:
"""
Delete instance with local volumes.
:param with_volumes: If True delete local volumes with instance.
"""
self.shutdown(method='HARD')
for disk in self.list_disks():
if disk.disk_type == 'file':
disks = self.list_disks(persistent=True)
log.debug('Disks list: %s', disks)
for disk in disks:
if with_volumes and disk.type == 'file':
volume = self.connection.storageVolLookupByPath(disk.source)
log.debug('Delete volume: %s', volume.path())
volume.delete()
log.debug('Undefine instance')
self.domain.undefine()

41
compute/instance/schemas.py
View File

@@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Compute instance related objects schemas."""
@@ -19,7 +19,8 @@ import re
from enum import StrEnum
from pathlib import Path
from pydantic import validator
from pydantic import ValidationError, validator
from pydantic.error_wrappers import ErrorWrapper
from compute.common import EntityModel
from compute.utils.units import DataUnit
@@ -73,15 +74,26 @@ class VolumeCapacitySchema(EntityModel):
unit: DataUnit
class DiskDriverSchema(EntityModel):
"""Virtual disk driver model."""
name: str
type: str # noqa: A003
cache: str = 'writethrough'
class VolumeSchema(EntityModel):
"""Storage volume model."""
type: VolumeType # noqa: A003
target: str
capacity: VolumeCapacitySchema
driver: DiskDriverSchema
capacity: VolumeCapacitySchema | None
source: str | None = None
is_readonly: bool = False
is_system: bool = False
bus: str = 'virtio'
device: str = 'disk'
class NetworkInterfaceSchema(EntityModel):
@@ -118,10 +130,10 @@ class InstanceSchema(EntityModel):
@validator('name')
def _check_name(cls, value: str) -> str: # noqa: N805
if not re.match(r'^[a-z0-9_]+$', value):
if not re.match(r'^[a-z0-9_-]+$', value):
msg = (
'Name can contain only lowercase letters, numbers '
'and underscore.'
'Name can contain only lowercase letters, numbers, '
'minus sign and underscore.'
)
raise ValueError(msg)
return value
@@ -140,13 +152,22 @@ class InstanceSchema(EntityModel):
if len([v for v in volumes if v.is_system is True]) != 1:
msg = 'volumes list must contain one system volume'
raise ValueError(msg)
vol_with_source = 0
for vol in volumes:
if vol.source is None and vol.capacity is None:
raise ValidationError(
[
ErrorWrapper(
Exception(
"capacity is required if 'source' is unset"
),
loc='X.capacity',
)
],
model=VolumeSchema,
)
if vol.is_system is True and vol.is_readonly is True:
msg = 'volume marked as system cannot be readonly'
raise ValueError(msg)
if vol.source is not None:
vol_with_source += 1
return volumes
@validator('network_interfaces')

80
compute/session.py
View File

@@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Hypervisor session manager."""
@@ -31,7 +31,8 @@ from .exceptions import (
StoragePoolNotFoundError,
)
from .instance import Instance, InstanceConfig, InstanceSchema
from .storage import DiskConfig, StoragePool, VolumeConfig
from .instance.devices import DiskConfig, DiskDriver
from .storage import StoragePool, VolumeConfig
from .utils import units
@@ -164,7 +165,7 @@ class Session(AbstractContextManager):
cpu_vendor=caps.xpath(f'{hprefix}/vendor/text()')[0],
cpu_model=caps.xpath(f'{hprefix}/model/text()')[0],
cpu_features=self._cap_get_cpu_features(caps),
usable_cpus=self._cap_get_cpus(caps),
usable_cpus=self._cap_get_usable_cpus(caps),
)
def create_instance(self, **kwargs: Any) -> Instance:
@@ -208,15 +209,15 @@ class Session(AbstractContextManager):
config = InstanceConfig(data)
log.info('Define XML...')
log.info(config.to_xml())
self.connection.defineXML(config.to_xml())
try:
self.connection.defineXML(config.to_xml())
except libvirt.libvirtError as e:
raise SessionError(f'Error defining instance: {e}') from e
log.info('Getting instance...')
instance = self.get_instance(config.name)
log.info('Creating volumes...')
log.info('Start processing volumes...')
for volume in data.volumes:
log.info('Creating volume=%s', volume)
capacity = units.to_bytes(
volume.capacity.value, volume.capacity.unit
)
log.info('Processing volume=%s', volume)
log.info('Connecting to images pool...')
images_pool = self.get_storage_pool(self.IMAGES_POOL)
log.info('Connecting to volumes pool...')
@@ -226,35 +227,48 @@ class Session(AbstractContextManager):
vol_name = f'{uuid4()}.qcow2'
else:
vol_name = volume.source
vol_conf = VolumeConfig(
name=vol_name,
path=str(volumes_pool.path.joinpath(vol_name)),
capacity=capacity,
)
log.info('Volume configuration is:\n %s', vol_conf.to_xml())
if volume.is_system is True and data.image:
log.info(
"Volume is marked as 'system', start cloning image..."
)
log.info('Get image %s', data.image)
image = images_pool.get_volume(data.image)
log.info('Cloning image into volumes pool...')
vol = volumes_pool.clone_volume(image, vol_conf)
log.info(
'Resize cloned volume to specified size: %s',
capacity,
)
vol.resize(capacity, unit=units.DataUnit.BYTES)
if volume.device == 'cdrom':
log.debug('Volume %s is CDROM device', vol_name)
else:
log.info('Create volume...')
volumes_pool.create_volume(vol_conf)
capacity = units.to_bytes(
volume.capacity.value, volume.capacity.unit
)
vol_conf = VolumeConfig(
name=vol_name,
path=str(volumes_pool.path.joinpath(vol_name)),
capacity=capacity,
)
log.info('Volume configuration is:\n %s', vol_conf.to_xml())
if volume.is_system is True and data.image:
log.info(
"Volume is marked as 'system', start cloning image..."
)
log.info('Get image %s', data.image)
image = images_pool.get_volume(data.image)
log.info('Cloning image into volumes pool...')
vol = volumes_pool.clone_volume(image, vol_conf)
log.info(
'Resize cloned volume to specified size: %s',
capacity,
)
vol.resize(capacity, unit=units.DataUnit.BYTES)
else:
log.info('Create volume...')
volumes_pool.create_volume(vol_conf)
log.info('Attaching volume to instance...')
instance.attach_device(
DiskConfig(
disk_type=volume.type,
type=volume.type,
device=volume.device,
source=vol_conf.path,
target=volume.target,
readonly=volume.is_readonly,
is_readonly=volume.is_readonly,
bus=volume.bus,
driver=DiskDriver(
volume.driver.name,
volume.driver.type,
volume.driver.cache,
),
)
)
return instance

6
compute/storage/__init__.py
View File

@@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
from .pool import StoragePool
from .volume import DiskConfig, Volume, VolumeConfig
from .volume import Volume, VolumeConfig

4
compute/storage/pool.py
View File

@@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Manage storage pools."""

55
compute/storage/volume.py
View File

@@ -5,26 +5,25 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Manage storage volumes."""
from dataclasses import dataclass
from pathlib import Path
from time import time
from typing import Union
import libvirt
from lxml import etree
from lxml.builder import E
from compute.common import DeviceConfig, EntityConfig
from compute.common import EntityConfig
from compute.utils import units
@@ -64,54 +63,6 @@ class VolumeConfig(EntityConfig):
return etree.tostring(xml, encoding='unicode', pretty_print=True)
@dataclass
class DiskConfig(DeviceConfig):
"""
Disk XML config builder.
Generate XML config for attaching or detaching storage volumes
to compute instances.
"""
disk_type: str
source: str | Path
target: str
readonly: bool = False
def to_xml(self) -> str:
"""Return XML config for libvirt."""
xml = E.disk(type=self.disk_type, device='disk')
xml.append(E.driver(name='qemu', type='qcow2', cache='writethrough'))
if self.disk_type == 'file':
xml.append(E.source(file=str(self.source)))
xml.append(E.target(dev=self.target, bus='virtio'))
if self.readonly:
xml.append(E.readonly())
return etree.tostring(xml, encoding='unicode', pretty_print=True)
@classmethod
def from_xml(cls, xml: Union[str, etree.Element]) -> 'DiskConfig': # noqa: UP007
"""
Return :class:`DiskConfig` instance using existing XML config.
:param xml: Disk device XML configuration as :class:`str` or lxml
:class:`etree.Element` object.
"""
if isinstance(xml, str):
xml = etree.fromstring(xml)
disk_params = {
'disk_type': xml.get('type'),
'source': xml.find('source').get('file'),
'target': xml.find('target').get('dev'),
'readonly': False if xml.find('readonly') is None else True, # noqa: SIM211
}
for param in disk_params:
if disk_params[param] is None:
msg = f"Bad XML config: parameter '{param}' is not defined"
raise ValueError(msg)
return cls(**disk_params)
class Volume:
"""Storage volume manipulating class."""

4
compute/utils/config_loader.py
View File

@@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Configuration loader."""

77
compute/utils/dictutil.py Normal file
View File

@@ -0,0 +1,77 @@
# This file is part of Compute
#
# Compute is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Dict tools."""
from compute.exceptions import DictMergeConflictError
def merge(a: dict, b: dict, path: list[str] | None = None) -> dict:
"""
Merge `b` into `a`. Return modified `a`.
:raise: :class:`DictMergeConflictError`
"""
if path is None:
path = []
for key in b:
if key in a:
if isinstance(a[key], dict) and isinstance(b[key], dict):
merge(a[key], b[key], [*path, str(key)])
elif a[key] != b[key]:
raise DictMergeConflictError('.'.join([*path, str(key)]))
else:
a[key] = b[key]
return a
def override(a: dict, b: dict) -> dict:
"""
Override dict `a` by `b` values.
Keys that not exists in `a`, but exists in `b` will be
appended to `a`.
.. code-block:: shell-session
>>> from compute.utils import dictutil
>>> default = {
... 'bus': 'virtio',
... 'driver': {'name': 'qemu', 'type': 'qcow2'}
... }
>>> user = {
... 'bus': 'ide',
... 'target': 'vda',
... 'driver': {'type': 'raw'}
... }
>>> dictutil.override(default, user)
{'bus': 'ide', 'driver': {'name': 'qemu', 'type': 'raw'},
'target': 'vda'}
NOTE: merging dicts contained in lists is not supported.
:param a: Dict to be overwritten.
:param b: A dict whose values will be used to rewrite dict `a`.
:return: Modified `a` dict.
"""
for key in b:
if key in a:
if isinstance(a[key], dict) and isinstance(b[key], dict):
override(a[key], b[key])
else:
a[key] = b[key] # replace existing key's values
else:
a[key] = b[key]
return a

4
compute/utils/ids.py
View File

@@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Random identificators."""

13
compute/utils/units.py
View File

@@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Tools for data units convertion."""
@@ -21,7 +21,7 @@ from compute.exceptions import InvalidDataUnitError
class DataUnit(StrEnum):
"""Data units enumerated."""
"""Data units enumeration."""
BYTES = 'bytes'
KIB = 'KiB'
@@ -29,6 +29,13 @@ class DataUnit(StrEnum):
GIB = 'GiB'
TIB = 'TiB'
@classmethod
def _missing_(cls, name: str) -> 'DataUnit':
for member in cls:
if member.name.lower() == name.lower():
return member
return None
def to_bytes(value: int, unit: DataUnit = DataUnit.BYTES) -> int:
"""Convert value to bytes. See :class:`DataUnit`."""