various improvements

This commit is contained in:
ge 2023-12-03 23:25:34 +03:00
parent 0d5246e95e
commit b0fa1b7b25
36 changed files with 842 additions and 277 deletions

View File

@ -10,14 +10,17 @@ all: docs build-deb
requirements.txt: requirements.txt:
poetry export -f requirements.txt -o requirements.txt poetry export -f requirements.txt -o requirements.txt
build: format lint build: version format lint
awk '/^version/{print $$3}' pyproject.toml \
| xargs -I {} sed "s/__version__ =.*/__version__ = '{}'/" -i $(SRCDIR)/__init__.py
poetry build poetry build
build-deb: build build-deb: build
cd packaging && $(MAKE) cd packaging && $(MAKE)
version:
VERSION=$$(awk '/^version/{print $$3}' pyproject.toml); \
sed "s/__version__ =.*/__version__ = $$VERSION/" -i $(SRCDIR)/__init__.py; \
sed "s/release =.*/release = $$VERSION/" -i $(DOCS_SRCDIR)/conf.py
format: format:
poetry run isort $(SRCDIR) poetry run isort $(SRCDIR)
poetry run ruff format $(SRCDIR) poetry run ruff format $(SRCDIR)
@ -32,7 +35,8 @@ docs-versions:
poetry run sphinx-multiversion $(DOCS_SRCDIR) $(DOCS_BUILDDIR) poetry run sphinx-multiversion $(DOCS_SRCDIR) $(DOCS_BUILDDIR)
serve-docs: serve-docs:
poetry run sphinx-autobuild $(DOCS_SRCDIR) $(DOCS_BUILDDIR) poetry run sphinx-autobuild $(DOCS_SRCDIR) $(DOCS_BUILDDIR) \
--pre-build 'make clean'
clean: clean:
[ -d $(DISTDIR) ] && rm -rf $(DISTDIR) || true [ -d $(DISTDIR) ] && rm -rf $(DISTDIR) || true
@ -42,3 +46,7 @@ clean:
test-build: build-deb test-build: build-deb
scp packaging/build/compute*.deb vm:~ scp packaging/build/compute*.deb vm:~
upload-docs: docs-versions
ssh root@hitomi 'rm -rf /srv/http/nixhacks.net/hstack/*'
scp -r $(DOCS_DUILDDIR) root@hitomi:/srv/http/nixhacks.net/hstack/

View File

@ -9,23 +9,23 @@ Run `make serve-docs`. See [Development](#development) below.
## Roadmap ## Roadmap
- [x] Create instances - [x] Create instances
- [ ] CDROM - [x] CDROM
- [ ] cloud-init for provisioning instances - [ ] cloud-init for provisioning instances
- [x] Instance power management - [x] Power management
- [x] Instance pause and resume - [x] Pause and resume
- [x] vCPU hotplug - [x] vCPU hotplug
- [x] Memory hotplug - [x] Memory hotplug
- [x] Hot disk resize [not tested] - [x] Hot disk resize [not tested]
- [ ] CPU topology customization
- [x] CPU customization (emulation mode, model, vendor, features) - [x] CPU customization (emulation mode, model, vendor, features)
- [ ] CPU topology customization
- [ ] BIOS/UEFI settings - [ ] BIOS/UEFI settings
- [x] Device attaching - [x] Device attaching
- [x] Device detaching - [x] Device detaching
- [ ] GPU passthrough - [ ] GPU passthrough
- [ ] CPU guarantied resource percent support - [ ] CPU guarantied resource percent support
- [x] QEMU Guest Agent management - [x] QEMU Guest Agent management
- [ ] Instance resources usage stats - [ ] Resource usage stats
- [ ] SSH-keys management - [x] SSH-keys management
- [x] Setting user passwords in guest - [x] Setting user passwords in guest
- [x] QCOW2 disks support - [x] QCOW2 disks support
- [ ] ZVOL support - [ ] ZVOL support
@ -35,10 +35,11 @@ Run `make serve-docs`. See [Development](#development) below.
- [ ] Idempotency - [ ] Idempotency
- [ ] CLI [in progress] - [ ] CLI [in progress]
- [ ] HTTP API - [ ] HTTP API
- [ ] Instance migrations - [ ] Migrations
- [ ] Instance snapshots - [ ] Snapshots
- [ ] Instance backups - [ ] Backups
- [ ] LXC - [ ] LXC
- [ ] Attaching CDROM from sources: block, (http|https|ftp|ftps|tftp)://
## Development ## Development
@ -78,6 +79,7 @@ After installation prepare environment, run following command to start libvirtd
virsh pool-define-as $pool dir - - - - "/$pool" virsh pool-define-as $pool dir - - - - "/$pool"
virsh pool-build $pool virsh pool-build $pool
virsh pool-start $pool virsh pool-start $pool
virsh pool-autostart $pool
done done
``` ```

View File

@ -5,17 +5,17 @@
# the Free Software Foundation, either version 3 of the License, or # the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version. # (at your option) any later version.
# #
# Ansible is distributed in the hope that it will be useful, # Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Compute instances management library.""" """Compute instances management library."""
__version__ = '0.1.0-dev1' __version__ = '0.1.0-dev2'
from .instance import Instance, InstanceConfig, InstanceSchema from .instance import Instance, InstanceConfig, InstanceSchema
from .session import Session from .session import Session

View File

@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or # the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version. # (at your option) any later version.
# #
# Ansible is distributed in the hope that it will be useful, # Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Command line interface for compute module.""" """Command line interface for compute module."""

View File

@ -5,25 +5,25 @@
# the Free Software Foundation, either version 3 of the License, or # the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version. # (at your option) any later version.
# #
# Ansible is distributed in the hope that it will be useful, # Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Command line interface.""" """Command line interface."""
import argparse import argparse
import io import json
import logging import logging
import os import os
import re
import shlex import shlex
import string
import sys import sys
from collections import UserDict import uuid
from typing import Any
from uuid import uuid4
import libvirt import libvirt
import yaml import yaml
@ -31,9 +31,10 @@ from pydantic import ValidationError
from compute import __version__ from compute import __version__
from compute.exceptions import ComputeError, GuestAgentTimeoutError from compute.exceptions import ComputeError, GuestAgentTimeoutError
from compute.instance import GuestAgent from compute.instance import GuestAgent, Instance, InstanceSchema
from compute.instance.devices import DiskConfig, DiskDriver
from compute.session import Session from compute.session import Session
from compute.utils import ids from compute.utils import dictutil, ids
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -128,78 +129,77 @@ def _exec_guest_agent_command(
sys.exit(output.exitcode) sys.exit(output.exitcode)
class _NotPresent: def _init_instance(session: Session, args: argparse.Namespace) -> None:
"""
Type for representing non-existent dictionary keys.
See :class:`_FillableDict`.
"""
class _FillableDict(UserDict):
"""Use :method:`fill` to add key if not present."""
def __init__(self, data: dict):
self.data = data
def fill(self, key: str, value: Any) -> None: # noqa: ANN401
if self.data.get(key, _NotPresent) is _NotPresent:
self.data[key] = value
def _merge_dicts(a: dict, b: dict, path: list[str] | None = None) -> dict:
"""Merge `b` into `a`. Return modified `a`."""
if path is None:
path = []
for key in b:
if key in a:
if isinstance(a[key], dict) and isinstance(b[key], dict):
_merge_dicts(a[key], b[key], [path + str(key)])
elif a[key] == b[key]:
pass # same leaf value
else:
a[key] = b[key] # replace existing key's values
else:
a[key] = b[key]
return a
def _create_instance(session: Session, file: io.TextIOWrapper) -> None:
try: try:
data = _FillableDict(yaml.load(file.read(), Loader=yaml.SafeLoader)) data = yaml.load(args.file.read(), Loader=yaml.SafeLoader)
log.debug('Read from file: %s', data) log.debug('Read from file: %s', data)
except yaml.YAMLError as e: except yaml.YAMLError as e:
sys.exit(f'error: cannot parse YAML: {e}') sys.exit(f'error: cannot parse YAML: {e}')
capabilities = session.get_capabilities() capabilities = session.get_capabilities()
node_info = session.get_node_info() node_info = session.get_node_info()
base_instance_config = {
data.fill('name', uuid4().hex) 'name': str(uuid.uuid4()),
data.fill('title', None) 'title': None,
data.fill('description', None) 'description': None,
data.fill('arch', capabilities.arch) 'arch': capabilities.arch,
data.fill('machine', capabilities.machine) 'machine': capabilities.machine,
data.fill('emulator', capabilities.emulator) 'emulator': capabilities.emulator,
data.fill('max_vcpus', node_info.cpus) 'max_vcpus': node_info.cpus,
data.fill('max_memory', node_info.memory) 'max_memory': node_info.memory,
data.fill('cpu', {}) 'cpu': {
cpu = { 'emulation_mode': 'host-passthrough',
'emulation_mode': 'host-passthrough', 'model': None,
'model': None, 'vendor': None,
'vendor': None, 'topology': None,
'topology': None, 'features': None,
'features': None, },
'network_interfaces': [
{
'source': 'default',
'mac': ids.random_mac(),
},
],
'boot': {'order': ['cdrom', 'hd']},
} }
data['cpu'] = _merge_dicts(data['cpu'], cpu) data = dictutil.override(base_instance_config, data)
data.fill( volumes = []
'network_interfaces', for volume in data['volumes']:
[{'source': 'default', 'mac': ids.random_mac()}], base_disk_config = {
) 'bus': 'virtio',
data.fill('boot', {'order': ['cdrom', 'hd']}) 'is_readonly': False,
'driver': {
'name': 'qemu',
'type': 'qcow2',
'cache': 'writethrough',
},
}
base_cdrom_config = {
'bus': 'ide',
'target': 'hda',
'is_readonly': True,
'driver': {
'name': 'qemu',
'type': 'raw',
'cache': 'writethrough',
},
}
if volume.get('device') is None:
volume['device'] = 'disk'
if volume['device'] == 'disk':
volumes.append(dictutil.override(base_disk_config, volume))
if volume['device'] == 'cdrom':
volumes.append(dictutil.override(base_cdrom_config, volume))
data['volumes'] = volumes
try: try:
log.debug('Input data: %s', data) log.debug('Input data: %s', data)
session.create_instance(**data) if args.test:
_ = InstanceSchema(**data)
print(json.dumps(dict(data), indent=4, sort_keys=True))
sys.exit()
instance = session.create_instance(**data)
print(f'initialised: {instance.name}')
if args.start:
instance.start()
except ValidationError as e: except ValidationError as e:
for error in e.errors(): for error in e.errors():
fields = '.'.join([str(lc) for lc in error['loc']]) fields = '.'.join([str(lc) for lc in error['loc']])
@ -223,11 +223,84 @@ def _shutdown_instance(session: Session, args: argparse.Namespace) -> None:
instance.shutdown(method) instance.shutdown(method)
def _confirm(message: str, *, default: bool | None = None) -> None:
while True:
match default:
case True:
prompt = 'default: yes'
case False:
prompt = 'default: no'
case _:
prompt = 'no default'
try:
answer = input(f'{message} ({prompt}) ')
except KeyboardInterrupt:
sys.exit('aborted')
if not answer and isinstance(default, bool):
return default
if re.match(r'^y(es)?$', answer, re.I):
return True
if re.match(r'^no?$', answer, re.I):
return False
print("Please respond 'yes' or 'no'")
def _delete_instance(session: Session, args: argparse.Namespace) -> None:
if args.yes is True or _confirm(
'this action is irreversible, continue?',
default=False,
):
instance = session.get_instance(args.instance)
if args.save_volumes is False:
instance.delete(with_volumes=True)
else:
instance.delete()
else:
print('aborted')
sys.exit()
def _get_disk_target(instance: Instance, prefix: str = 'hd') -> str:
disks_live = instance.list_disks(persistent=False)
disks_inactive = instance.list_disks(persistent=True)
disks = [d for d in disks_inactive if d not in disks_live]
devs = [d.target[-1] for d in disks if d.target.startswith(prefix)]
return prefix + [x for x in string.ascii_lowercase if x not in devs][0] # noqa: RUF015
def _manage_cdrom(session: Session, args: argparse.Namespace) -> None:
instance = session.get_instance(args.instance)
if args.detach:
for disk in instance.list_disks(persistent=True):
if disk.device == 'cdrom' and disk.source == args.source:
instance.detach_disk(disk.target, live=False)
print(
f"disk '{disk.target}' detached, "
'perform power reset to apply changes'
)
return
target = _get_disk_target(instance, 'hd')
cdrom = DiskConfig(
type='file',
device='cdrom',
source=args.source,
target=target,
is_readonly=True,
bus='ide',
driver=DiskDriver('qemu', 'raw', 'writethrough'),
)
instance.attach_device(cdrom, live=False)
print(
f"CDROM attached as disk '{target}', "
'perform power reset to apply changes'
)
def main(session: Session, args: argparse.Namespace) -> None: def main(session: Session, args: argparse.Namespace) -> None:
"""Perform actions.""" """Perform actions."""
match args.command: match args.command:
case 'init': case 'init':
_create_instance(session, args.file) _init_instance(session, args)
case 'exec': case 'exec':
_exec_guest_agent_command(session, args) _exec_guest_agent_command(session, args)
case 'ls': case 'ls':
@ -268,13 +341,17 @@ def main(session: Session, args: argparse.Namespace) -> None:
args.password, args.password,
encrypted=args.encrypted, encrypted=args.encrypted,
) )
case 'setcdrom':
_manage_cdrom(session, args)
case 'delete':
_delete_instance(session, args)
def cli() -> None: # noqa: PLR0915 def get_parser() -> argparse.ArgumentParser: # noqa: PLR0915
"""Return command line arguments parser.""" """Return command line arguments parser."""
root = argparse.ArgumentParser( root = argparse.ArgumentParser(
prog='compute', prog='compute',
description='manage compute instances', description='Manage compute instances.',
formatter_class=argparse.RawTextHelpFormatter, formatter_class=argparse.RawTextHelpFormatter,
) )
root.add_argument( root.add_argument(
@ -317,12 +394,27 @@ def cli() -> None: # noqa: PLR0915
default='instance.yaml', default='instance.yaml',
help='instance config [default: instance.yaml]', help='instance config [default: instance.yaml]',
) )
init.add_argument(
'-s',
'--start',
action='store_true',
default=False,
help='start instance after init',
)
init.add_argument(
'-t',
'--test',
action='store_true',
default=False,
help='just print resulting instance config as JSON and exit',
)
# exec subcommand # exec subcommand
execute = subparsers.add_parser( execute = subparsers.add_parser(
'exec', 'exec',
help='execute command in guest via guest agent', help='execute command in guest via guest agent',
description=( description=(
'Execute command in guest via guest agent. '
'NOTE: any argument after instance name will be passed into ' 'NOTE: any argument after instance name will be passed into '
'guest as shell command.' 'guest as shell command.'
), ),
@ -463,27 +555,60 @@ def cli() -> None: # noqa: PLR0915
help='set it if password is already encrypted', help='set it if password is already encrypted',
) )
# setcdrom subcommand
setcdrom = subparsers.add_parser('setcdrom', help='manage CDROM devices')
setcdrom.add_argument('instance')
setcdrom.add_argument('source', help='source for CDROM')
setcdrom.add_argument(
'-d',
'--detach',
action='store_true',
default=False,
help='detach CDROM device',
)
# delete subcommand
delete = subparsers.add_parser(
'delete',
help='delete instance',
)
delete.add_argument('instance')
delete.add_argument(
'-y',
'--yes',
action='store_true',
default=False,
help='automatic yes to prompt',
)
delete.add_argument(
'--save-volumes',
action='store_true',
default=False,
help='do not delete local storage volumes',
)
return root
def cli() -> None:
"""Run arguments parser."""
root = get_parser()
args = root.parse_args() args = root.parse_args()
if args.command is None: if args.command is None:
root.print_help() root.print_help()
sys.exit() sys.exit()
log_level = args.log_level or os.getenv('CMP_LOG') log_level = args.log_level or os.getenv('CMP_LOG')
if isinstance(log_level, str) and log_level.lower() in log_levels: if isinstance(log_level, str) and log_level.lower() in log_levels:
logging.basicConfig( logging.basicConfig(
level=logging.getLevelNamesMapping()[log_level.upper()] level=logging.getLevelNamesMapping()[log_level.upper()]
) )
log.debug('CLI started with args: %s', args) log.debug('CLI started with args: %s', args)
connect_uri = ( connect_uri = (
args.connect args.connect
or os.getenv('CMP_LIBVIRT_URI') or os.getenv('CMP_LIBVIRT_URI')
or os.getenv('LIBVIRT_DEFAULT_URI') or os.getenv('LIBVIRT_DEFAULT_URI')
or 'qemu:///system' or 'qemu:///system'
) )
try: try:
with Session(connect_uri) as session: with Session(connect_uri) as session:
main(session, args) main(session, args)
@ -493,8 +618,6 @@ def cli() -> None: # noqa: PLR0915
sys.exit() sys.exit()
except SystemExit as e: except SystemExit as e:
sys.exit(e) sys.exit(e)
except Exception as e: # noqa: BLE001
sys.exit(f'unexpected error {type(e)}: {e}')
if __name__ == '__main__': if __name__ == '__main__':

View File

@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or # the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version. # (at your option) any later version.
# #
# Ansible is distributed in the hope that it will be useful, # Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Common symbols.""" """Common symbols."""

View File

@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or # the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version. # (at your option) any later version.
# #
# Ansible is distributed in the hope that it will be useful, # Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Exceptions.""" """Exceptions."""
@ -80,9 +80,32 @@ class InstanceNotFoundError(InstanceError):
super().__init__(f"compute instance '{msg}' not found") super().__init__(f"compute instance '{msg}' not found")
class InvalidDeviceConfigError(ComputeError):
"""
Invalid device XML description.
:class:`DeviceCoonfig` instance cannot be created because
device config in libvirt XML config is not valid.
"""
def __init__(self, msg: str, xml: str):
"""Initialise InvalidDeviceConfigError."""
self.msg = f'Invalid device XML config: {msg}'
self.loc = f' {xml}'
super().__init__(f'{self.msg}\n:{self.loc}')
class InvalidDataUnitError(ValueError, ComputeError): class InvalidDataUnitError(ValueError, ComputeError):
"""Data unit is not valid.""" """Data unit is not valid."""
def __init__(self, msg: str, units: list): def __init__(self, msg: str, units: list):
"""Initialise InvalidDataUnitError.""" """Initialise InvalidDataUnitError."""
super().__init__(f'{msg}, valid units are: {", ".join(units)}') super().__init__(f'{msg}, valid units are: {", ".join(units)}')
class DictMergeConflictError(ComputeError):
"""Conflict when merging dicts."""
def __init__(self, key: str):
"""Initialise DictMergeConflictError."""
super().__init__(f'Conflicting key: {key}')

View File

@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or # the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version. # (at your option) any later version.
# #
# Ansible is distributed in the hope that it will be useful, # Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Compute. If not, see <http://www.gnu.org/licenses/>.
from .guest_agent import GuestAgent from .guest_agent import GuestAgent
from .instance import Instance, InstanceConfig from .instance import Instance, InstanceConfig

128
compute/instance/devices.py Normal file
View File

@ -0,0 +1,128 @@
# This file is part of Compute
#
# Compute is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
# ruff: noqa: SIM211, UP007, A003
"""Virtual devices configs."""
from dataclasses import dataclass, field
from pathlib import Path
from typing import Union
from lxml import etree
from lxml.builder import E
from compute.common import DeviceConfig
from compute.exceptions import InvalidDeviceConfigError
@dataclass
class DiskDriver:
"""Disk driver description for libvirt."""
name: str
type: str
cache: str
def __call__(self):
"""Return self."""
return self
@dataclass
class DiskConfig(DeviceConfig):
"""
Disk config builder.
Generate XML config for attaching or detaching storage volumes
to compute instances.
"""
type: str
source: str | Path
target: str
is_readonly: bool = False
device: str = 'disk'
bus: str = 'virtio'
driver: DiskDriver = field(
default_factory=DiskDriver(
name='qemu',
type='qcow2',
cache='writethrough',
)
)
def to_xml(self) -> str:
"""Return XML config for libvirt."""
xml = E.disk(type=self.type, device=self.device)
xml.append(
E.driver(
name=self.driver.name,
type=self.driver.type,
cache=self.driver.cache,
)
)
if self.source and self.type == 'file':
xml.append(E.source(file=str(self.source)))
xml.append(E.target(dev=self.target, bus=self.bus))
if self.is_readonly:
xml.append(E.readonly())
return etree.tostring(xml, encoding='unicode', pretty_print=True)
@classmethod
def from_xml(cls, xml: Union[str, etree.Element]) -> 'DiskConfig':
"""
Create :class:`DiskConfig` instance from XML config.
:param xml: Disk device XML configuration as :class:`str` or lxml
:class:`etree.Element` object.
"""
if isinstance(xml, str):
xml_str = xml
xml = etree.fromstring(xml)
else:
xml_str = etree.tostring(
xml,
encoding='unicode',
pretty_print=True,
).strip()
driver = xml.find('driver')
disk_params = {
'type': xml.get('type'),
'device': xml.get('device'),
'driver': DiskDriver(
name=driver.get('name'),
type=driver.get('type'),
cache=driver.get('cache'),
),
'source': xml.find('source').get('file'),
'target': xml.find('target').get('dev'),
'bus': xml.find('target').get('bus'),
'is_readonly': False if xml.find('readonly') is None else True,
}
for param in disk_params:
if disk_params[param] is None:
msg = f"missing XML tag '{param}'"
raise InvalidDeviceConfigError(msg, xml_str)
if param == 'driver':
driver = disk_params[param]
for driver_param in [driver.name, driver.type, driver.cache]:
if driver_param is None:
msg = (
"'driver' tag must have "
"'name', 'type' and 'cache' attributes"
)
raise InvalidDeviceConfigError(msg, xml_str)
return cls(**disk_params)

View File

@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or # the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version. # (at your option) any later version.
# #
# Ansible is distributed in the hope that it will be useful, # Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Interacting with the QEMU Guest Agent.""" """Interacting with the QEMU Guest Agent."""

View File

@ -5,19 +5,20 @@
# the Free Software Foundation, either version 3 of the License, or # the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version. # (at your option) any later version.
# #
# Ansible is distributed in the hope that it will be useful, # Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Manage compute instances.""" """Manage compute instances."""
__all__ = ['Instance', 'InstanceConfig', 'InstanceInfo'] __all__ = ['Instance', 'InstanceConfig', 'InstanceInfo']
import logging import logging
import time
from typing import NamedTuple from typing import NamedTuple
import libvirt import libvirt
@ -29,9 +30,9 @@ from compute.exceptions import (
GuestAgentCommandNotSupportedError, GuestAgentCommandNotSupportedError,
InstanceError, InstanceError,
) )
from compute.storage import DiskConfig
from compute.utils import units from compute.utils import units
from .devices import DiskConfig
from .guest_agent import GuestAgent from .guest_agent import GuestAgent
from .schemas import ( from .schemas import (
CPUEmulationMode, CPUEmulationMode,
@ -282,7 +283,7 @@ class Instance:
def start(self) -> None: def start(self) -> None:
"""Start defined instance.""" """Start defined instance."""
log.info('Starting instnce=%s', self.name) log.info("Starting instance '%s'", self.name)
if self.is_running(): if self.is_running():
log.warning( log.warning(
'Already started, nothing to do instance=%s', self.name 'Already started, nothing to do instance=%s', self.name
@ -292,7 +293,7 @@ class Instance:
self.domain.create() self.domain.create()
except libvirt.libvirtError as e: except libvirt.libvirtError as e:
raise InstanceError( raise InstanceError(
f'Cannot start instance={self.name}: {e}' f"Cannot start instance '{self.name}': {e}"
) from e ) from e
def shutdown(self, method: str | None = None) -> None: def shutdown(self, method: str | None = None) -> None:
@ -323,6 +324,7 @@ class Instance:
:param method: Method used to shutdown instance :param method: Method used to shutdown instance
""" """
if not self.is_running(): if not self.is_running():
log.warning('Instance is not running, nothing to do')
return return
methods = { methods = {
'SOFT': libvirt.VIR_DOMAIN_SHUTDOWN_GUEST_AGENT, 'SOFT': libvirt.VIR_DOMAIN_SHUTDOWN_GUEST_AGENT,
@ -339,6 +341,7 @@ class Instance:
method = method.upper() method = method.upper()
if method not in methods: if method not in methods:
raise ValueError(f"Unsupported shutdown method: '{method}'") raise ValueError(f"Unsupported shutdown method: '{method}'")
log.info("Performing instance shutdown with method '%s'", method)
try: try:
if method in ['SOFT', 'NORMAL']: if method in ['SOFT', 'NORMAL']:
self.domain.shutdownFlags(flags=methods[method]) self.domain.shutdownFlags(flags=methods[method])
@ -346,7 +349,7 @@ class Instance:
self.domain.destroyFlags(flags=methods[method]) self.domain.destroyFlags(flags=methods[method])
except libvirt.libvirtError as e: except libvirt.libvirtError as e:
raise InstanceError( raise InstanceError(
f'Cannot shutdown instance={self.name} ' f'{method=}: {e}' f"Cannot shutdown instance '{self.name}' with '{method=}': {e}"
) from e ) from e
def reboot(self) -> None: def reboot(self) -> None:
@ -375,7 +378,7 @@ class Instance:
self.domain.reset() self.domain.reset()
except libvirt.libvirtError as e: except libvirt.libvirtError as e:
raise InstanceError( raise InstanceError(
f'Cannot reset instance={self.name}: {e}' f"Cannot reset instance '{self.name}': {e}"
) from e ) from e
def power_reset(self) -> None: def power_reset(self) -> None:
@ -389,7 +392,13 @@ class Instance:
configuration change in libvirt and you need to restart the configuration change in libvirt and you need to restart the
instance to apply the new configuration. instance to apply the new configuration.
""" """
self.shutdown(method='NORMAL') log.debug("Performing power reset for instance '%s'", self.name)
self.shutdown('NORMAL')
time.sleep(3)
# TODO @ge: do safe shutdown insted of this shit
if self.is_running():
self.shutdown('HARD')
time.sleep(1)
self.start() self.start()
def set_autostart(self, *, enabled: bool) -> None: def set_autostart(self, *, enabled: bool) -> None:
@ -550,7 +559,9 @@ class Instance:
return return
self.domain.detachDeviceFlags(device.to_xml(), flags=flags) self.domain.detachDeviceFlags(device.to_xml(), flags=flags)
def get_disk(self, name: str) -> DiskConfig | None: def get_disk(
self, name: str, *, persistent: bool = False
) -> DiskConfig | None:
""" """
Return :class:`DiskConfig` by disk target name. Return :class:`DiskConfig` by disk target name.
@ -558,20 +569,27 @@ class Instance:
:param name: Disk name e.g. `vda`, `sda`, etc. This name may :param name: Disk name e.g. `vda`, `sda`, etc. This name may
not match the name of the disk inside the guest OS. not match the name of the disk inside the guest OS.
:param persistent: If True get only persistent volumes described
in instance XML config.
""" """
xml = etree.fromstring(self.dump_xml()) xml = etree.fromstring(self.dump_xml(inactive=persistent))
child = xml.xpath(f'/domain/devices/disk/target[@dev="{name}"]') child = xml.xpath(f'/domain/devices/disk/target[@dev="{name}"]')
if len(child) == 0: if len(child) == 0:
return None return None
return DiskConfig.from_xml(child[0].getparent()) return DiskConfig.from_xml(child[0].getparent())
def list_disks(self) -> list[DiskConfig]: def list_disks(self, *, persistent: bool = False) -> list[DiskConfig]:
"""Return list of attached disk devices.""" """
xml = etree.fromstring(self.dump_xml()) Return list of attached disk devices.
:param persistent: If True list only persistent volumes described
in instance XML config.
"""
xml = etree.fromstring(self.dump_xml(inactive=persistent))
disks = xml.xpath('/domain/devices/disk') disks = xml.xpath('/domain/devices/disk')
return [DiskConfig.from_xml(disk) for disk in disks] return [DiskConfig.from_xml(disk) for disk in disks]
def detach_disk(self, name: str) -> None: def detach_disk(self, name: str, *, live: bool = False) -> None:
""" """
Detach disk device by target name. Detach disk device by target name.
@ -580,15 +598,17 @@ class Instance:
:param name: Disk name e.g. `vda`, `sda`, etc. This name may :param name: Disk name e.g. `vda`, `sda`, etc. This name may
not match the name of the disk inside the guest OS. not match the name of the disk inside the guest OS.
:param live: Affect a running instance. Not supported for CDROM
devices.
""" """
disk = self.get_disk(name) disk = self.get_disk(name, persistent=live)
if disk is None: if disk is None:
log.warning( log.warning(
"Volume with target '%s' is already detached", "Volume with target '%s' is already detached",
name, name,
) )
return return
self.detach_device(disk, live=True) self.detach_device(disk, live=live)
def resize_disk( def resize_disk(
self, name: str, capacity: int, unit: units.DataUnit self, name: str, capacity: int, unit: units.DataUnit
@ -601,6 +621,7 @@ class Instance:
:param capacity: New capacity. :param capacity: New capacity.
:param unit: Capacity unit. :param unit: Capacity unit.
""" """
# TODO @ge: check actual size before making changes
self.domain.blockResize( self.domain.blockResize(
name, name,
units.to_bytes(capacity, unit=unit), units.to_bytes(capacity, unit=unit),
@ -619,7 +640,7 @@ class Instance:
def list_ssh_keys(self, user: str) -> list[str]: def list_ssh_keys(self, user: str) -> list[str]:
""" """
Return list of SSH keys on guest for specific user. Return list of authorized SSH keys in guest for specific user.
:param user: Username. :param user: Username.
""" """
@ -655,7 +676,7 @@ class Instance:
append: bool = False, append: bool = False,
) -> None: ) -> None:
""" """
Add SSH keys to guest for specific user. Add authorized SSH keys to guest for specific user.
:param user: Username. :param user: Username.
:param keys: List of authorized SSH keys. :param keys: List of authorized SSH keys.
@ -666,7 +687,7 @@ class Instance:
qemu_ga_commands = ['guest-ssh-add-authorized-keys'] qemu_ga_commands = ['guest-ssh-add-authorized-keys']
if remove and append: if remove and append:
raise InstanceError( raise InstanceError(
"'append' and 'remove' parameters is mutually exclusive" "'append' and 'remove' parameters are mutually exclusive"
) )
if not self.is_running(): if not self.is_running():
raise InstanceError( raise InstanceError(
@ -693,7 +714,7 @@ class Instance:
""" """
Set new user password in guest OS. Set new user password in guest OS.
This action performs by guest agent inside the guest. This action is performed by guest agent inside the guest.
:param user: Username. :param user: Username.
:param password: Password. :param password: Password.
@ -702,6 +723,7 @@ class Instance:
""" """
self.guest_agent.raise_for_commands(['guest-set-user-password']) self.guest_agent.raise_for_commands(['guest-set-user-password'])
flags = libvirt.VIR_DOMAIN_PASSWORD_ENCRYPTED if encrypted else 0 flags = libvirt.VIR_DOMAIN_PASSWORD_ENCRYPTED if encrypted else 0
log.debug("Setting up password for user '%s'", user)
self.domain.setUserPassword(user, password, flags=flags) self.domain.setUserPassword(user, password, flags=flags)
def dump_xml(self, *, inactive: bool = False) -> str: def dump_xml(self, *, inactive: bool = False) -> str:
@ -709,11 +731,19 @@ class Instance:
flags = libvirt.VIR_DOMAIN_XML_INACTIVE if inactive else 0 flags = libvirt.VIR_DOMAIN_XML_INACTIVE if inactive else 0
return self.domain.XMLDesc(flags) return self.domain.XMLDesc(flags)
def delete(self) -> None: def delete(self, *, with_volumes: bool = False) -> None:
"""Delete instance with local disks.""" """
Delete instance with local volumes.
:param with_volumes: If True delete local volumes with instance.
"""
self.shutdown(method='HARD') self.shutdown(method='HARD')
for disk in self.list_disks(): disks = self.list_disks(persistent=True)
if disk.disk_type == 'file': log.debug('Disks list: %s', disks)
for disk in disks:
if with_volumes and disk.type == 'file':
volume = self.connection.storageVolLookupByPath(disk.source) volume = self.connection.storageVolLookupByPath(disk.source)
log.debug('Delete volume: %s', volume.path())
volume.delete() volume.delete()
log.debug('Undefine instance')
self.domain.undefine() self.domain.undefine()

View File

@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or # the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version. # (at your option) any later version.
# #
# Ansible is distributed in the hope that it will be useful, # Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Compute instance related objects schemas.""" """Compute instance related objects schemas."""
@ -19,7 +19,8 @@ import re
from enum import StrEnum from enum import StrEnum
from pathlib import Path from pathlib import Path
from pydantic import validator from pydantic import ValidationError, validator
from pydantic.error_wrappers import ErrorWrapper
from compute.common import EntityModel from compute.common import EntityModel
from compute.utils.units import DataUnit from compute.utils.units import DataUnit
@ -73,15 +74,26 @@ class VolumeCapacitySchema(EntityModel):
unit: DataUnit unit: DataUnit
class DiskDriverSchema(EntityModel):
"""Virtual disk driver model."""
name: str
type: str # noqa: A003
cache: str = 'writethrough'
class VolumeSchema(EntityModel): class VolumeSchema(EntityModel):
"""Storage volume model.""" """Storage volume model."""
type: VolumeType # noqa: A003 type: VolumeType # noqa: A003
target: str target: str
capacity: VolumeCapacitySchema driver: DiskDriverSchema
capacity: VolumeCapacitySchema | None
source: str | None = None source: str | None = None
is_readonly: bool = False is_readonly: bool = False
is_system: bool = False is_system: bool = False
bus: str = 'virtio'
device: str = 'disk'
class NetworkInterfaceSchema(EntityModel): class NetworkInterfaceSchema(EntityModel):
@ -118,10 +130,10 @@ class InstanceSchema(EntityModel):
@validator('name') @validator('name')
def _check_name(cls, value: str) -> str: # noqa: N805 def _check_name(cls, value: str) -> str: # noqa: N805
if not re.match(r'^[a-z0-9_]+$', value): if not re.match(r'^[a-z0-9_-]+$', value):
msg = ( msg = (
'Name can contain only lowercase letters, numbers ' 'Name can contain only lowercase letters, numbers, '
'and underscore.' 'minus sign and underscore.'
) )
raise ValueError(msg) raise ValueError(msg)
return value return value
@ -140,13 +152,22 @@ class InstanceSchema(EntityModel):
if len([v for v in volumes if v.is_system is True]) != 1: if len([v for v in volumes if v.is_system is True]) != 1:
msg = 'volumes list must contain one system volume' msg = 'volumes list must contain one system volume'
raise ValueError(msg) raise ValueError(msg)
vol_with_source = 0
for vol in volumes: for vol in volumes:
if vol.source is None and vol.capacity is None:
raise ValidationError(
[
ErrorWrapper(
Exception(
"capacity is required if 'source' is unset"
),
loc='X.capacity',
)
],
model=VolumeSchema,
)
if vol.is_system is True and vol.is_readonly is True: if vol.is_system is True and vol.is_readonly is True:
msg = 'volume marked as system cannot be readonly' msg = 'volume marked as system cannot be readonly'
raise ValueError(msg) raise ValueError(msg)
if vol.source is not None:
vol_with_source += 1
return volumes return volumes
@validator('network_interfaces') @validator('network_interfaces')

View File

@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or # the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version. # (at your option) any later version.
# #
# Ansible is distributed in the hope that it will be useful, # Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Hypervisor session manager.""" """Hypervisor session manager."""
@ -31,7 +31,8 @@ from .exceptions import (
StoragePoolNotFoundError, StoragePoolNotFoundError,
) )
from .instance import Instance, InstanceConfig, InstanceSchema from .instance import Instance, InstanceConfig, InstanceSchema
from .storage import DiskConfig, StoragePool, VolumeConfig from .instance.devices import DiskConfig, DiskDriver
from .storage import StoragePool, VolumeConfig
from .utils import units from .utils import units
@ -164,7 +165,7 @@ class Session(AbstractContextManager):
cpu_vendor=caps.xpath(f'{hprefix}/vendor/text()')[0], cpu_vendor=caps.xpath(f'{hprefix}/vendor/text()')[0],
cpu_model=caps.xpath(f'{hprefix}/model/text()')[0], cpu_model=caps.xpath(f'{hprefix}/model/text()')[0],
cpu_features=self._cap_get_cpu_features(caps), cpu_features=self._cap_get_cpu_features(caps),
usable_cpus=self._cap_get_cpus(caps), usable_cpus=self._cap_get_usable_cpus(caps),
) )
def create_instance(self, **kwargs: Any) -> Instance: def create_instance(self, **kwargs: Any) -> Instance:
@ -208,15 +209,15 @@ class Session(AbstractContextManager):
config = InstanceConfig(data) config = InstanceConfig(data)
log.info('Define XML...') log.info('Define XML...')
log.info(config.to_xml()) log.info(config.to_xml())
self.connection.defineXML(config.to_xml()) try:
self.connection.defineXML(config.to_xml())
except libvirt.libvirtError as e:
raise SessionError(f'Error defining instance: {e}') from e
log.info('Getting instance...') log.info('Getting instance...')
instance = self.get_instance(config.name) instance = self.get_instance(config.name)
log.info('Creating volumes...') log.info('Start processing volumes...')
for volume in data.volumes: for volume in data.volumes:
log.info('Creating volume=%s', volume) log.info('Processing volume=%s', volume)
capacity = units.to_bytes(
volume.capacity.value, volume.capacity.unit
)
log.info('Connecting to images pool...') log.info('Connecting to images pool...')
images_pool = self.get_storage_pool(self.IMAGES_POOL) images_pool = self.get_storage_pool(self.IMAGES_POOL)
log.info('Connecting to volumes pool...') log.info('Connecting to volumes pool...')
@ -226,35 +227,48 @@ class Session(AbstractContextManager):
vol_name = f'{uuid4()}.qcow2' vol_name = f'{uuid4()}.qcow2'
else: else:
vol_name = volume.source vol_name = volume.source
vol_conf = VolumeConfig( if volume.device == 'cdrom':
name=vol_name, log.debug('Volume %s is CDROM device', vol_name)
path=str(volumes_pool.path.joinpath(vol_name)),
capacity=capacity,
)
log.info('Volume configuration is:\n %s', vol_conf.to_xml())
if volume.is_system is True and data.image:
log.info(
"Volume is marked as 'system', start cloning image..."
)
log.info('Get image %s', data.image)
image = images_pool.get_volume(data.image)
log.info('Cloning image into volumes pool...')
vol = volumes_pool.clone_volume(image, vol_conf)
log.info(
'Resize cloned volume to specified size: %s',
capacity,
)
vol.resize(capacity, unit=units.DataUnit.BYTES)
else: else:
log.info('Create volume...') capacity = units.to_bytes(
volumes_pool.create_volume(vol_conf) volume.capacity.value, volume.capacity.unit
)
vol_conf = VolumeConfig(
name=vol_name,
path=str(volumes_pool.path.joinpath(vol_name)),
capacity=capacity,
)
log.info('Volume configuration is:\n %s', vol_conf.to_xml())
if volume.is_system is True and data.image:
log.info(
"Volume is marked as 'system', start cloning image..."
)
log.info('Get image %s', data.image)
image = images_pool.get_volume(data.image)
log.info('Cloning image into volumes pool...')
vol = volumes_pool.clone_volume(image, vol_conf)
log.info(
'Resize cloned volume to specified size: %s',
capacity,
)
vol.resize(capacity, unit=units.DataUnit.BYTES)
else:
log.info('Create volume...')
volumes_pool.create_volume(vol_conf)
log.info('Attaching volume to instance...') log.info('Attaching volume to instance...')
instance.attach_device( instance.attach_device(
DiskConfig( DiskConfig(
disk_type=volume.type, type=volume.type,
device=volume.device,
source=vol_conf.path, source=vol_conf.path,
target=volume.target, target=volume.target,
readonly=volume.is_readonly, is_readonly=volume.is_readonly,
bus=volume.bus,
driver=DiskDriver(
volume.driver.name,
volume.driver.type,
volume.driver.cache,
),
) )
) )
return instance return instance

View File

@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or # the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version. # (at your option) any later version.
# #
# Ansible is distributed in the hope that it will be useful, # Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Compute. If not, see <http://www.gnu.org/licenses/>.
from .pool import StoragePool from .pool import StoragePool
from .volume import DiskConfig, Volume, VolumeConfig from .volume import Volume, VolumeConfig

View File

@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or # the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version. # (at your option) any later version.
# #
# Ansible is distributed in the hope that it will be useful, # Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Manage storage pools.""" """Manage storage pools."""

View File

@ -5,26 +5,25 @@
# the Free Software Foundation, either version 3 of the License, or # the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version. # (at your option) any later version.
# #
# Ansible is distributed in the hope that it will be useful, # Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Manage storage volumes.""" """Manage storage volumes."""
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from time import time from time import time
from typing import Union
import libvirt import libvirt
from lxml import etree from lxml import etree
from lxml.builder import E from lxml.builder import E
from compute.common import DeviceConfig, EntityConfig from compute.common import EntityConfig
from compute.utils import units from compute.utils import units
@ -64,54 +63,6 @@ class VolumeConfig(EntityConfig):
return etree.tostring(xml, encoding='unicode', pretty_print=True) return etree.tostring(xml, encoding='unicode', pretty_print=True)
@dataclass
class DiskConfig(DeviceConfig):
"""
Disk XML config builder.
Generate XML config for attaching or detaching storage volumes
to compute instances.
"""
disk_type: str
source: str | Path
target: str
readonly: bool = False
def to_xml(self) -> str:
"""Return XML config for libvirt."""
xml = E.disk(type=self.disk_type, device='disk')
xml.append(E.driver(name='qemu', type='qcow2', cache='writethrough'))
if self.disk_type == 'file':
xml.append(E.source(file=str(self.source)))
xml.append(E.target(dev=self.target, bus='virtio'))
if self.readonly:
xml.append(E.readonly())
return etree.tostring(xml, encoding='unicode', pretty_print=True)
@classmethod
def from_xml(cls, xml: Union[str, etree.Element]) -> 'DiskConfig': # noqa: UP007
"""
Return :class:`DiskConfig` instance using existing XML config.
:param xml: Disk device XML configuration as :class:`str` or lxml
:class:`etree.Element` object.
"""
if isinstance(xml, str):
xml = etree.fromstring(xml)
disk_params = {
'disk_type': xml.get('type'),
'source': xml.find('source').get('file'),
'target': xml.find('target').get('dev'),
'readonly': False if xml.find('readonly') is None else True, # noqa: SIM211
}
for param in disk_params:
if disk_params[param] is None:
msg = f"Bad XML config: parameter '{param}' is not defined"
raise ValueError(msg)
return cls(**disk_params)
class Volume: class Volume:
"""Storage volume manipulating class.""" """Storage volume manipulating class."""

View File

@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or # the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version. # (at your option) any later version.
# #
# Ansible is distributed in the hope that it will be useful, # Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Configuration loader.""" """Configuration loader."""

77
compute/utils/dictutil.py Normal file
View File

@ -0,0 +1,77 @@
# This file is part of Compute
#
# Compute is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Dict tools."""
from compute.exceptions import DictMergeConflictError
def merge(a: dict, b: dict, path: list[str] | None = None) -> dict:
"""
Merge `b` into `a`. Return modified `a`.
:raise: :class:`DictMergeConflictError`
"""
if path is None:
path = []
for key in b:
if key in a:
if isinstance(a[key], dict) and isinstance(b[key], dict):
merge(a[key], b[key], [*path, str(key)])
elif a[key] != b[key]:
raise DictMergeConflictError('.'.join([*path, str(key)]))
else:
a[key] = b[key]
return a
def override(a: dict, b: dict) -> dict:
"""
Override dict `a` by `b` values.
Keys that not exists in `a`, but exists in `b` will be
appended to `a`.
.. code-block:: shell-session
>>> from compute.utils import dictutil
>>> default = {
... 'bus': 'virtio',
... 'driver': {'name': 'qemu', 'type': 'qcow2'}
... }
>>> user = {
... 'bus': 'ide',
... 'target': 'vda',
... 'driver': {'type': 'raw'}
... }
>>> dictutil.override(default, user)
{'bus': 'ide', 'driver': {'name': 'qemu', 'type': 'raw'},
'target': 'vda'}
NOTE: merging dicts contained in lists is not supported.
:param a: Dict to be overwritten.
:param b: A dict whose values will be used to rewrite dict `a`.
:return: Modified `a` dict.
"""
for key in b:
if key in a:
if isinstance(a[key], dict) and isinstance(b[key], dict):
override(a[key], b[key])
else:
a[key] = b[key] # replace existing key's values
else:
a[key] = b[key]
return a

View File

@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or # the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version. # (at your option) any later version.
# #
# Ansible is distributed in the hope that it will be useful, # Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Random identificators.""" """Random identificators."""

View File

@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or # the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version. # (at your option) any later version.
# #
# Ansible is distributed in the hope that it will be useful, # Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Tools for data units convertion.""" """Tools for data units convertion."""
@ -21,7 +21,7 @@ from compute.exceptions import InvalidDataUnitError
class DataUnit(StrEnum): class DataUnit(StrEnum):
"""Data units enumerated.""" """Data units enumeration."""
BYTES = 'bytes' BYTES = 'bytes'
KIB = 'KiB' KIB = 'KiB'
@ -29,6 +29,13 @@ class DataUnit(StrEnum):
GIB = 'GiB' GIB = 'GiB'
TIB = 'TiB' TIB = 'TiB'
@classmethod
def _missing_(cls, name: str) -> 'DataUnit':
for member in cls:
if member.name.lower() == name.lower():
return member
return None
def to_bytes(value: int, unit: DataUnit = DataUnit.BYTES) -> int: def to_bytes(value: int, unit: DataUnit = DataUnit.BYTES) -> int:
"""Convert value to bytes. See :class:`DataUnit`.""" """Convert value to bytes. See :class:`DataUnit`."""

View File

@ -0,0 +1 @@
div.code-block-caption {background: #d0d0d0;}

View File

@ -0,0 +1,8 @@
CLI
===
.. toctree::
:maxdepth: 1
usage
reference

View File

@ -0,0 +1,7 @@
CLI Reference
=============
.. argparse::
:module: compute.cli.control
:func: get_parser
:prog: compute

102
docs/source/cli/usage.rst Normal file
View File

@ -0,0 +1,102 @@
Usage
=====
Creating compute instances
--------------------------
First place your image into images pool path.
Create :file:`inatance.yaml` config file with following content. Replace `debian_12.qcow2` with your actual image filename.
.. code-block:: yaml
:caption: Using prebuilt QCOW2 disk image
:emphasize-lines: 4
:linenos:
name: myinstance
memory: 2048
vcpus: 2
image: debian_12.qcow2
volumes:
- type: file
is_system: true
target: vda
capacity:
value: 10
unit: GiB
Check out what configuration will be applied when ``init``::
compute init -t
Initialise instance with command::
compute init
Also you can use following syntax::
compute init yourfile.yaml
Start instance::
compute start myinstance
Using ISO installation medium
`````````````````````````````
Download ISO image and set it as source for ``cdrom`` device.
Note that the ``image`` parameter is not used here.
.. code-block:: yaml
:caption: Using ISO image
:emphasize-lines: 11-13
:linenos:
name: myinstance
memory: 2048
vcpus: 2
volumes:
- type: file
is_system: true
target: vda
capacity:
value: 10
unit: GiB
- type: file
device: cdrom
source: /images/debian-12.2.0-amd64-netinst.iso
::
compute init
Now edit instance XML configuration to add VNC-server listen address::
virsh edit myinstance
Add ``address`` attribute to start listen on all host network interfaces.
.. code-block:: xml
:caption: libvirt XML config fragment
:emphasize-lines: 2
:linenos:
<graphics type='vnc' port='-1' autoport='yes' listen='0.0.0.0'>
<listen type='address' address='0.0.0.0'/>
</graphics>
Also you can specify VNC server port. This is **5900** by default.
Start instance and connect to VNC via any VNC client such as `Remmina <https://remmina.org/>`_ or something else.
::
compute start myinstance
Finish the OS installation over VNC and then do::
compute setcdrom myinstance /images/debian-12.2.0-amd64-netinst.iso --detach
compute powrst myinstance
CDROM will be detached. ``powrst`` command will perform instance shutdown and start. Instance will booted from `vda` disk.

View File

@ -1,4 +1,3 @@
# Add ../.. to path for autodoc Sphinx extension
import os import os
import sys import sys
sys.path.insert(0, os.path.abspath('../..')) sys.path.insert(0, os.path.abspath('../..'))
@ -7,16 +6,18 @@ sys.path.insert(0, os.path.abspath('../..'))
project = 'Compute' project = 'Compute'
copyright = '2023, Compute Authors' copyright = '2023, Compute Authors'
author = 'Compute Authors' author = 'Compute Authors'
release = '0.1.0' release = '0.1.0-dev2'
# Sphinx general settings # Sphinx general settings
extensions = [ extensions = [
'sphinx.ext.autodoc', 'sphinx.ext.autodoc',
'sphinx_multiversion', 'sphinx_multiversion',
'sphinxarg.ext',
] ]
templates_path = ['_templates'] templates_path = ['_templates']
exclude_patterns = [] exclude_patterns = []
language = 'en' language = 'en'
#pygments_style = 'monokai'
# HTML output settings # HTML output settings
html_theme = 'alabaster' html_theme = 'alabaster'

View File

@ -3,9 +3,14 @@ Compute
Compute instances management library. Compute instances management library.
.. toctree:: Contents
:maxdepth: 1 --------
.. toctree::
:maxdepth: 2
installation
cli/index
pyapi/index pyapi/index
Indices and tables Indices and tables

View File

@ -0,0 +1,47 @@
Installation
============
Install Debian 12 on your host system. If you want use virtual machine as host make sure that nested virtualization is enabled.
1. Download or build ``compute`` DEB packages.
2. Install packages::
apt-get install ./compute*
3. Make sure that ``libvirtd`` and ``dnsmasq`` are enabled and running::
systemctl enable --now libvirtd.service
systemctl enable --now dnsmasq.service
4. Prepare storage pools. You need storage pool for images and for instance volumes.
::
for pool in images volumes; do
virsh pool-define-as $pool dir - - - - "/$pool"
virsh pool-build $pool
virsh pool-start $pool
virsh pool-autostart $pool
done
5. Prepare env. Set environment variables in your `~/.profile`, `~/.bashrc` or global in `/etc/profile.d/compute` or `/etc/bash.bashrc`:
.. code-block:: sh
export CMP_IMAGES_POOL=images
export CMP_VOLUMES_POOL=volumes
Configuration file is yet not supported.
Make sure the variables are exported to the environment::
printenv | grep CMP_
If the command didn't show anything source your rc files or relogin.
6. Prepare network::
virsh net-start default
virsh net-autostart default
7. Done. Now you can follow `CLI instructions <cli/index.html>`_

View File

@ -1,38 +1,8 @@
Python API Python API
========== ==========
The API allows you to perform actions on instances programmatically. API Reference
-------------
.. code-block:: python
import compute
with compute.Session() as session:
instance = session.get_instance('myinstance')
info = instance.get_info()
print(info)
:class:`Session` context manager provides an abstraction over :class:`libvirt.virConnect`
and returns objects of other classes of the present library.
Entity representation
---------------------
Entities such as a compute-instance are represented as classes. These classes directly
call libvirt methods to perform operations on the hypervisor. An example class is
:class:`Volume`.
The configuration files of various libvirt objects in `compute` are described by special
dataclasses. The dataclass stores object parameters in its properties and can return an
XML config for libvirt using the ``to_xml()`` method. For example :class:`VolumeConfig`.
`Pydantic <https://docs.pydantic.dev/>`_ models are used to validate input data.
For example :class:`VolumeSchema`.
Modules documentation
---------------------
.. toctree:: .. toctree::
:maxdepth: 4 :maxdepth: 4

View File

@ -0,0 +1,5 @@
``devices``
===========
.. automodule:: compute.instance.devices
:members:

View File

@ -7,4 +7,5 @@
instance instance
guest_agent guest_agent
devices
schemas schemas

View File

@ -12,3 +12,10 @@
.. automodule:: compute.utils.ids .. automodule:: compute.utils.ids
:members: :members:
``utils.dictutil``
------------------
.. automodule:: compute.utils.dictutil
:members:

View File

@ -14,6 +14,7 @@ RUN apt-get update; \
python3-setuptools \ python3-setuptools \
python3-sphinx \ python3-sphinx \
python3-sphinx-multiversion \ python3-sphinx-multiversion \
python3-sphinx-argparse \
python3-libvirt \ python3-libvirt \
python3-lxml \ python3-lxml \
python3-yaml \ python3-yaml \

View File

@ -18,8 +18,10 @@ _compute_root_cmd="
status status
setvcpus setvcpus
setmem setmem
setpass" setpass
_compute_init_opts="" setcdrom
delete"
_compute_init_opts="--test --start"
_compute_exec_opts="--timeout --executable --env --no-join-args" _compute_exec_opts="--timeout --executable --env --no-join-args"
_compute_ls_opts="" _compute_ls_opts=""
_compute_start_opts="" _compute_start_opts=""
@ -33,6 +35,8 @@ _compute_status_opts=""
_compute_setvcpus_opts="" _compute_setvcpus_opts=""
_compute_setmem_opts="" _compute_setmem_opts=""
_compute_setpass_opts="--encrypted" _compute_setpass_opts="--encrypted"
_compute_setcdrom_opts="--detach"
_compute_delete_opts="--yes --save-volumes"
_compute_complete_instances() _compute_complete_instances()
{ {
@ -78,6 +82,8 @@ _compute_complete()
setvcpus) _compute_compreply "$_compute_setvcpus_opts";; setvcpus) _compute_compreply "$_compute_setvcpus_opts";;
setmem) _compute_compreply "$_compute_setmem_opts";; setmem) _compute_compreply "$_compute_setmem_opts";;
setpass) _compute_compreply "$_compute_setpass_opts";; setpass) _compute_compreply "$_compute_setpass_opts";;
setcdrom) _compute_compreply "$_compute_setcdrom_opts";;
delete) _compute_compreply "$_compute_delete_opts";;
*) COMPREPLY=() *) COMPREPLY=()
esac esac
;; ;;

View File

@ -13,6 +13,7 @@ Build-Depends:
python3-all, python3-all,
python3-sphinx, python3-sphinx,
python3-sphinx-multiversion, python3-sphinx-multiversion,
python3-sphinx-argparse,
python3-libvirt, python3-libvirt,
python3-lxml, python3-lxml,
python3-yaml, python3-yaml,

20
poetry.lock generated
View File

@ -696,6 +696,24 @@ docs = ["sphinxcontrib-websupport"]
lint = ["docutils-stubs", "flake8 (>=3.5.0)", "flake8-simplify", "isort", "mypy (>=0.990)", "ruff", "sphinx-lint", "types-requests"] lint = ["docutils-stubs", "flake8 (>=3.5.0)", "flake8-simplify", "isort", "mypy (>=0.990)", "ruff", "sphinx-lint", "types-requests"]
test = ["cython (>=3.0)", "filelock", "html5lib", "pytest (>=4.6)", "setuptools (>=67.0)"] test = ["cython (>=3.0)", "filelock", "html5lib", "pytest (>=4.6)", "setuptools (>=67.0)"]
[[package]]
name = "sphinx-argparse"
version = "0.4.0"
description = "A sphinx extension that automatically documents argparse commands and options"
category = "dev"
optional = false
python-versions = ">=3.7"
files = [
{file = "sphinx_argparse-0.4.0-py3-none-any.whl", hash = "sha256:73bee01f7276fae2bf621ccfe4d167af7306e7288e3482005405d9f826f9b037"},
{file = "sphinx_argparse-0.4.0.tar.gz", hash = "sha256:e0f34184eb56f12face774fbc87b880abdb9017a0998d1ec559b267e9697e449"},
]
[package.dependencies]
sphinx = ">=1.2.0"
[package.extras]
markdown = ["CommonMark (>=0.5.6)"]
[[package]] [[package]]
name = "sphinx-autobuild" name = "sphinx-autobuild"
version = "2021.3.14" version = "2021.3.14"
@ -895,4 +913,4 @@ zstd = ["zstandard (>=0.18.0)"]
[metadata] [metadata]
lock-version = "2.0" lock-version = "2.0"
python-versions = '^3.11' python-versions = '^3.11'
content-hash = "e5c07eebe683b92360ec12cada14fc5ccbe4e4add52549bf978f580e551abfb0" content-hash = "cbded73a481e7c6d6e4c4d5de8e37ac5a53848ab774090a913f1846ef4d7421e"

View File

@ -23,6 +23,7 @@ isort = '^5.12.0'
sphinx = '^7.2.6' sphinx = '^7.2.6'
sphinx-autobuild = '^2021.3.14' sphinx-autobuild = '^2021.3.14'
sphinx-multiversion = '^0.2.4' sphinx-multiversion = '^0.2.4'
sphinx-argparse = "^0.4.0"
[build-system] [build-system]
requires = ['poetry-core'] requires = ['poetry-core']