Compare commits

...

20 Commits

Author SHA1 Message Date
ge
90de626999 bump version to 0.1.0-dev5 2024-05-17 00:07:51 +03:00
ge
9b8b7be2d7 fix build issue 2024-05-17 00:06:14 +03:00
ge
7289248925 fix creating instances from ISO and existing volumes 2024-05-17 00:05:38 +03:00
ge
197e272f3e do not delete volumes from images pool 2024-05-17 00:04:27 +03:00
ge
baa511f678 fix logs 2024-05-17 00:03:29 +03:00
ge
32b9600554 add computed.toml 2024-04-22 14:06:29 +03:00
ge
71ef774060 upd deps 2024-01-16 22:51:45 +03:00
ge
eda3d50607 remove docs from all target 2024-01-16 22:29:56 +03:00
ge
10ff2ca297 add PKGBUILD, upd DEB builder 2024-01-16 22:26:59 +03:00
ge
f091b34854 upd instance.yaml example 2024-01-16 22:25:12 +03:00
ge
b211148c0a upd rendom_mac() 2024-01-16 22:24:42 +03:00
ge
d2515cace8 v0.1.0-dev4 2024-01-13 00:45:30 +03:00
ge
bdff33759c upd docs 2023-12-13 21:02:09 +03:00
ge
072e86f987 fix docs 2023-12-13 02:08:54 +03:00
ge
103d167ef7 fix docs 2023-12-13 02:05:12 +03:00
ge
d7a73e9bd1 various updates v.dev3 2023-12-13 01:42:50 +03:00
ge
b0fa1b7b25 various improvements 2023-12-03 23:25:34 +03:00
ge
0d5246e95e fix doc template 2023-12-01 01:45:50 +03:00
ge
dab71df3d0 various improvemets 2023-12-01 01:39:26 +03:00
ge
e00979dbb8 fix README 2023-11-23 02:39:06 +03:00
67 changed files with 3031 additions and 1060 deletions

3
.gitignore vendored
View File

@ -1,6 +1,7 @@
dist/
docs/build/
packaging/build/
packaging/*/build/
instance.yaml
.ruff_cache/
__pycache__/
*.pyc

View File

@ -5,18 +5,24 @@ DOCS_BUILDDIR = docs/build
.PHONY: docs
all: build
all: build debian archlinux
requirements.txt:
poetry export -f requirements.txt -o requirements.txt
build: format lint
awk '/^version/{print $$3}' pyproject.toml \
| xargs -I {} sed "s/__version__ =.*/__version__ = '{}'/" -i $(SRCDIR)/__init__.py
build: version format lint
poetry build
build-deb: build
cd packaging && $(MAKE)
debian:
cd packaging/debian && $(MAKE)
archlinux:
cd packaging/archlinux && $(MAKE)
version:
VERSION=$$(awk '/^version/{print $$3}' pyproject.toml); \
sed "s/__version__ =.*/__version__ = $$VERSION/" -i $(SRCDIR)/__init__.py; \
sed "s/release =.*/release = $$VERSION/" -i $(DOCS_SRCDIR)/conf.py
format:
poetry run isort $(SRCDIR)
@ -32,13 +38,19 @@ docs-versions:
poetry run sphinx-multiversion $(DOCS_SRCDIR) $(DOCS_BUILDDIR)
serve-docs:
poetry run sphinx-autobuild $(DOCS_SRCDIR) $(DOCS_BUILDDIR)
poetry run sphinx-autobuild $(DOCS_SRCDIR) $(DOCS_BUILDDIR) \
--pre-build 'make clean'
clean:
[ -d $(DISTDIR) ] && rm -rf $(DISTDIR) || true
[ -d $(DOCS_BUILDDIR) ] && rm -rf $(DOCS_BUILDDIR) || true
find . -type d -name __pycache__ -exec rm -rf {} \; > /dev/null 2>&1 || true
cd packaging && $(MAKE) clean
cd packaging/debian && $(MAKE) clean
cd packaging/archlinux && $(MAKE) clean
test-build: build-deb
scp packaging/build/compute*.deb vm:~
test-build: build debian
scp packaging/debian/build/compute*.deb vm:~
upload-docs:
ssh root@hitomi 'rm -rf /srv/http/nixhacks.net/hstack/compute/*'
scp -r $(DOCS_BUILDDIR)/* root@hitomi:/srv/http/nixhacks.net/hstack/compute/

111
README.md
View File

@ -1,31 +1,32 @@
# Compute
Compute instances management library and tools.
Compute instances management library.
## Docs
Run `make serve-docs`. See [Development](#development) below.
Documantation is available [here](https://nixhacks.net/hstack/compute/master/index.html).
To build actual docs run `make serve-docs`. See [Development](#development) below.
## Roadmap
- [x] Create instances
- [ ] CDROM
- [ ] cloud-init for provisioning instances
- [x] Instance power management
- [x] Instance pause and resume
- [x] CDROM
- [x] cloud-init for provisioning instances
- [x] Power management
- [x] Pause and resume
- [x] vCPU hotplug
- [x] Memory hotplug
- [x] Hot disk resize [not tested]
- [ ] CPU topology customization
- [x] CPU customization (emulation mode, model, vendor, features)
- [x] CPU topology customization
- [ ] BIOS/UEFI settings
- [x] Device attaching
- [x] Device detaching
- [ ] GPU passthrough
- [ ] CPU guarantied resource percent support
- [x] QEMU Guest Agent management
- [ ] Instance resources usage stats
- [ ] SSH-keys management
- [ ] Resource usage stats
- [x] SSH-keys management
- [x] Setting user passwords in guest
- [x] QCOW2 disks support
- [ ] ZVOL support
@ -35,10 +36,13 @@ Run `make serve-docs`. See [Development](#development) below.
- [ ] Idempotency
- [ ] CLI [in progress]
- [ ] HTTP API
- [ ] Instance migrations
- [ ] Instance snapshots
- [ ] Instance backups
- [ ] Migrations
- [ ] Snapshots
- [ ] Backups
- [ ] LXC
- [ ] Attaching CDROM from sources: block, (http|https|ftp|ftps|tftp)://
- [ ] Instance clones (thin, fat)
- [ ] MicroVM
## Development
@ -50,7 +54,7 @@ Install [poetry](https://python-poetry.org/), clone this repository and run:
poetry install --with dev --with docs
```
# Build Debian package
## Build Debian package
Install Docker first, then run:
@ -60,46 +64,11 @@ make build-deb
`compute` and `compute-doc` packages will built. See packaging/build directory.
# Installation
## Installation
Packages can be installed via `dpkg` or `apt-get`:
See [Installation](https://nixhacks.net/hstack/compute/master/installation.html).
```
# apt-get install ./compute*.deb
```
After installation prepare environment, run following command to start libvirtd and create required storage pools:
```
# systemctl enable --now libvirtd.service
# virsh net-start default
# virsh net-autostart default
# for pool in images volumes; do
virsh pool-define-as $pool dir - - - - "/$pool"
virsh pool-build $pool
virsh pool-start $pool
done
```
Then set environment variables in your `~/.profile`, `~/.bashrc` or global in `/etc/profile.d/compute` or `/etc/bash.bashrc`:
```
export CMP_IMAGES_POOL=images
export CMP_VOLUMES_POOL=volumes
```
Configuration file is yet not supported.
Make sure the variables are exported to the environment:
```
printenv | grep CMP_
```
If the command didn't show anything _source_ your rc files or relogin.
# Basic usage
## Basic usage
To get help run:
@ -107,49 +76,17 @@ To get help run:
compute --help
```
See [CLI docs](https://nixhacks.net/hstack/compute/master/cli/index.html) for more info.
Also you can use `compute` as generic Python library. For example:
```python
from compute import Session
import compute
with Session() as session:
with compute.Session() as session:
instance = session.get_instance('myinstance')
if not instance.is_running():
instance.start()
else:
print('instance is already running')
```
# Create compute instances
Place your qcow2 image in `/volumes` directory. For example `debian_12.qcow2`.
Create `instance.yaml` file with following content:
```yaml
name: myinstance
memory: 2048 # memory in MiB
vcpus: 2
image: debian_12.qcow2
volumes:
- type: file
is_system: true
target: vda
capacity:
value: 10
unit: GiB
```
Refer to `Instance` class docs for more info. Full `instance.yaml` example will be provided later.
To initialise instance run:
```
compute -l debug init instance.yaml
```
Start instance:
```
compute start myinstance
```

View File

@ -5,18 +5,19 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Compute instances management library."""
__version__ = '0.1.0-dev1'
__version__ = '0.1.0-dev5'
from .instance import Instance, InstanceConfig, InstanceSchema
from .config import Config
from .instance import CloudInit, Instance, InstanceConfig, InstanceSchema
from .session import Session
from .storage import StoragePool, Volume, VolumeConfig

View File

@ -5,17 +5,17 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Command line interface for compute module."""
from compute.cli import main
from compute.cli import parser
main.cli()
parser.run()

View File

@ -5,26 +5,38 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Common symbols."""
from abc import ABC, abstractmethod
from pydantic import BaseModel, Extra
class EntityModel(BaseModel):
"""Basic entity model."""
class Config:
"""Do not allow extra fields."""
extra = Extra.forbid
class EntityConfig(ABC):
"""An abstract entity XML config builder class."""
@abstractmethod
def to_xml(self) -> str:
"""Return device XML config."""
"""Return entity XML config."""
raise NotImplementedError
DeviceConfig = EntityConfig
class DeviceConfig(EntityConfig):
"""An abstract device XML config."""

421
compute/cli/commands.py Normal file
View File

@ -0,0 +1,421 @@
# This file is part of Compute
#
# Compute is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""CLI commands."""
import argparse
import base64
import json
import logging
import pathlib
import re
import shlex
import sys
import uuid
import libvirt
import pydantic
import yaml
from compute import Session
from compute.cli.term import Table, confirm
from compute.exceptions import GuestAgentTimeoutExpired
from compute.instance import CloudInit, GuestAgent, InstanceSchema
from compute.instance.devices import DiskConfig, DiskDriver
from compute.utils import dictutil, diskutils, ids
log = logging.getLogger(__name__)
libvirt.registerErrorHandler(
lambda userdata, err: None, # noqa: ARG005
ctx=None,
)
def init(session: Session, args: argparse.Namespace) -> None:
"""Initialise compute instance using YAML config."""
try:
data = yaml.load(args.file.read(), Loader=yaml.SafeLoader)
log.debug('Read from file: %s', data)
except yaml.YAMLError as e:
sys.exit(f'error: cannot parse YAML: {e}')
capabilities = session.get_capabilities()
node_info = session.get_node_info()
base_instance_config = {
'name': str(uuid.uuid4()).split('-')[0],
'title': None,
'description': None,
'arch': capabilities.arch,
'machine': capabilities.machine,
'emulator': capabilities.emulator,
'max_vcpus': node_info.cpus,
'max_memory': node_info.memory,
'cpu': {
'emulation_mode': 'host-passthrough',
'model': None,
'vendor': None,
'topology': None,
'features': None,
},
'boot': {'order': ['cdrom', 'hd']},
'cloud_init': None,
}
data = dictutil.override(base_instance_config, data)
net_default_interface = {
'model': 'virtio',
'source': 'default',
'mac': ids.random_mac(),
}
net_config = data.get('network', 'DEFAULT')
if net_config == 'DEFAULT' or net_config is True:
data['network'] = {'interfaces': [net_default_interface]}
elif net_config is None or net_config is False:
pass # allow creating instance without network interfaces
else:
interfaces = data['network'].get('interfaces')
if interfaces:
interfaces_configs = [
dictutil.override(net_default_interface, interface)
for interface in interfaces
]
data['network']['interfaces'] = interfaces_configs
volumes = []
targets = []
for volume in data['volumes']:
base_disk_config = {
'bus': 'virtio',
'is_readonly': False,
'driver': {
'name': 'qemu',
'type': 'qcow2',
'cache': 'writethrough',
},
}
base_cdrom_config = {
'bus': 'ide',
'is_readonly': True,
'driver': {
'name': 'qemu',
'type': 'raw',
'cache': 'writethrough',
},
}
if volume.get('device') is None:
volume['device'] = 'disk'
if volume.get('target') is None:
prefix = 'hd' if volume['device'] == 'cdrom' else 'vd'
target = diskutils.get_disk_target(targets, prefix)
volume['target'] = target
targets.append(target)
else:
targets.append(volume['target'])
if volume['device'] == 'disk':
volumes.append(dictutil.override(base_disk_config, volume))
if volume['device'] == 'cdrom':
volumes.append(dictutil.override(base_cdrom_config, volume))
data['volumes'] = volumes
if data['cloud_init'] is not None:
cloud_init_config = {
'user_data': None,
'meta_data': None,
'vendor_data': None,
'network_config': None,
}
data['cloud_init'] = dictutil.override(
cloud_init_config,
data['cloud_init'],
)
for item in data['cloud_init']:
cidata = data['cloud_init'][item]
if cidata is None:
pass
elif isinstance(cidata, str):
if cidata.startswith('base64:'):
data['cloud_init'][item] = base64.b64decode(
cidata.split(':')[1]
).decode('utf-8')
elif re.fullmatch(r'^[^\n]{1,1024}$', cidata, re.I):
data_file = pathlib.Path(cidata)
if data_file.exists():
with data_file.open('r') as f:
data['cloud_init'][item] = f.read()
else:
pass
else:
data['cloud_init'][item] = yaml.dump(cidata)
try:
log.debug('Input data: %s', data)
if args.test:
_ = InstanceSchema(**data)
print(json.dumps(dict(data), indent=4, sort_keys=True))
sys.exit()
instance = session.create_instance(**data)
print(f'Initialised: {instance.name}')
if args.start:
instance.start()
print(f'Started: {instance.name}')
except pydantic.ValidationError as e:
for error in e.errors():
fields = '.'.join([str(lc) for lc in error['loc']])
print(
f"validation error: {fields}: {error['msg']}",
file=sys.stderr,
)
def exec_(session: Session, args: argparse.Namespace) -> None:
"""
Execute command in guest via guest agent.
NOTE: any argument after instance name will be passed into guest's shell
"""
instance = session.get_instance(args.instance)
ga = GuestAgent(instance.domain, timeout=args.timeout)
arguments = args.arguments.copy()
if len(arguments) > 1 and not args.no_join_args:
arguments = [shlex.join(arguments)]
if not args.no_join_args:
arguments.insert(0, '-c')
stdin = None
if not sys.stdin.isatty():
stdin = sys.stdin.read()
try:
output = ga.guest_exec(
path=args.executable,
args=arguments,
env=args.env,
stdin=stdin,
capture_output=True,
decode_output=True,
poll=True,
)
except GuestAgentTimeoutExpired as e:
sys.exit(
f'{e}. NOTE: command may still running in guest, '
f'PID={ga.last_pid}'
)
if output.stderr:
print(output.stderr.strip(), file=sys.stderr)
if output.stdout:
print(output.stdout.strip(), file=sys.stdout)
sys.exit(output.exitcode)
def ls(session: Session, args: argparse.Namespace) -> None: # noqa: ARG001
"""List compute instances."""
table = Table()
table.header = ['NAME', 'STATE', 'NVCPUS', 'MEMORY']
for instance in session.list_instances():
info = instance.get_info()
table.add_row(
[
instance.name,
instance.get_status() + ' ',
info.nproc,
f'{int(info.memory / 1024)} MiB',
]
)
print(table)
def lsdisks(session: Session, args: argparse.Namespace) -> None:
"""List block devices attached to instance."""
instance = session.get_instance(args.instance)
if args.persistent:
disks = instance.list_disks(persistent=True)
else:
disks = instance.list_disks()
table = Table()
table.header = ['TARGET', 'SOURCE']
for disk in disks:
table.add_row([disk.target, disk.source])
print(table)
def start(session: Session, args: argparse.Namespace) -> None:
"""Start instance."""
instance = session.get_instance(args.instance)
instance.start()
def shutdown(session: Session, args: argparse.Namespace) -> None:
"""Shutdown instance."""
instance = session.get_instance(args.instance)
if args.soft:
method = 'SOFT'
elif args.hard:
method = 'HARD'
elif args.destroy:
method = 'DESTROY'
else:
method = 'NORMAL'
instance.shutdown(method)
def reboot(session: Session, args: argparse.Namespace) -> None:
"""Reboot instance."""
instance = session.get_instance(args.instance)
instance.reboot()
def reset(session: Session, args: argparse.Namespace) -> None:
"""Reset instance."""
instance = session.get_instance(args.instance)
instance.reset()
def powrst(session: Session, args: argparse.Namespace) -> None:
"""Power reset instance."""
instance = session.get_instance(args.instance)
instance.power_reset()
def pause(session: Session, args: argparse.Namespace) -> None:
"""Pause instance."""
instance = session.get_instance(args.instance)
instance.pause()
def resume(session: Session, args: argparse.Namespace) -> None:
"""Resume instance."""
instance = session.get_instance(args.instance)
instance.resume()
def status(session: Session, args: argparse.Namespace) -> None:
"""Display instance status."""
instance = session.get_instance(args.instance)
print(instance.get_status())
def setvcpus(session: Session, args: argparse.Namespace) -> None:
"""Set instance vCPU number."""
instance = session.get_instance(args.instance)
instance.set_vcpus(args.nvcpus, live=True)
def setmem(session: Session, args: argparse.Namespace) -> None:
"""Set instance memory size."""
instance = session.get_instance(args.instance)
instance.set_memory(args.memory, live=True)
def setpass(session: Session, args: argparse.Namespace) -> None:
"""Set user password in guest."""
instance = session.get_instance(args.instance)
instance.set_user_password(
args.username,
args.password,
encrypted=args.encrypted,
)
def setcdrom(session: Session, args: argparse.Namespace) -> None:
"""Manage CDROM devices."""
instance = session.get_instance(args.instance)
if args.detach:
for disk in instance.list_disks(persistent=True):
if disk.device == 'cdrom' and disk.source == args.source:
instance.detach_disk(disk.target, live=False)
print(
f"disk '{disk.target}' detached, "
'perform power reset to apply changes'
)
return
disks_live = instance.list_disks(persistent=False)
disks_inactive = instance.list_disks(persistent=True)
disks = [d.target for d in disks_inactive if d not in disks_live]
target = diskutils.get_disk_target(disks, 'hd')
cdrom = DiskConfig(
type='file',
device='cdrom',
source=args.source,
target=target,
is_readonly=True,
bus='ide',
driver=DiskDriver('qemu', 'raw', 'writethrough'),
)
instance.attach_device(cdrom, live=False)
print(
f"CDROM attached as disk '{target}', "
'perform power reset to apply changes'
)
def setcloudinit(session: Session, args: argparse.Namespace) -> None:
"""
Set cloud-init configuration.
The cloud-init disk must not be mounted to the host system while making
changes using this command! In this case, data may be damaged when writing
to disk - if the new content of the file is longer than the old one, it
will be truncated.
"""
if (
args.user_data is None
and args.vendor_data is None
and args.network_config is None
and args.meta_data is None
):
sys.exit('nothing to do')
instance = session.get_instance(args.instance)
disks = instance.list_disks()
cloud_init_disk_path = None
cloud_init_disk_target = diskutils.get_disk_target(
[d.target for d in disks], prefix='vd'
)
cloud_init = CloudInit()
if args.user_data:
cloud_init.user_data = args.user_data.read()
if args.vendor_data:
cloud_init.vendor_data = args.vendor_data.read()
if args.network_config:
cloud_init.network_config = args.network_config.read()
if args.meta_data:
cloud_init.meta_data = args.meta_data.read()
for disk in disks:
if disk.source.endswith('cloud-init.img'):
cloud_init_disk_path = disk.source
break
if cloud_init_disk_path is None:
volumes = session.get_storage_pool(session.VOLUMES_POOL)
cloud_init_disk_path = volumes.path.joinpath(
f'{instance.name}-cloud-init.img'
)
cloud_init.create_disk(cloud_init_disk_path)
volumes.refresh()
cloud_init.attach_disk(
cloud_init_disk_path,
cloud_init_disk_target,
instance,
)
else:
cloud_init.update_disk(cloud_init_disk_path)
def delete(session: Session, args: argparse.Namespace) -> None:
"""Delete instance with local storage volumes."""
if args.yes is True or confirm(
'this action is irreversible, continue?',
default=False,
):
instance = session.get_instance(args.instance)
if args.save_volumes is False:
instance.delete(with_volumes=True)
else:
instance.delete()
else:
print('aborted')

View File

@ -1,501 +0,0 @@
# This file is part of Compute
#
# Compute is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
"""Command line interface."""
import argparse
import io
import logging
import os
import shlex
import sys
from collections import UserDict
from typing import Any
from uuid import uuid4
import libvirt
import yaml
from pydantic import ValidationError
from compute import __version__
from compute.exceptions import ComputeError, GuestAgentTimeoutExceededError
from compute.instance import GuestAgent
from compute.session import Session
from compute.utils import ids
log = logging.getLogger(__name__)
log_levels = [lv.lower() for lv in logging.getLevelNamesMapping()]
libvirt.registerErrorHandler(
lambda userdata, err: None, # noqa: ARG005
ctx=None,
)
class Table:
"""Minimalistic text table constructor."""
def __init__(self, whitespace: str | None = None):
"""Initialise Table."""
self.whitespace = whitespace or '\t'
self.header = []
self.rows = []
self.table = ''
def add_row(self, row: list) -> None:
"""Add table row."""
self.rows.append([str(col) for col in row])
def add_rows(self, rows: list[list]) -> None:
"""Add multiple rows."""
for row in rows:
self.add_row(row)
def __str__(self) -> str:
"""Build table and return."""
widths = [max(map(len, col)) for col in zip(*self.rows, strict=True)]
self.rows.insert(0, [str(h).upper() for h in self.header])
for row in self.rows:
self.table += self.whitespace.join(
(
val.ljust(width)
for val, width in zip(row, widths, strict=True)
)
)
self.table += '\n'
return self.table.strip()
def _list_instances(session: Session) -> None:
table = Table()
table.header = ['NAME', 'STATE']
for instance in session.list_instances():
table.add_row(
[
instance.name,
instance.get_status(),
]
)
print(table)
sys.exit()
def _exec_guest_agent_command(
session: Session, args: argparse.Namespace
) -> None:
instance = session.get_instance(args.instance)
ga = GuestAgent(instance.domain, timeout=args.timeout)
arguments = args.arguments.copy()
if len(arguments) > 1 and not args.no_join_args:
arguments = [shlex.join(arguments)]
if not args.no_join_args:
arguments.insert(0, '-c')
stdin = None
if not sys.stdin.isatty():
stdin = sys.stdin.read()
try:
output = ga.guest_exec(
path=args.executable,
args=arguments,
env=args.env,
stdin=stdin,
capture_output=True,
decode_output=True,
poll=True,
)
except GuestAgentTimeoutExceededError as e:
sys.exit(
f'{e}. NOTE: command may still running in guest, '
f'PID={ga.last_pid}'
)
if output.stderr:
print(output.stderr.strip(), file=sys.stderr)
if output.stdout:
print(output.stdout.strip(), file=sys.stdout)
sys.exit(output.exitcode)
class _NotPresent:
"""
Type for representing non-existent dictionary keys.
See :class:`_FillableDict`.
"""
class _FillableDict(UserDict):
"""Use :method:`fill` to add key if not present."""
def __init__(self, data: dict):
self.data = data
def fill(self, key: str, value: Any) -> None: # noqa: ANN401
if self.data.get(key, _NotPresent) is _NotPresent:
self.data[key] = value
def _merge_dicts(a: dict, b: dict, path: list[str] | None = None) -> dict:
"""Merge `b` into `a`. Return modified `a`."""
if path is None:
path = []
for key in b:
if key in a:
if isinstance(a[key], dict) and isinstance(b[key], dict):
_merge_dicts(a[key], b[key], [path + str(key)])
elif a[key] == b[key]:
pass # same leaf value
else:
a[key] = b[key] # replace existing key's values
else:
a[key] = b[key]
return a
def _create_instance(session: Session, file: io.TextIOWrapper) -> None:
try:
data = _FillableDict(yaml.load(file.read(), Loader=yaml.SafeLoader))
log.debug('Read from file: %s', data)
except yaml.YAMLError as e:
sys.exit(f'error: cannot parse YAML: {e}')
capabilities = session.get_capabilities()
node_info = session.get_node_info()
data.fill('name', uuid4().hex)
data.fill('title', None)
data.fill('description', None)
data.fill('arch', capabilities.arch)
data.fill('machine', capabilities.machine)
data.fill('emulator', capabilities.emulator)
data.fill('max_vcpus', node_info.cpus)
data.fill('max_memory', node_info.memory)
data.fill('cpu', {})
cpu = {
'emulation_mode': 'host-passthrough',
'model': None,
'vendor': None,
'topology': None,
'features': None,
}
data['cpu'] = _merge_dicts(data['cpu'], cpu)
data.fill(
'network_interfaces',
[{'source': 'default', 'mac': ids.random_mac()}],
)
data.fill('boot', {'order': ['cdrom', 'hd']})
try:
log.debug('Input data: %s', data)
session.create_instance(**data)
except ValidationError as e:
for error in e.errors():
fields = '.'.join([str(lc) for lc in error['loc']])
print(
f"validation error: {fields}: {error['msg']}",
file=sys.stderr,
)
sys.exit()
def _shutdown_instance(session: Session, args: argparse.Namespace) -> None:
instance = session.get_instance(args.instance)
if args.soft:
method = 'SOFT'
elif args.hard:
method = 'HARD'
elif args.unsafe:
method = 'UNSAFE'
else:
method = 'NORMAL'
instance.shutdown(method)
def main(session: Session, args: argparse.Namespace) -> None:
"""Perform actions."""
match args.command:
case 'init':
_create_instance(session, args.file)
case 'exec':
_exec_guest_agent_command(session, args)
case 'ls':
_list_instances(session)
case 'start':
instance = session.get_instance(args.instance)
instance.start()
case 'shutdown':
_shutdown_instance(session, args)
case 'reboot':
instance = session.get_instance(args.instance)
instance.reboot()
case 'reset':
instance = session.get_instance(args.instance)
instance.reset()
case 'powrst':
instance = session.get_instance(args.instance)
instance.power_reset()
case 'pause':
instance = session.get_instance(args.instance)
instance.pause()
case 'resume':
instance = session.get_instance(args.instance)
instance.resume()
case 'status':
instance = session.get_instance(args.instance)
print(instance.status)
case 'setvcpus':
instance = session.get_instance(args.instance)
instance.set_vcpus(args.nvcpus, live=True)
case 'setmem':
instance = session.get_instance(args.instance)
instance.set_memory(args.memory, live=True)
case 'setpass':
instance = session.get_instance(args.instance)
instance.set_user_password(
args.username,
args.password,
encrypted=args.encrypted,
)
def cli() -> None: # noqa: PLR0915
"""Return command line arguments parser."""
root = argparse.ArgumentParser(
prog='compute',
description='manage compute instances',
formatter_class=argparse.RawTextHelpFormatter,
)
root.add_argument(
'-v',
'--verbose',
action='store_true',
default=False,
help='enable verbose mode',
)
root.add_argument(
'-c',
'--connect',
metavar='URI',
help='libvirt connection URI',
)
root.add_argument(
'-l',
'--log-level',
type=str.lower,
metavar='LEVEL',
choices=log_levels,
help='log level',
)
root.add_argument(
'-V',
'--version',
action='version',
version=__version__,
)
subparsers = root.add_subparsers(dest='command', metavar='COMMAND')
# init command
init = subparsers.add_parser(
'init', help='initialise instance using YAML config file'
)
init.add_argument(
'file',
type=argparse.FileType('r', encoding='UTF-8'),
nargs='?',
default='instance.yaml',
help='instance config [default: instance.yaml]',
)
# exec subcommand
execute = subparsers.add_parser(
'exec',
help='execute command in guest via guest agent',
description=(
'NOTE: any argument after instance name will be passed into '
'guest as shell command.'
),
)
execute.add_argument('instance')
execute.add_argument('arguments', nargs=argparse.REMAINDER)
execute.add_argument(
'-t',
'--timeout',
type=int,
default=60,
help=(
'waiting time in seconds for a command to be executed '
'in guest [default: 60]'
),
)
execute.add_argument(
'-x',
'--executable',
default='/bin/sh',
help='path to executable in guest [default: /bin/sh]',
)
execute.add_argument(
'-e',
'--env',
type=str,
nargs='?',
action='append',
help='environment variables to pass to executable in guest',
)
execute.add_argument(
'-n',
'--no-join-args',
action='store_true',
default=False,
help=(
"do not join arguments list and add '-c' option, suitable "
'for non-shell executables and other specific cases.'
),
)
# ls subcommand
listall = subparsers.add_parser('ls', help='list instances')
listall.add_argument(
'-a',
'--all',
action='store_true',
default=False,
help='list all instances including inactive',
)
# start subcommand
start = subparsers.add_parser('start', help='start instance')
start.add_argument('instance')
# shutdown subcommand
shutdown = subparsers.add_parser('shutdown', help='shutdown instance')
shutdown.add_argument('instance')
shutdown_opts = shutdown.add_mutually_exclusive_group()
shutdown_opts.add_argument(
'-s',
'--soft',
action='store_true',
help='normal guest OS shutdown, guest agent is used',
)
shutdown_opts.add_argument(
'-n',
'--normal',
action='store_true',
help='shutdown with hypervisor selected method [default]',
)
shutdown_opts.add_argument(
'-H',
'--hard',
action='store_true',
help=(
"gracefully destroy instance, it's like long "
'pressing the power button'
),
)
shutdown_opts.add_argument(
'-u',
'--unsafe',
action='store_true',
help=(
'destroy instance, this is similar to a power outage '
'and may result in data loss or corruption'
),
)
# reboot subcommand
reboot = subparsers.add_parser('reboot', help='reboot instance')
reboot.add_argument('instance')
# reset subcommand
reset = subparsers.add_parser('reset', help='reset instance')
reset.add_argument('instance')
# powrst subcommand
powrst = subparsers.add_parser('powrst', help='power reset instance')
powrst.add_argument('instance')
# pause subcommand
pause = subparsers.add_parser('pause', help='pause instance')
pause.add_argument('instance')
# resume subcommand
resume = subparsers.add_parser('resume', help='resume paused instance')
resume.add_argument('instance')
# status subcommand
status = subparsers.add_parser('status', help='display instance status')
status.add_argument('instance')
# setvcpus subcommand
setvcpus = subparsers.add_parser('setvcpus', help='set vCPU number')
setvcpus.add_argument('instance')
setvcpus.add_argument('nvcpus', type=int)
# setmem subcommand
setmem = subparsers.add_parser('setmem', help='set memory size')
setmem.add_argument('instance')
setmem.add_argument('memory', type=int, help='memory in MiB')
# setpass subcommand
setpass = subparsers.add_parser(
'setpass',
help='set user password in guest',
)
setpass.add_argument('instance')
setpass.add_argument('username')
setpass.add_argument('password')
setpass.add_argument(
'-e',
'--encrypted',
action='store_true',
default=False,
help='set it if password is already encrypted',
)
args = root.parse_args()
if args.command is None:
root.print_help()
sys.exit()
log_level = args.log_level or os.getenv('CMP_LOG')
if isinstance(log_level, str) and log_level.lower() in log_levels:
logging.basicConfig(
level=logging.getLevelNamesMapping()[log_level.upper()]
)
log.debug('CLI started with args: %s', args)
connect_uri = (
args.connect
or os.getenv('CMP_LIBVIRT_URI')
or os.getenv('LIBVIRT_DEFAULT_URI')
or 'qemu:///system'
)
try:
with Session(connect_uri) as session:
main(session, args)
except ComputeError as e:
sys.exit(f'error: {e}')
except KeyboardInterrupt:
sys.exit()
except SystemExit as e:
sys.exit(e)
except Exception as e: # noqa: BLE001
sys.exit(f'unexpected error {type(e)}: {e}')
if __name__ == '__main__':
cli()

471
compute/cli/parser.py Normal file
View File

@ -0,0 +1,471 @@
# This file is part of Compute
#
# Compute is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Command line argument parser."""
import argparse
import logging
import os
import sys
import textwrap
from collections.abc import Callable
from typing import NamedTuple
from compute import Session, __version__
from compute.cli import commands
from compute.exceptions import ComputeError
log = logging.getLogger(__name__)
log_levels = [lv.lower() for lv in logging.getLevelNamesMapping()]
class Doc(NamedTuple):
"""Parsed docstring."""
help: str # noqa: A003
desc: str
def get_doc(func: Callable) -> Doc:
"""Extract help message and description from function docstring."""
doc = func.__doc__
if isinstance(doc, str):
doc = textwrap.dedent(doc).strip().split('\n\n')
return Doc(doc[0][0].lower() + doc[0][1:], '\n\n'.join(doc))
return Doc('', '')
def get_parser() -> argparse.ArgumentParser:
"""Return command line argument parser."""
root = argparse.ArgumentParser(
prog='compute',
description='Manage compute instances.',
)
root.add_argument(
'-V',
'--version',
action='version',
version=__version__,
)
root.add_argument(
'-c',
'--connect',
dest='root_connect',
metavar='URI',
help='libvirt connection URI',
)
root.add_argument(
'-l',
'--log-level',
dest='root_log_level',
type=str.lower,
choices=log_levels,
metavar='LEVEL',
help='log level',
)
# common options
common = argparse.ArgumentParser(add_help=False)
common.add_argument(
'-c',
'--connect',
metavar='URI',
help='libvirt connection URI',
)
common.add_argument(
'-l',
'--log-level',
type=str.lower,
choices=log_levels,
metavar='LEVEL',
help='log level',
)
subparsers = root.add_subparsers(dest='command', metavar='COMMAND')
# init command
init = subparsers.add_parser(
'init',
parents=[common],
formatter_class=argparse.RawTextHelpFormatter,
help=get_doc(commands.init).help,
description=get_doc(commands.init).desc,
)
init.add_argument(
'file',
type=argparse.FileType('r', encoding='UTF-8'),
nargs='?',
default='instance.yaml',
help='instance config [default: instance.yaml]',
)
init.add_argument(
'-s',
'--start',
action='store_true',
default=False,
help='start instance after init',
)
init.add_argument(
'-t',
'--test',
action='store_true',
default=False,
help='just print resulting instance config as JSON and exit',
)
init.set_defaults(func=commands.init)
# exec subcommand
execute = subparsers.add_parser(
'exec',
parents=[common],
formatter_class=argparse.RawTextHelpFormatter,
help=get_doc(commands.exec_).help,
description=get_doc(commands.exec_).desc,
)
execute.add_argument('instance')
execute.add_argument('arguments', nargs=argparse.REMAINDER)
execute.add_argument(
'-t',
'--timeout',
type=int,
default=60,
help=(
'waiting time in seconds for a command to be executed '
'in guest [default: 60]'
),
)
execute.add_argument(
'-x',
'--executable',
default='/bin/sh',
help='path to executable in guest [default: /bin/sh]',
)
execute.add_argument(
'-e',
'--env',
type=str,
nargs='?',
action='append',
help='environment variables to pass to executable in guest',
)
execute.add_argument(
'-n',
'--no-join-args',
action='store_true',
default=False,
help=(
"do not join arguments list and add '-c' option, suitable "
'for non-shell executables and other specific cases.'
),
)
execute.set_defaults(func=commands.exec_)
# ls subcommand
ls = subparsers.add_parser(
'ls',
parents=[common],
formatter_class=argparse.RawTextHelpFormatter,
help=get_doc(commands.ls).help,
description=get_doc(commands.ls).desc,
)
ls.set_defaults(func=commands.ls)
# lsdisks subcommand
lsdisks = subparsers.add_parser(
'lsdisks',
parents=[common],
formatter_class=argparse.RawTextHelpFormatter,
help=get_doc(commands.lsdisks).help,
description=get_doc(commands.lsdisks).desc,
)
lsdisks.add_argument('instance')
lsdisks.add_argument(
'-p',
'--persistent',
action='store_true',
default=False,
help='display only persisnent devices',
)
lsdisks.set_defaults(func=commands.lsdisks)
# start subcommand
start = subparsers.add_parser(
'start',
parents=[common],
formatter_class=argparse.RawTextHelpFormatter,
help=get_doc(commands.start).help,
description=get_doc(commands.start).desc,
)
start.add_argument('instance')
start.set_defaults(func=commands.start)
# shutdown subcommand
shutdown = subparsers.add_parser(
'shutdown',
parents=[common],
formatter_class=argparse.RawTextHelpFormatter,
help=get_doc(commands.shutdown).help,
description=get_doc(commands.shutdown).desc,
)
shutdown.add_argument('instance')
shutdown_opts = shutdown.add_mutually_exclusive_group()
shutdown_opts.add_argument(
'-s',
'--soft',
action='store_true',
help='guest OS shutdown using guest agent',
)
shutdown_opts.add_argument(
'-n',
'--normal',
action='store_true',
help='shutdown with hypervisor selected method [default]',
)
shutdown_opts.add_argument(
'-H',
'--hard',
action='store_true',
help=(
"gracefully destroy instance, it's like long "
'pressing the power button'
),
)
shutdown_opts.add_argument(
'-d',
'--destroy',
action='store_true',
help=(
'destroy instance, this is similar to a power outage '
'and may result in data corruption'
),
)
shutdown.set_defaults(func=commands.shutdown)
# reboot subcommand
reboot = subparsers.add_parser(
'reboot',
parents=[common],
formatter_class=argparse.RawTextHelpFormatter,
help=get_doc(commands.reboot).help,
description=get_doc(commands.reboot).desc,
)
reboot.add_argument('instance')
reboot.set_defaults(func=commands.reboot)
# reset subcommand
reset = subparsers.add_parser(
'reset',
parents=[common],
formatter_class=argparse.RawTextHelpFormatter,
help=get_doc(commands.reset).help,
description=get_doc(commands.reset).desc,
)
reset.add_argument('instance')
reset.set_defaults(func=commands.reset)
# powrst subcommand
powrst = subparsers.add_parser(
'powrst',
parents=[common],
formatter_class=argparse.RawTextHelpFormatter,
help=get_doc(commands.powrst).help,
description=get_doc(commands.powrst).desc,
)
powrst.add_argument('instance')
powrst.set_defaults(func=commands.powrst)
# pause subcommand
pause = subparsers.add_parser(
'pause',
parents=[common],
formatter_class=argparse.RawTextHelpFormatter,
help=get_doc(commands.pause).help,
description=get_doc(commands.pause).desc,
)
pause.add_argument('instance')
pause.set_defaults(func=commands.pause)
# resume subcommand
resume = subparsers.add_parser(
'resume',
parents=[common],
formatter_class=argparse.RawTextHelpFormatter,
help=get_doc(commands.resume).help,
description=get_doc(commands.resume).desc,
)
resume.add_argument('instance')
resume.set_defaults(func=commands.resume)
# status subcommand
status = subparsers.add_parser(
'status',
parents=[common],
formatter_class=argparse.RawTextHelpFormatter,
help=get_doc(commands.status).help,
description=get_doc(commands.status).desc,
)
status.add_argument('instance')
status.set_defaults(func=commands.status)
# setvcpus subcommand
setvcpus = subparsers.add_parser(
'setvcpus',
parents=[common],
formatter_class=argparse.RawTextHelpFormatter,
help=get_doc(commands.setvcpus).help,
description=get_doc(commands.setvcpus).desc,
)
setvcpus.add_argument('instance')
setvcpus.add_argument('nvcpus', type=int)
setvcpus.set_defaults(func=commands.setvcpus)
# setmem subcommand
setmem = subparsers.add_parser(
'setmem',
parents=[common],
formatter_class=argparse.RawTextHelpFormatter,
help=get_doc(commands.setmem).help,
description=get_doc(commands.setmem).desc,
)
setmem.add_argument('instance')
setmem.add_argument('memory', type=int, help='memory in MiB')
setmem.set_defaults(func=commands.setmem)
# setpass subcommand
setpass = subparsers.add_parser(
'setpass',
parents=[common],
formatter_class=argparse.RawTextHelpFormatter,
help=get_doc(commands.setpass).help,
description=get_doc(commands.setpass).desc,
)
setpass.add_argument('instance')
setpass.add_argument('username')
setpass.add_argument('password')
setpass.add_argument(
'-e',
'--encrypted',
action='store_true',
default=False,
help='set it if password is already encrypted',
)
setpass.set_defaults(func=commands.setpass)
# setcdrom subcommand
setcdrom = subparsers.add_parser(
'setcdrom',
parents=[common],
formatter_class=argparse.RawTextHelpFormatter,
help=get_doc(commands.setcdrom).help,
description=get_doc(commands.setcdrom).desc,
)
setcdrom.add_argument('instance')
setcdrom.add_argument('source', help='source for CDROM')
setcdrom.add_argument(
'-d',
'--detach',
action='store_true',
default=False,
help='detach CDROM device',
)
setcdrom.set_defaults(func=commands.setcdrom)
# setcloudinit subcommand
setcloudinit = subparsers.add_parser(
'setcloudinit',
parents=[common],
formatter_class=argparse.RawTextHelpFormatter,
help=get_doc(commands.setcloudinit).help,
description=get_doc(commands.setcloudinit).desc,
)
setcloudinit.add_argument('instance')
setcloudinit.add_argument(
'--user-data',
type=argparse.FileType('r'),
help='user-data file',
)
setcloudinit.add_argument(
'--vendor-data',
type=argparse.FileType('r'),
help='vendor-data file',
)
setcloudinit.add_argument(
'--meta-data',
type=argparse.FileType('r'),
help='meta-data file',
)
setcloudinit.add_argument(
'--network-config',
type=argparse.FileType('r'),
help='network-config file',
)
setcloudinit.set_defaults(func=commands.setcloudinit)
# delete subcommand
delete = subparsers.add_parser(
'delete',
parents=[common],
formatter_class=argparse.RawTextHelpFormatter,
help=get_doc(commands.delete).help,
description=get_doc(commands.delete).desc,
)
delete.add_argument('instance')
delete.add_argument(
'-y',
'--yes',
action='store_true',
default=False,
help='automatic yes to prompt',
)
delete.add_argument(
'--save-volumes',
action='store_true',
default=False,
help='do not delete local storage volumes',
)
delete.set_defaults(func=commands.delete)
return root
def run() -> None:
"""Run argument parser."""
parser = get_parser()
args = parser.parse_args()
if args.command is None:
parser.print_help()
sys.exit()
log_level = args.root_log_level or args.log_level or os.getenv('CMP_LOG')
if isinstance(log_level, str) and log_level.lower() in log_levels:
logging.basicConfig(
level=logging.getLevelNamesMapping()[log_level.upper()]
)
log.debug('CLI started with args: %s', args)
connect_uri = (
args.root_connect
or args.connect
or os.getenv('CMP_LIBVIRT_URI')
or os.getenv('LIBVIRT_DEFAULT_URI')
or 'qemu:///system'
)
try:
with Session(connect_uri) as session:
# Invoke command
args.func(session, args)
except ComputeError as e:
sys.exit(f'error: {e}')
except KeyboardInterrupt:
sys.exit()

77
compute/cli/term.py Normal file
View File

@ -0,0 +1,77 @@
# This file is part of Compute
#
# Compute is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Utils for creating terminal output and interface elements."""
import re
import sys
class Table:
"""Minimalistic text table constructor."""
def __init__(self, whitespace: str | None = None):
"""Initialise Table."""
self.whitespace = whitespace or '\t'
self.header = []
self.rows = []
self.table = ''
def add_row(self, row: list) -> None:
"""Add table row."""
self.rows.append([str(col) for col in row])
def add_rows(self, rows: list[list]) -> None:
"""Add multiple rows."""
for row in rows:
self.add_row(row)
def __str__(self) -> str:
"""Return table."""
widths = [max(map(len, col)) for col in zip(*self.rows, strict=True)]
self.rows.insert(0, [str(h).upper() for h in self.header])
for row in self.rows:
widths = widths or [len(i) for i in row]
self.table += self.whitespace.join(
(
val.ljust(width)
for val, width in zip(row, widths, strict=True)
)
)
self.table += '\n'
return self.table.strip()
def confirm(message: str, *, default: bool | None = None) -> None:
"""Start yes/no interactive dialog."""
while True:
match default:
case True:
prompt = 'default: yes'
case False:
prompt = 'default: no'
case _:
prompt = 'no default'
try:
answer = input(f'{message} ({prompt}) ')
except KeyboardInterrupt:
sys.exit('aborted')
if not answer and isinstance(default, bool):
return default
if re.match(r'^y(es)?$', answer, re.I):
return True
if re.match(r'^no?$', answer, re.I):
return False
print("Please respond 'yes' or 'no'")

121
compute/config.py Normal file
View File

@ -0,0 +1,121 @@
# This file is part of Compute
#
# Compute is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Configuration loader."""
__all__ = ['Config', 'ConfigSchema']
import os
import tomllib
from collections import UserDict
from pathlib import Path
from typing import ClassVar
from .abstract import EntityModel
from .exceptions import ConfigLoaderError
from .utils import dictutil
class LibvirtConfigSchema(EntityModel):
"""Schema for libvirt config."""
uri: str
class LogConfigSchema(EntityModel):
"""Logger congif schema."""
level: str | None = None
file: str | None = None
class StorageConfigSchema(EntityModel):
"""Storage config schema."""
volumes: str
images: str
class ConfigSchema(EntityModel):
"""Configuration file schema."""
libvirt: LibvirtConfigSchema | None
log: LogConfigSchema | None
storage: StorageConfigSchema
class Config(UserDict):
"""
UserDict for storing configuration.
Environment variables prefix is ``CMP_``. Environment variables
have higher proirity then configuration file.
:cvar str IMAGES_POOL: images storage pool name taken from env
:cvar str VOLUMES_POOL: volumes storage pool name taken from env
:cvar Path DEFAULT_CONFIG_FILE: :file:`/etc/computed/computed.toml`
:cvar dict DEFAULT_CONFIGURATION:
"""
LIBVIRT_URI = os.getenv('CMP_LIBVIRT_URI')
IMAGES_POOL = os.getenv('CMP_IMAGES_POOL')
VOLUMES_POOL = os.getenv('CMP_VOLUMES_POOL')
DEFAULT_CONFIG_FILE = Path('/etc/compute/computed.toml')
DEFAULT_CONFIGURATION: ClassVar[dict] = {
'libvirt': {
'uri': 'qemu:///system',
},
'log': {
'level': None,
'file': None,
},
'storage': {
'images': 'images',
'volumes': 'volumes',
},
}
def __init__(self, file: Path | None = None):
"""
Initialise Config.
:param file: Path to configuration file. If `file` is None
use default path from :var:`Config.DEFAULT_CONFIG_FILE`.
"""
self.file = Path(file) if file else self.DEFAULT_CONFIG_FILE
try:
if self.file.exists():
with self.file.open('rb') as configfile:
loaded = tomllib.load(configfile)
else:
loaded = {}
except tomllib.TOMLDecodeError as etoml:
raise ConfigLoaderError(
f'Bad TOML syntax: {self.file}: {etoml}'
) from etoml
except (OSError, ValueError) as eread:
raise ConfigLoaderError(
f'Config read error: {self.file}: {eread}'
) from eread
config = dictutil.override(self.DEFAULT_CONFIGURATION, loaded)
if self.LIBVIRT_URI:
config['libvirt']['uri'] = self.LIBVIRT_URI
if self.VOLUMES_POOL:
config['storage']['volumes'] = self.VOLUMES_POOL
if self.IMAGES_POOL:
config['storage']['images'] = self.IMAGES_POOL
ConfigSchema(**config)
super().__init__(config)

View File

@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Exceptions."""
@ -36,12 +36,12 @@ class GuestAgentUnavailableError(GuestAgentError):
"""Guest agent is not connected or is unavailable."""
class GuestAgentTimeoutExceededError(GuestAgentError):
"""QEMU timeout exceeded."""
class GuestAgentTimeoutExpired(GuestAgentError): # noqa: N818
"""QEMU timeout expired."""
def __init__(self, msg: int):
"""Initialise GuestAgentTimeoutExceededError."""
super().__init__(f'QEMU timeout ({msg} sec) exceeded')
def __init__(self, seconds: int):
"""Initialise GuestAgentTimeoutExpired."""
super().__init__(f'QEMU timeout ({seconds} sec) expired')
class GuestAgentCommandNotSupportedError(GuestAgentError):
@ -78,3 +78,34 @@ class InstanceNotFoundError(InstanceError):
def __init__(self, msg: str):
"""Initialise InstanceNotFoundError."""
super().__init__(f"compute instance '{msg}' not found")
class InvalidDeviceConfigError(ComputeError):
"""
Invalid device XML description.
:class:`DeviceCoonfig` instance cannot be created because
device config in libvirt XML config is not valid.
"""
def __init__(self, msg: str, xml: str):
"""Initialise InvalidDeviceConfigError."""
self.msg = f'Invalid device XML config: {msg}'
self.loc = f' {xml}'
super().__init__(f'{self.msg}:\n{self.loc}')
class InvalidDataUnitError(ValueError, ComputeError):
"""Data unit is not valid."""
def __init__(self, msg: str, units: list):
"""Initialise InvalidDataUnitError."""
super().__init__(f'{msg}, valid units are: {", ".join(units)}')
class DictMergeConflictError(ComputeError):
"""Conflict when merging dicts."""
def __init__(self, key: str):
"""Initialise DictMergeConflictError."""
super().__init__(f'Conflicting key: {key}')

View File

@ -5,14 +5,15 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
from .cloud_init import CloudInit
from .guest_agent import GuestAgent
from .instance import Instance, InstanceConfig
from .schemas import InstanceSchema

View File

@ -0,0 +1,222 @@
# This file is part of Compute
#
# Compute is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
# ruff: noqa: S603
"""
`Cloud-init`_ integration for bootstraping compute instances.
.. _Cloud-init: https://cloudinit.readthedocs.io
"""
import logging
import subprocess
import tempfile
from pathlib import Path
from compute.exceptions import InstanceError
from .devices import DiskConfig, DiskDriver
from .instance import Instance
log = logging.getLogger(__name__)
class CloudInit:
"""
Cloud-init integration.
:ivar str user_data: user-data.
:ivar str vendor_data: vendor-data.
:ivar str network_config: network-config.
:ivar str meta_data: meta-data.
"""
def __init__(self):
"""Initialise :class:`CloudInit`."""
self.user_data = None
self.vendor_data = None
self.network_config = None
self.meta_data = None
def __repr__(self) -> str:
"""Represent :class:`CloudInit` object."""
return (
self.__class__.__name__
+ '('
+ ', '.join(
[
f'{self.user_data=}',
f'{self.vendor_data=}',
f'{self.network_config=}',
f'{self.meta_data=}',
]
)
+ ')'
).replace('self.', '')
def _write_to_disk(
self,
disk: str,
filename: str,
data: str | None,
*,
force_file_create: bool = False,
delete_existing_file: bool = False,
default_data: str | None = None,
) -> None:
data = data or default_data
log.debug('Input data %s: %r', filename, data)
if isinstance(data, str):
data = data.encode()
if data is None and force_file_create is False:
return
with tempfile.NamedTemporaryFile() as data_file:
if data is not None:
data_file.write(data)
data_file.flush()
if delete_existing_file:
log.debug('Deleting existing file')
filelist = subprocess.run(
['/usr/bin/mdir', '-i', disk, '-b'],
capture_output=True,
check=True,
)
files = [
f.replace('::/', '')
for f in filelist.stdout.decode().splitlines()
]
log.debug('Files on disk: %s', files)
log.debug("Removing '%s'", filename)
if filename in files:
subprocess.run(
['/usr/bin/mdel', '-i', disk, f'::{filename}'],
check=True,
)
log.debug("Writing file '%s'", filename)
subprocess.run(
[
'/usr/bin/mcopy',
'-i',
disk,
data_file.name,
f'::{filename}',
],
check=True,
)
def create_disk(self, disk: Path, *, force: bool = False) -> None:
"""
Create disk with cloud-init config files.
:param path: Disk path.
:param force: Replace existing disk.
"""
if not isinstance(disk, Path):
disk = Path(disk)
if disk.exists():
if disk.is_file() is False:
raise InstanceError('Cloud-init disk must be regular file')
if force:
disk.unlink()
else:
raise InstanceError('File already exists')
subprocess.run(
['/usr/sbin/mkfs.vfat', '-n', 'CIDATA', '-C', str(disk), '1024'],
check=True,
stdout=subprocess.DEVNULL,
)
self._write_to_disk(
disk=disk,
filename='user-data',
data=self.user_data,
force_file_create=True,
default_data='#cloud-config',
)
self._write_to_disk(
disk=disk,
filename='vendor-data',
data=self.vendor_data,
)
self._write_to_disk(
disk=disk,
filename='network-config',
data=self.network_config,
)
self._write_to_disk(
disk=disk,
filename='meta-data',
data=self.meta_data,
force_file_create=True,
)
def update_disk(self, disk: Path) -> None:
"""Update files on existing disk."""
if not isinstance(disk, Path):
disk = Path(disk)
if not disk.exists():
raise InstanceError(f"File '{disk}' does not exists")
if self.user_data:
self._write_to_disk(
disk=disk,
filename='user-data',
data=self.user_data,
force_file_create=True,
default_data='#cloud-config',
delete_existing_file=True,
)
if self.vendor_data:
self._write_to_disk(
disk=disk,
filename='vendor-data',
data=self.vendor_data,
delete_existing_file=True,
)
if self.network_config:
self._write_to_disk(
disk=disk,
filename='network-config',
data=self.network_config,
delete_existing_file=True,
)
if self.meta_data:
self._write_to_disk(
disk=disk,
filename='meta-data',
data=self.meta_data,
force_file_create=True,
delete_existing_file=True,
)
def attach_disk(self, disk: Path, target: str, instance: Instance) -> None:
"""
Attach cloud-init disk to instance.
:param disk: Path to disk.
:param target: Disk target name e.g. `vda`.
:param instance: Compute instance object.
"""
instance.attach_device(
DiskConfig(
type='file',
device='disk',
source=disk,
target=target,
is_readonly=True,
bus='virtio',
driver=DiskDriver('qemu', 'raw'),
)
)

125
compute/instance/devices.py Normal file
View File

@ -0,0 +1,125 @@
# This file is part of Compute
#
# Compute is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
# ruff: noqa: SIM211, UP007, A003
"""Virtual devices configs."""
from dataclasses import dataclass, field
from pathlib import Path
from typing import Union
from lxml import etree
from lxml.builder import E
from compute.abstract import DeviceConfig
from compute.exceptions import InvalidDeviceConfigError
@dataclass
class DiskDriver:
"""Disk driver description for libvirt."""
name: str = 'qemu'
type: str = 'qcow2'
cache: str = 'default'
def __call__(self):
"""Return self."""
return self
@dataclass
class DiskConfig(DeviceConfig):
"""
Disk config builder.
Generate XML config for attaching or detaching storage volumes
to compute instances.
"""
type: str
source: str | Path
target: str
is_readonly: bool = False
device: str = 'disk'
bus: str = 'virtio'
driver: DiskDriver = field(default_factory=DiskDriver())
def to_xml(self) -> str:
"""Return XML config for libvirt."""
xml = E.disk(type=self.type, device=self.device)
xml.append(
E.driver(
name=self.driver.name,
type=self.driver.type,
cache=self.driver.cache,
)
)
if self.source and self.type == 'file':
xml.append(E.source(file=str(self.source)))
xml.append(E.target(dev=self.target, bus=self.bus))
if self.is_readonly:
xml.append(E.readonly())
return etree.tostring(xml, encoding='unicode', pretty_print=True)
@classmethod
def from_xml(cls, xml: Union[str, etree.Element]) -> 'DiskConfig':
"""
Create :class:`DiskConfig` instance from XML config.
:param xml: Disk device XML configuration as :class:`str` or lxml
:class:`etree.Element` object.
"""
if isinstance(xml, str):
xml_str = xml
xml = etree.fromstring(xml)
else:
xml_str = etree.tostring(
xml,
encoding='unicode',
pretty_print=True,
).strip()
source = xml.find('source')
target = xml.find('target')
driver = xml.find('driver')
cachetype = driver.get('cache')
disk_params = {
'type': xml.get('type'),
'device': xml.get('device'),
'driver': DiskDriver(
name=driver.get('name'),
type=driver.get('type'),
**({'cache': cachetype} if cachetype else {}),
),
'source': source.get('file') if source is not None else None,
'target': target.get('dev') if target is not None else None,
'bus': target.get('bus') if target is not None else None,
'is_readonly': False if xml.find('readonly') is None else True,
}
for param in disk_params:
if disk_params[param] is None:
msg = f"missing tag '{param}'"
raise InvalidDeviceConfigError(msg, xml_str)
if param == 'driver':
driver = disk_params[param]
for driver_param in [driver.name, driver.type, driver.cache]:
if driver_param is None:
msg = (
"'driver' tag must have "
"'name' and 'type' attributes"
)
raise InvalidDeviceConfigError(msg, xml_str)
return cls(**disk_params)

View File

@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Interacting with the QEMU Guest Agent."""
@ -27,7 +27,7 @@ import libvirt_qemu
from compute.exceptions import (
GuestAgentCommandNotSupportedError,
GuestAgentError,
GuestAgentTimeoutExceededError,
GuestAgentTimeoutExpired,
GuestAgentUnavailableError,
)
@ -114,7 +114,7 @@ class GuestAgent:
if command not in supported:
raise GuestAgentCommandNotSupportedError(command)
def guest_exec( # noqa: PLR0913
def guest_exec(
self,
path: str,
args: list[str] | None = None,
@ -199,7 +199,7 @@ class GuestAgent:
sleep(poll_interval)
now = time()
if now - start_time > self.timeout:
raise GuestAgentTimeoutExceededError(self.timeout)
raise GuestAgentTimeoutExpired(self.timeout)
log.debug(
'Polling command pid=%s finished, time taken: %s seconds',
pid,

View File

@ -5,33 +5,35 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Manage compute instances."""
__all__ = ['Instance', 'InstanceConfig', 'InstanceInfo']
import logging
import time
from typing import NamedTuple
from uuid import UUID
import libvirt
from lxml import etree
from lxml.builder import E
from compute.common import DeviceConfig, EntityConfig
from compute.abstract import DeviceConfig, EntityConfig
from compute.exceptions import (
GuestAgentCommandNotSupportedError,
InstanceError,
)
from compute.storage import DiskConfig
from compute.utils import units
from .devices import DiskConfig
from .guest_agent import GuestAgent
from .schemas import (
CPUEmulationMode,
@ -65,7 +67,7 @@ class InstanceConfig(EntityConfig):
self.emulator = schema.emulator
self.arch = schema.arch
self.boot = schema.boot
self.network_interfaces = schema.network_interfaces
self.network = schema.network
def _gen_cpu_xml(self, cpu: CPUSchema) -> etree.Element:
options = {
@ -118,6 +120,7 @@ class InstanceConfig(EntityConfig):
return E.interface(
E.source(network=interface.source),
E.mac(address=interface.mac),
E.model(type=interface.model),
type='network',
)
@ -140,6 +143,14 @@ class InstanceConfig(EntityConfig):
)
)
xml.append(self._gen_cpu_xml(self.cpu))
xml.append(
E.clock(
E.timer(name='rtc', tickpolicy='catchup'),
E.timer(name='pit', tickpolicy='delay'),
E.timer(name='hpet', present='no'),
offset='utc',
)
)
os = E.os(E.type('hvm', machine=self.machine, arch=self.arch))
for dev in self.boot.order:
os.append(E.boot(dev=dev))
@ -156,9 +167,10 @@ class InstanceConfig(EntityConfig):
)
devices = E.devices()
devices.append(E.emulator(str(self.emulator)))
for interface in self.network_interfaces:
devices.append(self._gen_network_interface_xml(interface))
devices.append(E.graphics(type='vnc', port='-1', autoport='yes'))
if self.network:
for interface in self.network.interfaces:
devices.append(self._gen_network_interface_xml(interface))
devices.append(E.graphics(type='vnc', autoport='yes'))
devices.append(E.input(type='tablet', bus='usb'))
devices.append(
E.channel(
@ -170,6 +182,7 @@ class InstanceConfig(EntityConfig):
type='unix',
)
)
devices.append(E.serial(E.target(port='0'), type='pty'))
devices.append(
E.console(E.target(type='serial', port='0'), type='pty')
)
@ -202,19 +215,40 @@ class Instance:
def __init__(self, domain: libvirt.virDomain):
"""
Initialise Instance.
:ivar libvirt.virDomain domain: domain object
:ivar libvirt.virConnect connection: connection object
:ivar str name: domain name
:ivar GuestAgent guest_agent: :class:`GuestAgent` object
Initialise Compute Instance object.
:param domain: libvirt domain object
"""
self.domain = domain
self.connection = domain.connect()
self.name = domain.name()
self.guest_agent = GuestAgent(domain)
self._domain = domain
self._connection = domain.connect()
self._name = domain.name()
self._uuid = domain.UUID()
self._guest_agent = GuestAgent(domain)
@property
def connection(self) -> libvirt.virConnect:
"""Libvirt connection object."""
return self._connection
@property
def domain(self) -> libvirt.virDomain:
"""Libvirt domain object."""
return self._domain
@property
def name(self) -> str:
"""Instance name."""
return self._name
@property
def uuid(self) -> UUID:
"""Instance UUID."""
return UUID(bytes=self._uuid)
@property
def guest_agent(self) -> GuestAgent:
""":class:`GuestAgent` object."""
return self._guest_agent
def _expand_instance_state(self, state: int) -> str:
states = {
@ -257,10 +291,9 @@ class Instance:
def is_running(self) -> bool:
"""Return True if instance is running, else return False."""
if self.domain.isActive() != 1:
# 0 - is inactive, -1 - is error
return False
return True
if self.domain.isActive() == 1:
return True
return False
def is_autostart(self) -> bool:
"""Return True if instance autostart is enabled, else return False."""
@ -278,21 +311,24 @@ class Instance:
def get_max_vcpus(self) -> int:
"""Maximum vCPUs number for domain."""
if not self.is_running():
xml = etree.fromstring(self.dump_xml(inactive=True))
return int(xml.xpath('/domain/vcpu/text()')[0])
return self.domain.maxVcpus()
def start(self) -> None:
"""Start defined instance."""
log.info('Starting instnce=%s', self.name)
log.info("Starting instance '%s'", self.name)
if self.is_running():
log.warning(
'Already started, nothing to do instance=%s', self.name
"Instance '%s' is already started, nothing to do", self.name
)
return
try:
self.domain.create()
except libvirt.libvirtError as e:
raise InstanceError(
f'Cannot start instance={self.name}: {e}'
f"Cannot start instance '{self.name}': {e}"
) from e
def shutdown(self, method: str | None = None) -> None:
@ -314,19 +350,21 @@ class Instance:
to unplugging machine from power. Internally send SIGTERM to
instance process and destroy it gracefully.
UNSAFE
Force shutdown. Internally send SIGKILL to instance process.
DESTROY
Forced shutdown. Internally send SIGKILL to instance process.
There is high data corruption risk!
If method is None NORMAL method will used.
:param method: Method used to shutdown instance
"""
if not self.is_running():
return
methods = {
'SOFT': libvirt.VIR_DOMAIN_SHUTDOWN_GUEST_AGENT,
'NORMAL': libvirt.VIR_DOMAIN_SHUTDOWN_DEFAULT,
'HARD': libvirt.VIR_DOMAIN_DESTROY_GRACEFUL,
'UNSAFE': libvirt.VIR_DOMAIN_DESTROY_DEFAULT,
'DESTROY': libvirt.VIR_DOMAIN_DESTROY_DEFAULT,
}
if method is None:
method = 'NORMAL'
@ -337,14 +375,17 @@ class Instance:
method = method.upper()
if method not in methods:
raise ValueError(f"Unsupported shutdown method: '{method}'")
if method == 'SOFT' and self.guest_agent.is_available() is False:
method = 'NORMAL'
log.info("Performing instance shutdown with method '%s'", method)
try:
if method in ['SOFT', 'NORMAL']:
self.domain.shutdownFlags(flags=methods[method])
elif method in ['HARD', 'UNSAFE']:
elif method in ['HARD', 'DESTROY']:
self.domain.destroyFlags(flags=methods[method])
except libvirt.libvirtError as e:
raise InstanceError(
f'Cannot shutdown instance={self.name} ' f'{method=}: {e}'
f"Cannot shutdown instance '{self.name}' with '{method=}': {e}"
) from e
def reboot(self) -> None:
@ -373,7 +414,7 @@ class Instance:
self.domain.reset()
except libvirt.libvirtError as e:
raise InstanceError(
f'Cannot reset instance={self.name}: {e}'
f"Cannot reset instance '{self.name}': {e}"
) from e
def power_reset(self) -> None:
@ -387,7 +428,13 @@ class Instance:
configuration change in libvirt and you need to restart the
instance to apply the new configuration.
"""
self.shutdown(method='NORMAL')
log.debug("Performing power reset for instance '%s'", self.name)
self.shutdown('NORMAL')
time.sleep(3)
# TODO @ge: do safe shutdown insted of this shit
if self.is_running():
self.shutdown('HARD')
time.sleep(1)
self.start()
def set_autostart(self, *, enabled: bool) -> None:
@ -401,8 +448,7 @@ class Instance:
self.domain.setAutostart(autostart)
except libvirt.libvirtError as e:
raise InstanceError(
f'Cannot set autostart flag for instance={self.name} '
f'{autostart=}: {e}'
f"Cannot set {autostart=} flag for instance '{self.name}': {e}"
) from e
def set_vcpus(self, nvcpus: int, *, live: bool = False) -> None:
@ -424,7 +470,7 @@ class Instance:
raise InstanceError('vCPUs count is greather than max_vcpus')
if nvcpus == self.get_info().nproc:
log.warning(
'Instance instance=%s already have %s vCPUs, nothing to do',
"Instance '%s' already have %s vCPUs, nothing to do",
self.name,
nvcpus,
)
@ -450,18 +496,17 @@ class Instance:
self.domain.setVcpusFlags(nvcpus, flags=flags)
except GuestAgentCommandNotSupportedError:
log.warning(
'Cannot set vCPUs in guest via agent, you may '
'need to apply changes in guest manually.'
"'guest-set-vcpus' command is not supported, '"
'you may need to enable CPUs in guest manually.'
)
else:
log.warning(
'Cannot set vCPUs in guest OS on instance=%s. '
'You may need to apply CPUs in guest manually.',
self.name,
'Guest agent is not installed or not connected, '
'you may need to enable CPUs in guest manually.'
)
except libvirt.libvirtError as e:
raise InstanceError(
f'Cannot set vCPUs for instance={self.name}: {e}'
f"Cannot set vCPUs for instance '{self.name}': {e}"
) from e
def set_memory(self, memory: int, *, live: bool = False) -> None:
@ -498,11 +543,6 @@ class Instance:
msg = f'Cannot set memory for instance={self.name} {memory=}: {e}'
raise InstanceError(msg) from e
def _get_disk_by_target(self, target: str) -> etree.Element:
xml = etree.fromstring(self.dump_xml()) # noqa: S320
child = xml.xpath(f'/domain/devices/disk/target[@dev="{target}"]')
return child[0].getparent() if child else None
def attach_device(
self, device: DeviceConfig, *, live: bool = False
) -> None:
@ -520,7 +560,7 @@ class Instance:
else:
flags = libvirt.VIR_DOMAIN_AFFECT_CONFIG
if isinstance(device, DiskConfig): # noqa: SIM102
if self._get_disk_by_target(device.target):
if self.get_disk(device.target):
log.warning(
"Volume with target '%s' is already attached",
device.target,
@ -545,7 +585,7 @@ class Instance:
else:
flags = libvirt.VIR_DOMAIN_AFFECT_CONFIG
if isinstance(device, DiskConfig): # noqa: SIM102
if self._get_disk_by_target(device.target) is None:
if self.get_disk(device.target) is None:
log.warning(
"Volume with target '%s' is already detached",
device.target,
@ -553,38 +593,56 @@ class Instance:
return
self.domain.detachDeviceFlags(device.to_xml(), flags=flags)
def detach_disk(self, name: str) -> None:
def get_disk(
self, name: str, *, persistent: bool = False
) -> DiskConfig | None:
"""
Return :class:`DiskConfig` by disk target name.
Return None if disk with specified target not found.
:param name: Disk name e.g. `vda`, `sda`, etc. This name may
not match the name of the disk inside the guest OS.
:param persistent: If True get only persistent volumes described
in instance XML config.
"""
xml = etree.fromstring(self.dump_xml(inactive=persistent))
child = xml.xpath(f'/domain/devices/disk/target[@dev="{name}"]')
if len(child) == 0:
return None
return DiskConfig.from_xml(child[0].getparent())
def list_disks(self, *, persistent: bool = False) -> list[DiskConfig]:
"""
Return list of attached disk devices.
:param persistent: If True list only persistent volumes described
in instance XML config.
"""
xml = etree.fromstring(self.dump_xml(inactive=persistent))
disks = xml.xpath('/domain/devices/disk')
return [DiskConfig.from_xml(disk) for disk in disks]
def detach_disk(self, name: str, *, live: bool = False) -> None:
"""
Detach disk device by target name.
There is no ``attach_disk()`` method. Use :func:`attach_device`
with :class:`DiskConfig` as argument.
:param name: Disk name e.g. 'vda', 'sda', etc. This name may
:param name: Disk name e.g. `vda`, `sda`, etc. This name may
not match the name of the disk inside the guest OS.
:param live: Affect a running instance. Not supported for CDROM
devices.
"""
xml = self._get_disk_by_target(name)
if xml is None:
disk = self.get_disk(name, persistent=live)
if disk is None:
log.warning(
"Volume with target '%s' is already detached",
name,
)
return
disk_params = {
'disk_type': xml.get('type'),
'source': xml.find('source').get('file'),
'target': xml.find('target').get('dev'),
'readonly': False if xml.find('readonly') is None else True, # noqa: SIM211
}
for param in disk_params:
if disk_params[param] is None:
msg = (
f"Cannot detach volume with target '{name}': "
f"parameter '{param}' is not defined in libvirt XML "
'config on host.'
)
raise InstanceError(msg)
self.detach_device(DiskConfig(**disk_params), live=True)
self.detach_device(disk, live=live)
def resize_disk(
self, name: str, capacity: int, unit: units.DataUnit
@ -592,20 +650,18 @@ class Instance:
"""
Resize attached block device.
:param name: Disk device name e.g. `vda`, `sda`, etc.
:param name: Disk name e.g. `vda`, `sda`, etc. This name may
not match the name of the disk inside the guest OS.
:param capacity: New capacity.
:param unit: Capacity unit.
"""
# TODO @ge: check actual size before making changes
self.domain.blockResize(
name,
units.to_bytes(capacity, unit=unit),
flags=libvirt.VIR_DOMAIN_BLOCK_RESIZE_BYTES,
)
def get_disks(self) -> list[DiskConfig]:
"""Return list of attached disks."""
raise NotImplementedError
def pause(self) -> None:
"""Pause instance."""
if not self.is_running():
@ -616,31 +672,75 @@ class Instance:
"""Resume paused instance."""
self.domain.resume()
def get_ssh_keys(self, user: str) -> list[str]:
def list_ssh_keys(self, user: str) -> list[str]:
"""
Return list of SSH keys on guest for specific user.
Return list of authorized SSH keys in guest for specific user.
:param user: Username.
"""
raise NotImplementedError
self.guest_agent.raise_for_commands(['guest-ssh-get-authorized-keys'])
exc = self.guest_agent.guest_exec(
path='/bin/sh',
args=[
'-c',
(
'su -c "'
'if ! [ -f ~/.ssh/authorized_keys ]; then '
'mkdir -p ~/.ssh && touch ~/.ssh/authorized_keys; '
'fi" '
f'{user}'
),
],
capture_output=True,
decode_output=True,
poll=True,
)
log.debug(exc)
try:
return self.domain.authorizedSSHKeysGet(user)
except libvirt.libvirtError as e:
raise InstanceError(e) from e
def set_ssh_keys(self, user: str, ssh_keys: list[str]) -> None:
def set_ssh_keys(
self,
user: str,
keys: list[str],
*,
remove: bool = False,
append: bool = False,
) -> None:
"""
Add SSH keys to guest for specific user.
Add authorized SSH keys to guest for specific user.
:param user: Username.
:param ssh_keys: List of public SSH keys.
:param keys: List of authorized SSH keys.
:param append: Append keys to authorized SSH keys instead of
overriding authorized_keys file.
:param remove: Remove authorized keys listed in `keys` parameter.
"""
raise NotImplementedError
def delete_ssh_keys(self, user: str, ssh_keys: list[str]) -> None:
"""
Remove SSH keys from guest for specific user.
:param user: Username.
:param ssh_keys: List of public SSH keys.
"""
raise NotImplementedError
qemu_ga_commands = ['guest-ssh-add-authorized-keys']
if remove and append:
raise InstanceError(
"'append' and 'remove' parameters are mutually exclusive"
)
if not self.is_running():
raise InstanceError(
'Cannot add authorized SSH keys to inactive instance'
)
if append:
flags = libvirt.VIR_DOMAIN_AUTHORIZED_SSH_KEYS_SET_APPEND
elif remove:
flags = libvirt.VIR_DOMAIN_AUTHORIZED_SSH_KEYS_SET_REMOVE
qemu_ga_commands = ['guest-ssh-remove-authorized-keys']
else:
flags = 0
if keys.sort() == self.list_ssh_keys().sort():
return
self.guest_agent.raise_for_commands(qemu_ga_commands)
try:
self.domain.authorizedSSHKeysSet(user, keys, flags=flags)
except libvirt.libvirtError as e:
raise InstanceError(e) from e
def set_user_password(
self, user: str, password: str, *, encrypted: bool = False
@ -648,19 +748,16 @@ class Instance:
"""
Set new user password in guest OS.
This action performs by guest agent inside the guest.
This action is performed by guest agent inside the guest.
:param user: Username.
:param password: Password.
:param encrypted: Set it to True if password is already encrypted.
Right encryption method depends on guest OS.
"""
if not self.guest_agent.is_available():
raise InstanceError(
'Cannot change password: guest agent is unavailable'
)
self.guest_agent.raise_for_commands(['guest-set-user-password'])
flags = libvirt.VIR_DOMAIN_PASSWORD_ENCRYPTED if encrypted else 0
log.debug("Setting up password for user '%s'", user)
self.domain.setUserPassword(user, password, flags=flags)
def dump_xml(self, *, inactive: bool = False) -> str:
@ -668,8 +765,36 @@ class Instance:
flags = libvirt.VIR_DOMAIN_XML_INACTIVE if inactive else 0
return self.domain.XMLDesc(flags)
def delete(self) -> None:
"""Undefine instance."""
# TODO @ge: delete local disks
def delete(self, *, with_volumes: bool = False) -> None:
"""
Delete instance with local volumes.
:param with_volumes: If True delete local volumes with instance.
"""
log.info("Shutdown instance '%s'", self.name)
self.shutdown(method='HARD')
disks = self.list_disks(persistent=True)
log.debug('Disks list: %s', disks)
for disk in disks:
if with_volumes and disk.type == 'file':
try:
volume = self.connection.storageVolLookupByPath(
disk.source
)
except libvirt.libvirtError as e:
if e.get_error_code() == libvirt.VIR_ERR_NO_STORAGE_VOL:
log.warning(
"Volume '%s' not found, skipped",
disk.source,
)
continue
if volume.storagePoolLookupByVolume().name() == 'images':
log.info(
'Volume %s skipped because it is from images pool',
volume.path(),
)
continue
log.info('Delete volume: %s', volume.path())
volume.delete()
log.info('Undefine instance')
self.domain.undefine()

View File

@ -5,34 +5,27 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Compute instance related objects schemas."""
import re
from collections import Counter
from enum import StrEnum
from pathlib import Path
from pydantic import BaseModel, Extra, validator
from pydantic import validator
from compute.abstract import EntityModel
from compute.utils.units import DataUnit
class EntityModel(BaseModel):
"""Basic entity model."""
class Config:
"""Do not allow extra fields."""
extra = Extra.forbid
class CPUEmulationMode(StrEnum):
"""CPU emulation mode enumerated."""
@ -81,15 +74,52 @@ class VolumeCapacitySchema(EntityModel):
unit: DataUnit
class DiskCache(StrEnum):
"""Possible disk cache mechanisms enumeration."""
NONE = 'none'
WRITETHROUGH = 'writethrough'
WRITEBACK = 'writeback'
DIRECTSYNC = 'directsync'
UNSAFE = 'unsafe'
class DiskDriverSchema(EntityModel):
"""Virtual disk driver model."""
name: str
type: str # noqa: A003
cache: DiskCache = DiskCache.WRITETHROUGH
class DiskBus(StrEnum):
"""Possible disk buses enumeration."""
VIRTIO = 'virtio'
IDE = 'ide'
SATA = 'sata'
class VolumeSchema(EntityModel):
"""Storage volume model."""
type: VolumeType # noqa: A003
target: str
capacity: VolumeCapacitySchema
driver: DiskDriverSchema
capacity: VolumeCapacitySchema | None
source: str | None = None
is_readonly: bool = False
is_system: bool = False
bus: DiskBus = DiskBus.VIRTIO
device: str = 'disk'
class NetworkAdapterModel(StrEnum):
"""Network adapter models."""
VIRTIO = 'virtio'
E1000 = 'e1000'
RTL8139 = 'rtl8139'
class NetworkInterfaceSchema(EntityModel):
@ -97,6 +127,13 @@ class NetworkInterfaceSchema(EntityModel):
source: str
mac: str
model: NetworkAdapterModel
class NetworkSchema(EntityModel):
"""Network configuration schema."""
interfaces: list[NetworkInterfaceSchema]
class BootOptionsSchema(EntityModel):
@ -105,6 +142,15 @@ class BootOptionsSchema(EntityModel):
order: tuple
class CloudInitSchema(EntityModel):
"""Cloud-init config model."""
user_data: str | None = None
meta_data: str | None = None
vendor_data: str | None = None
network_config: str | None = None
class InstanceSchema(EntityModel):
"""Compute instance model."""
@ -121,15 +167,16 @@ class InstanceSchema(EntityModel):
arch: str
boot: BootOptionsSchema
volumes: list[VolumeSchema]
network_interfaces: list[NetworkInterfaceSchema]
network: NetworkSchema | None | bool
image: str | None = None
cloud_init: CloudInitSchema | None = None
@validator('name')
def _check_name(cls, value: str) -> str: # noqa: N805
if not re.match(r'^[a-z0-9_]+$', value):
if not re.match(r'^[a-z0-9_-]+$', value):
msg = (
'Name can contain only lowercase letters, numbers '
'and underscore.'
'Name must contain only lowercase letters, numbers, '
'minus sign and underscore.'
)
raise ValueError(msg)
return value
@ -148,18 +195,33 @@ class InstanceSchema(EntityModel):
if len([v for v in volumes if v.is_system is True]) != 1:
msg = 'volumes list must contain one system volume'
raise ValueError(msg)
vol_with_source = 0
for vol in volumes:
if vol.is_system is True and vol.is_readonly is True:
index = 0
for volume in volumes:
index += 1
if volume.source is None and volume.capacity is None:
msg = f"{index}: capacity is required if 'source' is unset"
raise ValueError(msg)
if volume.is_system is True and volume.is_readonly is True:
msg = 'volume marked as system cannot be readonly'
raise ValueError(msg)
if vol.source is not None:
vol_with_source += 1
sources = [v.source for v in volumes if v.source is not None]
targets = [v.target for v in volumes]
for item in [sources, targets]:
duplicates = Counter(item) - Counter(set(item))
if duplicates:
msg = f'find duplicate values: {list(duplicates)}'
raise ValueError(msg)
return volumes
@validator('network_interfaces')
def _check_network_interfaces(cls, value: list) -> list: # noqa: N805
if not value:
msg = 'Network interfaces list must contain at least one element'
@validator('network')
def _check_network(
cls, # noqa: N805
network: NetworkSchema | None | bool,
) -> NetworkSchema | None | bool:
if network is True:
msg = (
"'network' cannot be True, set it to False "
'or provide network configuration'
)
raise ValueError(msg)
return value
return network

View File

@ -5,19 +5,19 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Hypervisor session manager."""
import logging
import os
from contextlib import AbstractContextManager
from pathlib import Path
from types import TracebackType
from typing import Any, NamedTuple
from uuid import uuid4
@ -25,18 +25,23 @@ from uuid import uuid4
import libvirt
from lxml import etree
from .config import Config
from .exceptions import (
InstanceNotFoundError,
SessionError,
StoragePoolNotFoundError,
)
from .instance import Instance, InstanceConfig, InstanceSchema
from .storage import DiskConfig, StoragePool, VolumeConfig
from .utils import units
from .instance.cloud_init import CloudInit
from .instance.devices import DiskConfig, DiskDriver
from .storage import StoragePool, VolumeConfig
from .utils import diskutils, units
log = logging.getLogger(__name__)
config = Config()
class Capabilities(NamedTuple):
"""Store domain capabilities info."""
@ -71,27 +76,20 @@ class NodeInfo(NamedTuple):
class Session(AbstractContextManager):
"""
Hypervisor session context manager.
:cvar IMAGES_POOL: images storage pool name taken from env
:cvar VOLUMES_POOL: volumes storage pool name taken from env
"""
IMAGES_POOL = os.getenv('CMP_IMAGES_POOL')
VOLUMES_POOL = os.getenv('CMP_VOLUMES_POOL')
"""Hypervisor session context manager."""
def __init__(self, uri: str | None = None):
"""
Initialise session with hypervisor.
:ivar str uri: libvirt connection URI.
:ivar libvirt.virConnect connection: libvirt connection object.
:param uri: libvirt connection URI.
"""
self.uri = uri or 'qemu:///system'
self.connection = libvirt.open(self.uri)
log.debug('Config=%s', config)
self.LIBVIRT_URI = config['libvirt']['uri']
self.IMAGES_POOL = config['storage']['images']
self.VOLUMES_POOL = config['storage']['volumes']
self._uri = uri or self.LIBVIRT_URI
self._connection = libvirt.open(self._uri)
def __enter__(self):
"""Return Session object."""
@ -106,6 +104,16 @@ class Session(AbstractContextManager):
"""Close the connection when leaving the context."""
self.close()
@property
def uri(self) -> str:
"""Libvirt connection URI."""
return self._uri
@property
def connection(self) -> libvirt.virConnect:
"""Libvirt connection object."""
return self._connection
def close(self) -> None:
"""Close connection to libvirt daemon."""
self.connection.close()
@ -154,7 +162,7 @@ class Session(AbstractContextManager):
"""Return capabilities e.g. arch, virt, emulator, etc."""
prefix = '/domainCapabilities'
hprefix = f'{prefix}/cpu/mode[@name="host-model"]'
caps = etree.fromstring(self.connection.getDomainCapabilities()) # noqa: S320
caps = etree.fromstring(self.connection.getDomainCapabilities())
return Capabilities(
arch=caps.xpath(f'{prefix}/arch/text()')[0],
virt_type=caps.xpath(f'{prefix}/domain/text()')[0],
@ -164,7 +172,7 @@ class Session(AbstractContextManager):
cpu_vendor=caps.xpath(f'{hprefix}/vendor/text()')[0],
cpu_model=caps.xpath(f'{hprefix}/model/text()')[0],
cpu_features=self._cap_get_cpu_features(caps),
usable_cpus=self._cap_get_cpus(caps),
usable_cpus=self._cap_get_usable_cpus(caps),
)
def create_instance(self, **kwargs: Any) -> Instance:
@ -200,63 +208,116 @@ class Session(AbstractContextManager):
:param volumes: List of storage volume configs. For more info
see :class:`VolumeSchema`.
:type volumes: list[dict]
:param network_interfaces: List of virtual network interfaces
configs. See :class:`NetworkInterfaceSchema` for more info.
:param network: List of virtual network interfaces configs.
See :class:`NetworkSchema` for more info.
:type network_interfaces: list[dict]
:param cloud_init: Cloud-init configuration. See
:class:`CloudInitSchema` for info.
:type cloud_init: dict
"""
data = InstanceSchema(**kwargs)
config = InstanceConfig(data)
log.info('Define XML...')
log.info(config.to_xml())
self.connection.defineXML(config.to_xml())
log.info('Getting instance...')
log.info('Define instance XML')
log.debug(config.to_xml())
try:
self.connection.defineXML(config.to_xml())
except libvirt.libvirtError as e:
raise SessionError(f'Error defining instance: {e}') from e
log.info('Getting instance object...')
instance = self.get_instance(config.name)
log.info('Creating volumes...')
log.info('Start processing volumes...')
log.info('Connecting to images pool...')
images_pool = self.get_storage_pool(self.IMAGES_POOL)
images_pool.refresh()
log.info('Connecting to volumes pool...')
volumes_pool = self.get_storage_pool(self.VOLUMES_POOL)
volumes_pool.refresh()
disk_targets = []
for volume in data.volumes:
log.info('Creating volume=%s', volume)
capacity = units.to_bytes(
volume.capacity.value, volume.capacity.unit
)
log.info('Connecting to images pool...')
images_pool = self.get_storage_pool(self.IMAGES_POOL)
log.info('Connecting to volumes pool...')
volumes_pool = self.get_storage_pool(self.VOLUMES_POOL)
log.info('Processing volume=%s', volume)
log.info('Building volume configuration...')
capacity = None
disk_targets.append(volume.target)
if not volume.source:
vol_name = f'{uuid4()}.qcow2'
volume_name = f'{uuid4()}.qcow2'
else:
vol_name = volume.source
vol_conf = VolumeConfig(
name=vol_name,
path=str(volumes_pool.path.joinpath(vol_name)),
capacity=capacity,
)
log.info('Volume configuration is:\n %s', vol_conf.to_xml())
if volume.is_system is True and data.image:
log.info(
"Volume is marked as 'system', start cloning image..."
)
log.info('Get image %s', data.image)
image = images_pool.get_volume(data.image)
log.info('Cloning image into volumes pool...')
vol = volumes_pool.clone_volume(image, vol_conf)
log.info(
'Resize cloned volume to specified size: %s',
capacity,
)
vol.resize(capacity, unit=units.DataUnit.BYTES)
volume_name = volume.source
if volume.device == 'cdrom':
log.info('Volume %s is CDROM device', volume_name)
elif volume.source is not None:
log.info('Using volume %s as source', volume_name)
if volume.capacity:
capacity = units.to_bytes(
volume.capacity.value, volume.capacity.unit
)
log.info('Getting volume %s', volume.source)
vol = volumes_pool.get_volume(Path(volume_name).name)
log.info(
'Resize volume to specified size: %s',
capacity,
)
vol.resize(capacity, unit=units.DataUnit.BYTES)
else:
log.info('Create volume...')
volumes_pool.create_volume(vol_conf)
capacity = units.to_bytes(
volume.capacity.value, volume.capacity.unit
)
volume_config = VolumeConfig(
name=volume_name,
path=str(volumes_pool.path.joinpath(volume_name)),
capacity=capacity,
)
volume.source = volume_config.path
log.debug('Volume config: %s', volume_config)
if volume.is_system is True and data.image:
log.info(
"Volume is marked as 'system', start cloning image..."
)
log.info('Get image %s', data.image)
image = images_pool.get_volume(data.image)
log.info('Cloning image into volumes pool...')
vol = volumes_pool.clone_volume(image, volume_config)
log.info(
'Resize cloned volume to specified size: %s',
capacity,
)
vol.resize(capacity, unit=units.DataUnit.BYTES)
else:
log.info('Create volume %s', volume_config.name)
volumes_pool.create_volume(volume_config)
log.info('Attaching volume to instance...')
instance.attach_device(
DiskConfig(
disk_type=volume.type,
source=vol_conf.path,
type=volume.type,
device=volume.device,
source=volume.source,
target=volume.target,
readonly=volume.is_readonly,
is_readonly=volume.is_readonly,
bus=volume.bus,
driver=DiskDriver(
volume.driver.name,
volume.driver.type,
volume.driver.cache,
),
)
)
if data.cloud_init:
log.info('Crating disk for cloud-init...')
cloud_init = CloudInit()
cloud_init.user_data = data.cloud_init.user_data
cloud_init.vendor_data = data.cloud_init.vendor_data
cloud_init.network_config = data.cloud_init.network_config
cloud_init.meta_data = data.cloud_init.meta_data
cloud_init_disk_path = volumes_pool.path.joinpath(
f'{instance.name}-cloud-init.img'
)
cloud_init.create_disk(cloud_init_disk_path)
log.info('Attaching cloud-init disk to instance...')
volumes_pool.refresh()
cloud_init.attach_disk(
cloud_init_disk_path,
diskutils.get_disk_target(disk_targets, prefix='vd'),
instance,
)
return instance
def get_instance(self, name: str) -> Instance:

View File

@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
from .pool import StoragePool
from .volume import DiskConfig, Volume, VolumeConfig
from .volume import Volume, VolumeConfig

View File

@ -5,17 +5,21 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Manage storage pools."""
import datetime
import logging
import time
from datetime import datetime as dt
from datetime import timedelta
from pathlib import Path
from typing import NamedTuple
@ -49,12 +53,12 @@ class StoragePool:
def _get_path(self) -> Path:
"""Return storage pool path."""
xml = etree.fromstring(self.pool.XMLDesc()) # noqa: S320
xml = etree.fromstring(self.pool.XMLDesc())
return Path(xml.xpath('/pool/target/path/text()')[0])
def get_usage_info(self) -> StoragePoolUsageInfo:
"""Return info about storage pool usage."""
xml = etree.fromstring(self.pool.XMLDesc()) # noqa: S320
xml = etree.fromstring(self.pool.XMLDesc())
return StoragePoolUsageInfo(
capacity=int(xml.xpath('/pool/capacity/text()')[0]),
allocation=int(xml.xpath('/pool/allocation/text()')[0]),
@ -65,10 +69,32 @@ class StoragePool:
"""Return storage pool XML description as string."""
return self.pool.XMLDesc()
def refresh(self) -> None:
"""Refresh storage pool."""
# TODO @ge: handle libvirt asynchronous job related exceptions
self.pool.refresh()
def refresh(self, *, retry: bool = True, timeout: int = 30) -> None:
"""
Refresh storage pool.
:param retry: If True retry pool refresh on 'pool have running
asynchronous jobs' error.
:param timeout: Retry timeout in seconds. Affects only if `retry`
is True.
"""
retry_timeout = dt.now(tz=datetime.UTC) + timedelta(seconds=timeout)
while dt.now(tz=datetime.UTC) < retry_timeout:
try:
self.pool.refresh()
except libvirt.libvirtError as e:
if 'asynchronous jobs running' in e.get_error_message():
if retry is False:
raise StoragePoolError(e) from e
log.debug(
'An error ocurred when refreshing storage pool '
'retrying after 1 sec...'
)
time.sleep(1)
else:
raise StoragePoolError(e) from e
else:
return
def create_volume(self, vol_conf: VolumeConfig) -> Volume:
"""Create storage volume and return Volume instance."""
@ -93,7 +119,7 @@ class StoragePool:
'src_pool=%s src_vol=%s dst_pool=%s dst_vol=%s',
src.pool_name,
src.name,
self.pool.name,
self.pool.name(),
dst.name,
)
vol = self.pool.createXMLFrom(
@ -108,7 +134,9 @@ class StoragePool:
def get_volume(self, name: str) -> Volume | None:
"""Lookup and return Volume instance or None."""
log.info(
'Lookup for storage volume vol=%s in pool=%s', name, self.pool.name
'Lookup for storage volume vol=%s in pool=%s',
name,
self.pool.name(),
)
try:
vol = self.pool.storageVolLookupByName(name)

View File

@ -5,13 +5,13 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Manage storage volumes."""
@ -23,7 +23,7 @@ import libvirt
from lxml import etree
from lxml.builder import E
from compute.common import DeviceConfig, EntityConfig
from compute.abstract import EntityConfig
from compute.utils import units
@ -63,32 +63,6 @@ class VolumeConfig(EntityConfig):
return etree.tostring(xml, encoding='unicode', pretty_print=True)
@dataclass
class DiskConfig(DeviceConfig):
"""
Disk XML config builder.
Generate XML config for attaching or detaching storage volumes
to compute instances.
"""
disk_type: str
source: str | Path
target: str
readonly: bool = False
def to_xml(self) -> str:
"""Return XML config for libvirt."""
xml = E.disk(type=self.disk_type, device='disk')
xml.append(E.driver(name='qemu', type='qcow2', cache='writethrough'))
if self.disk_type == 'file':
xml.append(E.source(file=str(self.source)))
xml.append(E.target(dev=self.target, bus='virtio'))
if self.readonly:
xml.append(E.readonly())
return etree.tostring(xml, encoding='unicode', pretty_print=True)
class Volume:
"""Storage volume manipulating class."""

View File

@ -1,56 +0,0 @@
# This file is part of Compute
#
# Compute is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
"""Configuration loader."""
import tomllib
from collections import UserDict
from pathlib import Path
from compute.exceptions import ConfigLoaderError
DEFAULT_CONFIGURATION = {}
DEFAULT_CONFIG_FILE = '/etc/computed/computed.toml'
class ConfigLoader(UserDict):
"""UserDict for storing configuration."""
def __init__(self, file: Path | None = None):
"""
Initialise ConfigLoader.
:param file: Path to configuration file. If `file` is None
use default path from DEFAULT_CONFIG_FILE constant.
"""
# TODO @ge: load deafult configuration
self.file = Path(file) if file else Path(DEFAULT_CONFIG_FILE)
super().__init__(self.load())
def load(self) -> dict:
"""Load confguration object from TOML file."""
try:
with Path(self.file).open('rb') as configfile:
return tomllib.load(configfile)
# TODO @ge: add config schema validation
except tomllib.TOMLDecodeError as tomlerr:
raise ConfigLoaderError(
f'Bad TOML syntax in config file: {self.file}: {tomlerr}'
) from tomlerr
except (OSError, ValueError) as readerr:
raise ConfigLoaderError(
f'Cannot read config file: {self.file}: {readerr}'
) from readerr

77
compute/utils/dictutil.py Normal file
View File

@ -0,0 +1,77 @@
# This file is part of Compute
#
# Compute is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Dict tools."""
from compute.exceptions import DictMergeConflictError
def merge(a: dict, b: dict, path: list[str] | None = None) -> dict:
"""
Merge `b` into `a`. Return modified `a`.
:raise: :class:`DictMergeConflictError`
"""
if path is None:
path = []
for key in b:
if key in a:
if isinstance(a[key], dict) and isinstance(b[key], dict):
merge(a[key], b[key], [*path, str(key)])
elif a[key] != b[key]:
raise DictMergeConflictError('.'.join([*path, str(key)]))
else:
a[key] = b[key]
return a
def override(a: dict, b: dict) -> dict:
"""
Override dict `a` by `b` values.
Keys that not exists in `a`, but exists in `b` will be
appended to `a`.
.. code-block:: shell-session
>>> from compute.utils import dictutil
>>> default = {
... 'bus': 'virtio',
... 'driver': {'name': 'qemu', 'type': 'qcow2'}
... }
>>> user = {
... 'bus': 'ide',
... 'target': 'vda',
... 'driver': {'type': 'raw'}
... }
>>> dictutil.override(default, user)
{'bus': 'ide', 'driver': {'name': 'qemu', 'type': 'raw'},
'target': 'vda'}
NOTE: merging dicts contained in lists is not supported.
:param a: Dict to be overwritten.
:param b: A dict whose values will be used to rewrite dict `a`.
:return: Modified `a` dict.
"""
for key in b:
if key in a:
if isinstance(a[key], dict) and isinstance(b[key], dict):
override(a[key], b[key])
else:
a[key] = b[key] # replace existing key's values
else:
a[key] = b[key]
return a

View File

@ -0,0 +1,45 @@
# This file is part of Compute
#
# Compute is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Auxiliary functions for working with disks."""
import string
def get_disk_target(
disks: list[str], prefix: str, *, from_end: bool = False
) -> str:
"""
Return free disk name.
.. code-block:: shell-session
>>> get_disk_target(['vda', 'vdb'], 'vd')
'vdc'
>>> get_disk_target(['vda', 'vdc'], 'vd')
'vdb'
>>> get_disk_target(['vda', 'vdd'], 'vd', from_end=True)
'vdz'
>>> get_disk_target(['vda', 'hda'], 'hd')
'hdb'
:param disks: List of attached disk names.
:param prefix: Disk name prefix.
:param from_end: If True select a drive letter starting from the
end of the alphabet.
"""
index = -1 if from_end else 0
devs = [d[-1] for d in disks if d.startswith(prefix)]
return prefix + [x for x in string.ascii_lowercase if x not in devs][index]

View File

@ -5,29 +5,29 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Random identificators."""
# ruff: noqa: S311, C417
# ruff: noqa: S311
import random
def random_mac() -> str:
"""Retrun random MAC address."""
mac = [
0x00,
0x16,
0x3E,
random.randint(0x00, 0x7F),
bits = [
0x0A,
random.randint(0x00, 0xFF),
random.randint(0x00, 0xFF),
random.randint(0x00, 0xFF),
random.randint(0x00, 0xFF),
random.randint(0x00, 0xFF),
]
return ':'.join(map(lambda x: '%02x' % x, mac))
return ':'.join([f'{b:02x}' for b in bits])

View File

@ -5,50 +5,115 @@
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
"""Tools for data units convertion."""
from collections.abc import Callable
from enum import StrEnum
from compute.exceptions import InvalidDataUnitError
class DataUnit(StrEnum):
"""Data units enumerated."""
"""Data units enumeration."""
BYTES = 'bytes'
KIB = 'KiB'
MIB = 'MiB'
GIB = 'GiB'
TIB = 'TiB'
KB = 'kb'
MB = 'Mb'
GB = 'Gb'
TB = 'Tb'
KBIT = 'kbit'
MBIT = 'Mbit'
GBIT = 'Gbit'
TBIT = 'Tbit'
@classmethod
def _missing_(cls, name: str) -> 'DataUnit':
for member in cls:
if member.name.lower() == name.lower():
return member
return None
class InvalidDataUnitError(ValueError):
"""Data unit is not valid."""
def validate_input(*args: str) -> Callable:
"""Validate data units in functions input."""
to_validate = args
def __init__(self, msg: str):
"""Initialise InvalidDataUnitError."""
super().__init__(
f'{msg}, valid units are: {", ".join(list(DataUnit))}'
)
def decorator(func: Callable) -> Callable:
def wrapper(*args: float | str, **kwargs: str) -> Callable:
try:
if kwargs:
for arg in to_validate:
unit = kwargs[arg]
DataUnit(unit)
else:
for arg in args[1:]:
unit = arg
DataUnit(unit)
except ValueError as e:
raise InvalidDataUnitError(e, list(DataUnit)) from e
return func(*args, **kwargs)
return wrapper
return decorator
def to_bytes(value: int, unit: DataUnit = DataUnit.BYTES) -> int:
"""Convert value to bytes. See :class:`DataUnit`."""
try:
_ = DataUnit(unit)
except ValueError as e:
raise InvalidDataUnitError(e) from e
powers = {
@validate_input('unit')
def to_bytes(value: float, unit: DataUnit = DataUnit.BYTES) -> float:
"""Convert value to bytes."""
unit = DataUnit(unit)
basis = 2 if unit.endswith('iB') else 10
factor = 125 if unit.endswith('bit') else 1
power = {
DataUnit.BYTES: 0,
DataUnit.KIB: 1,
DataUnit.MIB: 2,
DataUnit.GIB: 3,
DataUnit.TIB: 4,
DataUnit.KIB: 10,
DataUnit.MIB: 20,
DataUnit.GIB: 30,
DataUnit.TIB: 40,
DataUnit.KB: 3,
DataUnit.MB: 6,
DataUnit.GB: 9,
DataUnit.TB: 12,
DataUnit.KBIT: 0,
DataUnit.MBIT: 3,
DataUnit.GBIT: 6,
DataUnit.TBIT: 9,
}
return value * pow(1024, powers[unit])
return value * factor * pow(basis, power[unit])
@validate_input('from_unit', 'to_unit')
def convert(value: float, from_unit: DataUnit, to_unit: DataUnit) -> float:
"""Convert units."""
value_in_bits = to_bytes(value, from_unit) * 8
to_unit = DataUnit(to_unit)
basis = 2 if to_unit.endswith('iB') else 10
divisor = 1 if to_unit.endswith('bit') else 8
power = {
DataUnit.BYTES: 0,
DataUnit.KIB: 10,
DataUnit.MIB: 20,
DataUnit.GIB: 30,
DataUnit.TIB: 40,
DataUnit.KB: 3,
DataUnit.MB: 6,
DataUnit.GB: 9,
DataUnit.TB: 12,
DataUnit.KBIT: 3,
DataUnit.MBIT: 6,
DataUnit.GBIT: 9,
DataUnit.TBIT: 12,
}
return value_in_bits / divisor / pow(basis, power[to_unit])

13
computed.toml Normal file
View File

@ -0,0 +1,13 @@
[libvirt]
# Libvirt connection URI.
# See https://libvirt.org/uri.html#qemu-qemu-and-kvm-uris
#uri = 'qemu:///system'
[storage]
# Name of libvirt storage pool to store compute instance etalon images.
# compute takes images from here and creates disks for compute instances
# based on them.
#images = 'images'
# Name of libvirt storage pool to store compute instance disks.
#volumes = 'volumes'

View File

@ -0,0 +1,2 @@
div.code-block-caption {background: #d0d0d0;}
a:visited {color: #004B6B;}

View File

@ -1,8 +1,8 @@
{% if versions %}
<h3>{{ _('Версии') }}</h3>
<h3 style="margin-top: 16px">{{ _('Versions') }}</h3>
<ul>
{%- for item in versions %}
<li><a href="{{ item.url }}">{{ item.name }}</a></li>
<li><a style="font-size: 120%" href="{{ item.url }}">{{ item.name }}</a></li>
{%- endfor %}
</ul>
{% endif %}

View File

@ -0,0 +1,124 @@
Using Cloud-init
================
Cloud-init for new instances
----------------------------
Cloud-init configs may be set inplace into :file:`instance.yaml`.
.. code-block:: yaml
:caption: Example with Debian generic QCOW2 image
:linenos:
name: genericdebian
memory: 1024
vcpus: 1
image: debian-12-generic-amd64.qcow2
volumes:
- type: file
is_system: true
capacity:
value: 5
unit: GiB
cloud_init:
meta_data:
hostname: genericdebian
root_pass: secure_pass
user_data: |
## template: jinja
#cloud-config
hostname: {{ ds.meta_data.hostname }}
fqdn: {{ ds.meta_data.hostname }}.instances.generic.cloud
manage_etc_hosts: true
chpasswd:
users:
- name: root
password: {{ ds.meta_data.root_pass }}
type: text
expire: False
ssh_pwauth: True
package_update: true
package_upgrade: true
packages:
- qemu-guest-agent
- vim
- psmisc
- htop
runcmd:
- [ systemctl, daemon-reload ]
- [ systemctl, enable, qemu-guest-agent.service ]
- [ systemctl, start, --no-block, qemu-guest-agent.service ]
You can use separate file in this way:
.. code-block:: yaml
:caption: user-data in separate file
:emphasize-lines: 11-
:linenos:
name: genericdebian
memory: 1024
vcpus: 1
image: debian-12-generic-amd64.qcow2
volumes:
- type: file
is_system: true
capacity:
value: 25
unit: GiB
cloud_init:
user_data: user-data.yaml
Base64 encoded string with data must be ``base64:`` prefixed:
.. code-block:: yaml
:caption: user-data as base64 encoded string
:emphasize-lines: 11-
:linenos:
name: genericdebian
memory: 1024
vcpus: 1
image: debian-12-generic-amd64.qcow2
volumes:
- type: file
is_system: true
capacity:
value: 25
unit: GiB
cloud_init:
user_data: base64:I2Nsb3VkLWNvbmZpZwpob3N0bmFtZTogY2xvdWRlYmlhbgpmcWRuOiBjbG91ZGViaWFuLmV4YW1wbGUuY29tCm1hbmFnZV9ldGNfaG9zdHM6IHRydWUK
Also you can write config in YAML. Please note that in this case you will not be able to use the ``#cloud-config`` shebang.
.. code-block:: yaml
:caption: meta-data as nested YAML
:emphasize-lines: 12-14
:linenos:
name: genericdebian
memory: 1024
vcpus: 1
image: debian-12-generic-amd64.qcow2
volumes:
- type: file
is_system: true
capacity:
value: 25
unit: GiB
cloud_init:
meta_data:
myvar: example
another_one: example_2
user_data: |
#cloud-config
#something here
Edit Cloud-init config files on existing instance
-------------------------------------------------
Use ``setcloudinit`` subcommand::
compute setcloudinit myinstance --user-data user_data.yaml
See `setcloudinit <../cli/reference.html#setcloudinit>`_ for details.

View File

@ -0,0 +1,132 @@
Getting started
===============
Creating compute instances
--------------------------
Compute instances are created through a description in yaml format. The description may be partial, the configuration will be supplemented with default parameters.
This page describes how to start up a basic instance, you'll probably want to use cloud-init to get the guest up and running, see the instructions at `Using cloud-init <cloud_init.html>`_.
The following examples contains minimal instance configuration. See also full example `here <instance_file.html>`_
Using prebuilt QCOW2 disk image
```````````````````````````````
First place your image into ``images`` pool path.
Create :file:`instance.yaml` config file with following content. Replace `debian_12.qcow2` with your actual image filename.
.. code-block:: yaml
:caption: Using prebuilt QCOW2 disk image
:emphasize-lines: 4
:linenos:
name: myinstance
memory: 2048
vcpus: 2
image: debian_12.qcow2
volumes:
- type: file
is_system: true
capacity:
value: 10
unit: GiB
Check out what configuration will be applied when ``init``::
compute init --test
Initialise instance with command::
compute init
Also you can use following syntax::
compute init yourfile.yaml
Start instance::
compute start myinstance
Using ISO installation medium
`````````````````````````````
Download ISO image and set it as source for ``cdrom`` device.
Note that the ``image`` parameter is not used here.
.. code-block:: yaml
:caption: Using ISO image
:emphasize-lines: 10-12
:linenos:
name: myinstance
memory: 2048
vcpus: 2
volumes:
- type: file
is_system: true
capacity:
value: 10
unit: GiB
- type: file
device: cdrom
source: /images/debian-12.2.0-amd64-netinst.iso
::
compute init
Now edit instance XML configuration to add VNC-server listen address::
virsh edit myinstance
Add ``address`` attribute to start listen on all host network interfaces.
.. code-block:: xml
:caption: libvirt XML config fragment
:emphasize-lines: 2
<graphics type='vnc' port='-1' autoport='yes'>
<listen type='address' address='0.0.0.0'/>
</graphics>
Also you can specify VNC server port. This is **5900** by default.
Start instance and connect to VNC via any VNC client such as `Remmina <https://remmina.org/>`_ or something else.
::
compute start myinstance
Finish the OS installation over VNC and then do::
compute setcdrom myinstance --detach /images/debian-12.2.0-amd64-netinst.iso
compute powrst myinstance
CDROM will be detached. ``powrst`` command will perform instance shutdown and start. Instance will booted from `vda` disk.
Using existing disk
```````````````````
Place your disk image in ``volumes`` storage pool.
Replace `/volume/myvolume.qcow2` with actual path to disk.
.. code-block:: yaml
:caption: Using existing disk
:emphasize-lines: 7
:linenos:
name: myinstance
memory: 2048
vcpus: 2
volumes:
- type: file
is_system: true
source: /volumes/myvolume.qcow2
Initialise and start instance::
compute init --start

10
docs/source/cli/index.rst Normal file
View File

@ -0,0 +1,10 @@
CLI
===
.. toctree::
:maxdepth: 3
getting_started
cloud_init
instance_file
reference

View File

@ -0,0 +1,8 @@
Instance file reference
=======================
There is full example of :file:`instance.yaml` with comments.
.. literalinclude:: ../../../instance.full.yaml
:caption: instance.yaml
:language: yaml

View File

@ -0,0 +1,7 @@
CLI Reference
=============
.. argparse::
:module: compute.cli.parser
:func: get_parser
:prog: compute

View File

@ -1,4 +1,3 @@
# Add ../.. to path for autodoc Sphinx extension
import os
import sys
sys.path.insert(0, os.path.abspath('../..'))
@ -7,12 +6,13 @@ sys.path.insert(0, os.path.abspath('../..'))
project = 'Compute'
copyright = '2023, Compute Authors'
author = 'Compute Authors'
release = '0.1.0'
release = '0.1.0-dev5'
# Sphinx general settings
extensions = [
'sphinx.ext.autodoc',
'sphinx_multiversion',
'sphinxarg.ext',
]
templates_path = ['_templates']
exclude_patterns = []

View File

@ -0,0 +1,37 @@
Configuration
=============
Configuration can be stored in configration file or in environment variables prefixed with ``CMP_``.
Configuration file must have TOML format. Example configuration:
.. literalinclude:: ../../computed.toml
:caption: /etc/compute/computed.toml
:language: toml
There are:
``libvirt.uri``
Libvirt connection URI.
| Env: ``CMP_LIBVIRT_URI``
| Default: ``qemu:///system``
``storage.images``
Name of libvirt storage pool to store compute instance etalon images.
`compute` takes images from here and creates disks for compute instances
based on them.
| Env: ``CMP_IMAGES_POOL``
| Default: ``images``
``storage.volumes``
Name of libvirt storage pool to store compute instance disks.
| Env: ``CMP_VOLUMES_POOL``
| Default: ``volumes``
.. NOTE::
``storage.images`` and ``storage.volumes`` must be exist. Make sure that these
pools are defined, running, and have the autostart flag.

View File

@ -3,9 +3,15 @@ Compute
Compute instances management library.
.. toctree::
:maxdepth: 1
Contents
--------
.. toctree::
:maxdepth: 3
installation
configuration
cli/index
pyapi/index
Indices and tables

View File

@ -0,0 +1,51 @@
Installation
============
Install Debian 12 on your host system. If you want use virtual machine as host make sure that nested virtualization is enabled.
1. Download or build ``compute`` DEB packages.
2. Install packages::
apt-get install -y --no-install-recommends ./compute*.deb
apt-get install -y --no-install-recommends dnsmasq
3. Make sure that ``libvirtd`` and ``dnsmasq`` are enabled and running::
systemctl enable --now libvirtd.service
systemctl enable --now dnsmasq.service
4. Prepare storage pools. You need storage pool for images and for instance volumes.
::
for pool in images volumes; do
virsh pool-define-as $pool dir - - - - "/$pool"
virsh pool-build $pool
virsh pool-start $pool
virsh pool-autostart $pool
done
5. Setup configration if you want create another storage pools. See
`Configuration <configuration.html>`_
You can use configuration file :file:`/etc/compute/computed.toml` or environment
variables.
You can set environment variables in your :file:`~/.profile`, :file:`~/.bashrc`
or globally in :file:`/etc/profile.d/compute` or :file:`/etc/bash.bashrc`. For example:
.. code-block:: sh
export CMP_LIBVIRT_URI=qemu:///system
export CMP_IMAGES_POOL=images
export CMP_VOLUMES_POOL=volumes
Make sure the variables are exported to the environment::
printenv | grep CMP_
6. Prepare network::
virsh net-start default
virsh net-autostart default
7. Done. Now you can follow `CLI instructions <cli/index.html>`_

View File

@ -1,5 +1,5 @@
``exceptions``
==============
``exceptions`` — Exceptions
===========================
.. automodule:: compute.exceptions
:members:

View File

@ -1,43 +1,8 @@
Python API
==========
The API allows you to perform actions on instances programmatically. Below is
an example of changing parameters and launching the `myinstance` instance.
.. code-block:: python
import logging
from compute import Session
logging.basicConfig(level=logging.DEBUG)
with Session() as session:
instance = session.get_instance('myinstance')
instance.set_vcpus(4)
instance.start()
instance.set_autostart(enabled=True)
:class:`Session` context manager provides an abstraction over :class:`libvirt.virConnect`
and returns objects of other classes of the present library.
Entity representation
---------------------
Entities such as a compute-instance are represented as classes. These classes directly
call libvirt methods to perform operations on the hypervisor. An example class is
:class:`Volume`.
The configuration files of various libvirt objects in `compute` are described by special
dataclasses. The dataclass stores object parameters in its properties and can return an
XML config for libvirt using the ``to_xml()`` method. For example :class:`VolumeConfig`.
`Pydantic <https://docs.pydantic.dev/>`_ models are used to validate input data.
For example :class:`VolumeSchema`.
Modules documentation
---------------------
API Reference
-------------
.. toctree::
:maxdepth: 4

View File

@ -0,0 +1,5 @@
``cloud_init``
==============
.. automodule:: compute.instance.cloud_init
:members:

View File

@ -0,0 +1,5 @@
``devices``
===========
.. automodule:: compute.instance.devices
:members:

View File

@ -3,4 +3,3 @@
.. automodule:: compute.instance.guest_agent
:members:
:special-members: __init__

View File

@ -1,10 +1,12 @@
``instance``
============
``instance`` — Manage compute instances
=======================================
.. toctree::
:maxdepth: 1
:maxdepth: 3
:caption: Contents:
instance
guest_agent
devices
cloud_init
schemas

View File

@ -3,4 +3,3 @@
.. automodule:: compute.instance.instance
:members:
:special-members: __init__

View File

@ -1,6 +1,5 @@
``session``
===========
``session`` — Hypervisor session manager
========================================
.. automodule:: compute.session
:members:
:special-members: __init__

View File

@ -1,8 +1,8 @@
``storage``
============
``storage`` — Manage storage pools and volumes
==============================================
.. toctree::
:maxdepth: 1
:maxdepth: 3
:caption: Contents:
pool

View File

@ -3,4 +3,3 @@
.. automodule:: compute.storage.pool
:members:
:special-members: __init__

View File

@ -3,4 +3,3 @@
.. automodule:: compute.storage.volume
:members:
:special-members: __init__

View File

@ -1,5 +1,5 @@
``utils``
=========
``utils`` — Common utils
========================
``utils.units``
---------------
@ -12,3 +12,17 @@
.. automodule:: compute.utils.ids
:members:
``utils.dictutil``
------------------
.. automodule:: compute.utils.dictutil
:members:
``utils.diskutils``
-------------------
.. automodule:: compute.utils.diskutils
:members:

View File

@ -1,13 +1,13 @@
# compute bash completion script
_compute_global_opts="--connect --log-level"
_compute_root_cmd="
$_compute_global_opts
--version
--verbose
--connect
--log-level
init
exec
ls
lsdisks
start
shutdown
reboot
@ -18,42 +18,61 @@ _compute_root_cmd="
status
setvcpus
setmem
setpasswd"
_compute_init_opts=""
_compute_exec_opts="
setpass
setcdrom
setcloudinit
delete"
_compute_init_opts="$_compute_global_opts --test --start"
_compute_exec_opts="$_compute_global_opts
--timeout
--executable
--env
--no-join-args"
_compute_ls_opts=""
_compute_start_opts=""
_compute_shutdown_opts="--method"
_compute_reboot_opts=""
_compute_reset_opts=""
_compute_powrst_opts=""
_compute_pause_opts=""
_compute_resume_opts=""
_compute_status_opts=""
_compute_setvcpus_opts=""
_compute_setmem_opts=""
_compute_setpasswd_opts="--encrypted"
_compute_ls_opts="$_compute_global_opts"
_compute_lsdisks_opts="$_compute_global_opts --persistent"
_compute_start_opts="$_compute_global_opts"
_compute_shutdown_opts="$_compute_global_opts --soft --normal --hard --unsafe"
_compute_reboot_opts="$_compute_global_opts"
_compute_reset_opts="$_compute_global_opts"
_compute_powrst_opts="$_compute_global_opts"
_compute_pause_opts="$_compute_global_opts"
_compute_resume_opts="$_compute_global_opts"
_compute_status_opts="$_compute_global_opts"
_compute_setvcpus_opts="$_compute_global_opts"
_compute_setmem_opts="$_compute_global_opts"
_compute_setpass_opts="$_compute_global_opts --encrypted"
_compute_setcdrom_opts="$_compute_global_opts --detach"
_compute_setcloudinit_opts="$_compute_global_opts
--user-data
--vendor-data
--meta-data
--network-config"
_compute_delete_opts="$_compute_global_opts --yes --save-volumes"
_compute_complete_instances()
{
local base_name
for file in /etc/libvirt/qemu/*.xml; do
nodir="${file##*/}"
printf '%s ' "${nodir//\.xml}"
base_name="${file##*/}"
printf '%s ' "${base_name//\.xml}"
done
}
_compute_compreply()
{
local cgopts=
if [[ "$1" == '-f' ]]; then
cgopts="-f"
shift
fi
if [[ "$current" = [a-z]* ]]; then
_compute_compwords="$(_compute_complete_instances)"
else
_compute_compwords="$*"
fi
COMPREPLY=($(compgen -W "$_compute_compwords" -- "$current"))
COMPREPLY=($(compgen $cgopts -W "$_compute_compwords" -- "$current"))
}
_compute_complete()
@ -67,9 +86,10 @@ _compute_complete()
nshift=$((COMP_CWORD-1))
previous="${COMP_WORDS[COMP_CWORD-nshift]}"
case "$previous" in
init) COMPREPLY=($(compgen -f -- "$current"));;
init) COMPREPLY=($(compgen -f -W "$_compute_init_opts" -- "$current"));;
exec) _compute_compreply "$_compute_exec_opts";;
ls) COMPREPLY=($(compgen -W "$_compute_ls_opts" -- "$current"));;
lsdisks) _compute_compreply "$_compute_lsdisks_opts";;
start) _compute_compreply "$_compute_start_opts";;
shutdown) _compute_compreply "$_compute_shutdown_opts";;
reboot) _compute_compreply "$_compute_reboot_opts";;
@ -80,7 +100,10 @@ _compute_complete()
status) _compute_compreply "$_compute_status_opts";;
setvcpus) _compute_compreply "$_compute_setvcpus_opts";;
setmem) _compute_compreply "$_compute_setmem_opts";;
setpasswd) _compute_compreply "$_compute_setpasswd_opts";;
setpass) _compute_compreply "$_compute_setpass_opts";;
setcdrom) _compute_compreply "$_compute_setcdrom_opts";;
setcloudinit) _compute_compreply -f "$_compute_setcloudinit_opts";;
delete) _compute_compreply "$_compute_delete_opts";;
*) COMPREPLY=()
esac
;;

162
instance.full.yaml Normal file
View File

@ -0,0 +1,162 @@
# Instance name. This name is used as ID and must contain only lowercase
# letters, numbers, minus sign and underscore. If name is not set random UUID
# will used as name.
name: myinstance
# Title is optional human readable title.
title: my_title
# Optional instance description
description: Take instance description here
# Number of vCPUs.
vcpus: 2
# The maximum number of vCPUs to which you can scale without restarting the
# instance. By default equals to number of threads on host.
max_vcpus: 4
# Memory size in MiB (mebibytes: value in power of 1024).
memory: 2048
# The maximum amount of memory in MiB (mebibytes) to which you can scale
# without restarting the instance. By default equals to host memory size.
max_memory: 4096
# Emulated CPU settings
cpu:
# CPU emulation mode. Can be one of:
# - host-passthrough (default) -- passthrough host processor
# - host-model
# - custom
# - maximum
# See Libvirt docs for more info:
# https://libvirt.org/formatdomain.html#cpu-model-and-topology
emulation_mode: custom
# CPU vendor and model
# See usable CPUs supported by hypervisor run Python script with contents:
#
# import compute
# with compute.Session() as s:
# for cpu in s.get_capabilities().usable_cpus:
# print(cpu)
#
# Also see https://www.qemu.org/docs/master/system/i386/cpu.html
vendor: Intel
model: Snowridge
# CPU features. Refer to QEMU documentation and host capabilities.
# Python script to get available features for CPU in 'host-model' mode:
#
# import compute
# with compute.Session() as s:
# features = s.get_capabilities().cpu_features
# print('require:')
# for feat in features['require']:
# print(f' - {feat}')
# print('disable:')
# for feat in features['disable']:
# print(f' - {feat}')
features:
require:
- ss
- vmx
- fma
- avx
- f16c
- hypervisor
- tsc_adjust
- bmi1
- avx2
- bmi2
- invpcid
- adx
- pku
- vaes
- vpclmulqdq
- rdpid
- fsrm
- md-clear
- serialize
- stibp
- avx-vnni
- xsaves
- abm
- ibpb
- amd-stibp
- amd-ssbd
- rdctl-no
- ibrs-all
- skip-l1dfl-vmentry
- mds-no
- pschange-mc-no
disable:
- mpx
- cldemote
- core-capability
- split-lock-detect
# CPU topology
# The product of the values of all parameters must equal the maximum number
# of vcpu:
# sockets * dies * cores * threads = max_vcpus
# dies is optional and equals 1 by default.
#
# If you need a complex topology, you will have to sacrifice the ability to
# hotplug vCPUS. You will need to set 'max_vcpus' to equal 'vcpus'. To apply
# the changes you will need to perform a power reset or manually shutdown
# and start instance (not reboot or reset).
#
# By default, the number of sockets will be set to the number of vCPUS. You
# may want to use a single socket without sacrificing the vCPUS hotplug, so
# you can set the following values:
#
# topology:
# sockets: 1
# cores: 4
# threads: 1
#
# Note that the value of 'cores' must be equal to 'max_vcpus'.
topology:
sockets: 1
dies: 1
cores: 2
threads: 1
# QEMU emulated machine
machine: pc-i440fx-8.1
# Path to emulator on host
emulator: /usr/bin/qemu-system-x86_64
# Emulated platform arch
arch: x86_64
# Machine boot setting
boot:
# Disks boot order. Boot from CDROM first.
order:
- cdrom
- hd
# Network configuration. This decision is temporary and will be changed in
# the future. We recommend not using this option.
network:
interfaces:
- mac: 00:16:3e:7e:8c:4a
source: default
model: virtio
# Disk image
image: /images/debian-12-generic-amd64.qcow2
# Storage volumes list
volumes:
- type: file
device: disk
bus: virtio
# Disk target name. This name is used only for the hypervisor and may not be
# the same as the drive name in the guest operating system.
targer: vda
# 'source' may used for connect existing volumes. In this example it is
# improper.
#source: /images/debian-12-generic-amd64.qcow2
capacity:
value: 10
unit: GiB
# Make volume read only.
is_readonly: false
# Mark the disk as system disk. This label is needed for use in conjunction
# with the image parameter. The contents of the disk specified in image will
# be copied to this volume.
is_system: true
# Cloud-init configuration. See `cli/cloud_init.rst` file for more info.
cloud_init:
user_data: null
meta_data: null
vendor_data: null
network_config: null

View File

@ -1,24 +0,0 @@
DOCKER_CMD ?= docker
DOCKER_IMG = pybuilder:bookworm
DEBBUILDDIR = build
all: docker-build build
clean:
test -d $(DEBBUILDDIR) && rm -rf $(DEBBUILDDIR) || true
docker-build:
$(DOCKER_CMD) build -f Dockerfile -t $(DOCKER_IMG) .
build: clean
mkdir -p $(DEBBUILDDIR)
cp -v ../dist/compute-*[.tar.gz] $(DEBBUILDDIR)/
cp -r ../docs $(DEBBUILDDIR)/
if [ -f build.sh.bak ]; then mv build.sh{.bak,}; fi
cp build.sh{,.bak}
awk '/authors/{gsub(/[\[\]]/,"");print $$3" "$$4}' ../pyproject.toml \
| sed "s/['<>]//g" \
| tr ' ' '\n' \
| xargs -I {} sed "0,/%placeholder%/s//{}/" -i build.sh
$(DOCKER_CMD) run --rm -i -v $$PWD:/mnt $(DOCKER_IMG) bash < build.sh
mv build.sh{.bak,}

View File

@ -0,0 +1,10 @@
FROM archlinux:latest
WORKDIR /mnt
RUN chown 1000:1000 /mnt; \
pacman -Sy --noconfirm \
fakeroot \
binutils \
python \
python-pip; \
echo "alias ll='ls -alFh'" >> /etc/bash.bashrc
USER 1000:1000

View File

@ -0,0 +1,22 @@
DOCKER_CMD ?= docker
DOCKER_IMG = computebuilder:archlinux
BUILDDIR = build
all: docker-build build
clean:
test -d $(BUILDDIR) && rm -rf $(BUILDDIR) || true
docker-build:
$(DOCKER_CMD) build -f Dockerfile -t $(DOCKER_IMG) .
build: clean
mkdir -p $(BUILDDIR)
VERSION=$$(awk '/^version/{print $$3}' ../../pyproject.toml | sed s'/-/\./'); \
sed "s/pkgver=.*/pkgver=$$VERSION/" PKGBUILD > $(BUILDDIR)/PKGBUILD
cp -v ../../dist/compute-*[.tar.gz] $(BUILDDIR)/
cp ../../extra/completion.bash $(BUILDDIR)/
$(DOCKER_CMD) run --rm -i -v $$PWD/$(BUILDDIR):/mnt --ulimit "nofile=1024:1048576" \
$(DOCKER_IMG) makepkg --nodeps --clean
# Remove unwanted files from build dir
find $(BUILDDIR) ! -name '*.pkg.tar.zst' -type f -exec rm -f {} +

View File

@ -0,0 +1,21 @@
pkgname=compute
pkgver='%placeholder%'
pkgrel=1
pkgdesc='Compute instances management library'
arch=(any)
url=https://get.lulzette.ru/hstack/compute
license=('GPL-3-or-later')
makedepends=(python python-pip)
depends=(python libvirt libvirt-python qemu-base qemu-system-x86 qemu-img)
optdepends=(
'dnsmasq: required for default NAT/DHCP'
'iptables-nft: required for default NAT'
)
provides=(compute)
conflicts=()
package() {
pip install --no-cache-dir --no-deps --root $pkgdir ../$pkgname-*.tar.gz
install -Dm644 ../completion.bash $pkgdir/usr/share/bash-completion/completions/compute
install -Dm644 $pkgdir/usr/lib/*/site-packages/computed.toml $pkgdir/etc/compute/computed.toml
}

View File

@ -14,6 +14,7 @@ RUN apt-get update; \
python3-setuptools \
python3-sphinx \
python3-sphinx-multiversion \
python3-sphinx-argparse \
python3-libvirt \
python3-lxml \
python3-yaml \

29
packaging/debian/Makefile Normal file
View File

@ -0,0 +1,29 @@
DOCKER_CMD ?= docker
DOCKER_IMG = computebuilder:debian-bookworm
BUILDDIR = build
KEEP_BUILDFILES ?=
all: docker-build build
clean:
test -d $(BUILDDIR) && rm -rf $(BUILDDIR) || true
docker-build:
$(DOCKER_CMD) build -f Dockerfile -t $(DOCKER_IMG) .
build: clean
mkdir -p $(BUILDDIR)
cp -v ../../dist/compute-*[.tar.gz] $(BUILDDIR)/
cp -r ../../docs $(BUILDDIR)/
cp ../../extra/completion.bash $(BUILDDIR)/compute.bash-completion
if [ -f build.sh.bak ]; then mv build.sh.bak build.sh; fi
cp build.sh{,.bak}
awk '/authors/{gsub(/[\[\]]/,"");print $$3" "$$4}' ../pyproject.toml \
| sed "s/['<>]//g" \
| tr ' ' '\n' \
| xargs -I {} sed "0,/%placeholder%/s//{}/" -i build.sh
$(DOCKER_CMD) run --rm -i -v $$PWD:/mnt $(DOCKER_IMG) bash < build.sh
mv build.sh{.bak,}
# Remove unwanted files from build dir
find $(BUILDDIR) -mindepth 1 -type d -exec rm -rf {} +
[ -z $(KEEP_BUILDFILES) ] && find $(BUILDDIR) ! -name '*.deb' -type f -exec rm -f {} + || true

View File

@ -11,5 +11,6 @@ sed -e "s%\.\./\.\.%$PWD%" -i ../docs/source/conf.py
dh_make --copyright gpl3 --yes --python --file ../compute-*[.tar.gz]
rm debian/*.ex debian/README.{Debian,source} debian/*.docs
sed -e 's/\* Initial release.*/\* This is the development build, see commits in upstream repo for info./' -i debian/changelog
cp -v ../../files/{control,rules,copyright,docs,compute.bash-completion} debian/
cp -v ../../files/{control,rules,copyright,docs,install} debian/
mv ../compute.bash-completion debian/
dpkg-buildpackage -us -uc

View File

@ -13,6 +13,7 @@ Build-Depends:
python3-all,
python3-sphinx,
python3-sphinx-multiversion,
python3-sphinx-argparse,
python3-libvirt,
python3-lxml,
python3-yaml,
@ -27,17 +28,22 @@ Depends:
${misc:Depends},
qemu-system,
qemu-utils,
libvirt-daemon,
libvirt-daemon-system,
libvirt-daemon-driver-qemu,
libvirt-clients,
python3-libvirt,
python3-lxml,
python3-yaml,
python3-pydantic
python3-pydantic,
mtools,
dosfstools,
Recommends:
dnsmasq
dnsmasq,
dnsmasq-base
Suggests:
compute-doc
Description: Compute instances management library and tools (Python 3)
Description: Compute instances management library (Python 3)
Package: compute-doc
Section: doc
@ -45,4 +51,4 @@ Architecture: all
Depends:
${sphinxdoc:Depends},
${misc:Depends},
Description: Compute instances management library and tools (documentation)
Description: Compute instances management library (documentation)

View File

@ -0,0 +1 @@
computed.toml etc/compute/

20
poetry.lock generated
View File

@ -696,6 +696,24 @@ docs = ["sphinxcontrib-websupport"]
lint = ["docutils-stubs", "flake8 (>=3.5.0)", "flake8-simplify", "isort", "mypy (>=0.990)", "ruff", "sphinx-lint", "types-requests"]
test = ["cython (>=3.0)", "filelock", "html5lib", "pytest (>=4.6)", "setuptools (>=67.0)"]
[[package]]
name = "sphinx-argparse"
version = "0.4.0"
description = "A sphinx extension that automatically documents argparse commands and options"
category = "dev"
optional = false
python-versions = ">=3.7"
files = [
{file = "sphinx_argparse-0.4.0-py3-none-any.whl", hash = "sha256:73bee01f7276fae2bf621ccfe4d167af7306e7288e3482005405d9f826f9b037"},
{file = "sphinx_argparse-0.4.0.tar.gz", hash = "sha256:e0f34184eb56f12face774fbc87b880abdb9017a0998d1ec559b267e9697e449"},
]
[package.dependencies]
sphinx = ">=1.2.0"
[package.extras]
markdown = ["CommonMark (>=0.5.6)"]
[[package]]
name = "sphinx-autobuild"
version = "2021.3.14"
@ -895,4 +913,4 @@ zstd = ["zstandard (>=0.18.0)"]
[metadata]
lock-version = "2.0"
python-versions = '^3.11'
content-hash = "e5c07eebe683b92360ec12cada14fc5ccbe4e4add52549bf978f580e551abfb0"
content-hash = "cbded73a481e7c6d6e4c4d5de8e37ac5a53848ab774090a913f1846ef4d7421e"

View File

@ -1,9 +1,11 @@
[tool.poetry]
name = 'compute'
version = '0.1.0-dev1'
description = 'Compute instances management library and tools'
version = '0.1.0-dev5'
description = 'Compute instances management library'
license = 'GPL-3.0-or-later'
authors = ['ge <ge@nixhacks.net>']
readme = 'README.md'
include = ['computed.toml', 'instance.full.yaml']
[tool.poetry.dependencies]
python = '^3.11'
@ -13,7 +15,7 @@ pydantic = '1.10.4'
pyyaml = "^6.0.1"
[tool.poetry.scripts]
compute = 'compute.cli.control:cli'
compute = 'compute.cli.parser:run'
[tool.poetry.group.dev.dependencies]
ruff = '^0.1.3'
@ -23,6 +25,7 @@ isort = '^5.12.0'
sphinx = '^7.2.6'
sphinx-autobuild = '^2021.3.14'
sphinx-multiversion = '^0.2.4'
sphinx-argparse = "^0.4.0"
[build-system]
requires = ['poetry-core']
@ -42,11 +45,21 @@ target-version = 'py311'
[tool.ruff.lint]
select = ['ALL']
ignore = [
'Q000', 'Q003', 'D211', 'D212',
'ANN101', 'ISC001', 'COM812',
'D203', 'ANN204', 'T201',
'EM102', 'TRY003', 'EM101',
'TD003', 'TD006', 'FIX002', # 'todo' strings linting
'Q000', 'Q003',
'D211', 'D212',
'ANN101', 'ANN102', 'ANN204',
'ISC001',
'COM812',
'D203',
'T201',
'S320',
'EM102',
'TRY003',
'EM101',
'TD003', 'TD006',
'FIX002',
'C901',
'PLR0912', 'PLR0913', 'PLR0915',
]
exclude = ['__init__.py']