ge/python-compute
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

various updates v.dev3

Browse Source
This commit is contained in:
ge
2023-12-13 01:42:50 +03:00
parent b0fa1b7b25
commit d7a73e9bd1
49 changed files with 1872 additions and 904 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
compute/instance/__init__.py
View File

@ -13,6 +13,7 @@
# You should have received a copy of the GNU General Public License
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
from .cloud_init import CloudInit
from .guest_agent import GuestAgent
from .instance import Instance, InstanceConfig
from .schemas import InstanceSchema

221
compute/instance/cloud_init.py Normal file
View File

@ -0,0 +1,221 @@
# This file is part of Compute
#
# Compute is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Compute is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Compute. If not, see <http://www.gnu.org/licenses/>.
# ruff: noqa: S603
"""
`Cloud-init`_ integration for bootstraping compute instances.
.. _Cloud-init: https://cloudinit.readthedocs.io
"""
import logging
import subprocess
import tempfile
from pathlib import Path
from compute.exceptions import InstanceError
from .devices import DiskConfig, DiskDriver
from .instance import Instance
log = logging.getLogger(__name__)
class CloudInit:
"""
Cloud-init integration.
:ivar str user_data: user-data.
:ivar str vendor_data: vendor-data.
:ivar str network_config: network-config.
:ivar str meta_data: meta-data.
"""
def __init__(self):
"""Initialise :class:`CloudInit`."""
self.user_data = None
self.vendor_data = None
self.network_config = None
self.meta_data = None
def __repr__(self) -> str:
"""Represent :class:`CloudInit` object."""
return (
self.__class__.__name__
+ '('
+ ', '.join(
[
f'{self.user_data=}',
f'{self.vendor_data=}',
f'{self.network_config=}',
f'{self.meta_data=}',
]
)
+ ')'
).replace('self.', '')
def _write_to_disk(
self,
disk: str,
filename: str,
data: str | None,
*,
force_file_create: bool = False,
delete_existing_file: bool = False,
default_data: str | None = None,
) -> None:
data = data or default_data
log.debug('Input data %s: %r', filename, data)
if isinstance(data, str):
data = data.encode()
if data is None and force_file_create is False:
return
with tempfile.NamedTemporaryFile() as data_file:
if data is not None:
data_file.write(data)
data_file.flush()
if delete_existing_file:
log.debug('Deleting existing file')
filelist = subprocess.run(
['/usr/bin/mdir', '-i', disk, '-b'],
capture_output=True,
check=True,
)
files = [
f.replace('::/', '')
for f in filelist.stdout.decode().splitlines()
]
log.debug('Files on disk: %s', files)
log.debug("Removing '%s'", filename)
if filename in files:
subprocess.run(
['/usr/bin/mdel', '-i', disk, f'::{filename}'],
check=True,
)
log.debug("Writing file '%s'", filename)
subprocess.run(
[
'/usr/bin/mcopy',
'-i',
disk,
data_file.name,
f'::{filename}',
],
check=True,
)
def create_disk(self, disk: Path, *, force: bool = False) -> None:
"""
Create disk with cloud-init config files.
:param path: Disk path.
:param force: Replace existing disk.
"""
if not isinstance(disk, Path):
disk = Path(disk)
if disk.exists():
if disk.is_file() is False:
raise InstanceError('Cloud-init disk must be regular file')
if force:
disk.unlink()
else:
raise InstanceError('File already exists')
subprocess.run(
['/usr/sbin/mkfs.vfat', '-n', 'CIDATA', '-C', str(disk), '1024'],
check=True,
stderr=subprocess.DEVNULL,
)
self._write_to_disk(
disk=disk,
filename='user-data',
data=self.user_data,
force_file_create=True,
default_data='#cloud-config',
)
self._write_to_disk(
disk=disk,
filename='vendor-data',
data=self.vendor_data,
)
self._write_to_disk(
disk=disk,
filename='network-config',
data=self.network_config,
)
self._write_to_disk(
disk=disk,
filename='meta-data',
data=self.meta_data,
force_file_create=True,
)
def update_disk(self, disk: Path) -> None:
"""Update files on existing disk."""
if not isinstance(disk, Path):
disk = Path(disk)
if not disk.exists():
raise InstanceError(f"File '{disk}' does not exists")
if self.user_data:
self._write_to_disk(
disk=disk,
filename='user-data',
data=self.user_data,
force_file_create=True,
default_data='#cloud-config',
delete_existing_file=True,
)
if self.vendor_data:
self._write_to_disk(
disk=disk,
filename='vendor-data',
data=self.vendor_data,
delete_existing_file=True,
)
if self.network_config:
self._write_to_disk(
disk=disk,
filename='network-config',
data=self.network_config,
delete_existing_file=True,
)
if self.meta_data:
self._write_to_disk(
disk=disk,
filename='meta-data',
data=self.meta_data,
force_file_create=True,
delete_existing_file=True,
)
def attach_disk(self, disk: Path, target: str, instance: Instance) -> None:
"""
Attach cloud-init disk to instance.
:param disk: Path to disk.
:param instance: Compute instance object.
"""
instance.attach_device(
DiskConfig(
type='file',
device='disk',
source=disk,
target=target,
is_readonly=True,
bus='virtio',
driver=DiskDriver('qemu', 'raw'),
)
)

21
compute/instance/devices.py
View File

@ -24,7 +24,7 @@ from typing import Union
from lxml import etree
from lxml.builder import E
from compute.common import DeviceConfig
from compute.abstract import DeviceConfig
from compute.exceptions import InvalidDeviceConfigError
@ -32,9 +32,9 @@ from compute.exceptions import InvalidDeviceConfigError
class DiskDriver:
"""Disk driver description for libvirt."""
name: str
type: str
cache: str
name: str = 'qemu'
type: str = 'qcow2'
cache: str = 'default'
def __call__(self):
"""Return self."""
@ -56,13 +56,7 @@ class DiskConfig(DeviceConfig):
is_readonly: bool = False
device: str = 'disk'
bus: str = 'virtio'
driver: DiskDriver = field(
default_factory=DiskDriver(
name='qemu',
type='qcow2',
cache='writethrough',
)
)
driver: DiskDriver = field(default_factory=DiskDriver())
def to_xml(self) -> str:
"""Return XML config for libvirt."""
@ -99,13 +93,14 @@ class DiskConfig(DeviceConfig):
pretty_print=True,
).strip()
driver = xml.find('driver')
cachetype = driver.get('cache')
disk_params = {
'type': xml.get('type'),
'device': xml.get('device'),
'driver': DiskDriver(
name=driver.get('name'),
type=driver.get('type'),
cache=driver.get('cache'),
**({'cache': cachetype} if cachetype else {}),
),
'source': xml.find('source').get('file'),
'target': xml.find('target').get('dev'),
@ -122,7 +117,7 @@ class DiskConfig(DeviceConfig):
if driver_param is None:
msg = (
"'driver' tag must have "
"'name', 'type' and 'cache' attributes"
"'name' and 'type' attributes"
)
raise InvalidDeviceConfigError(msg, xml_str)
return cls(**disk_params)

6
compute/instance/guest_agent.py
View File

@ -27,7 +27,7 @@ import libvirt_qemu
from compute.exceptions import (
GuestAgentCommandNotSupportedError,
GuestAgentError,
GuestAgentTimeoutError,
GuestAgentTimeoutExpired,
GuestAgentUnavailableError,
)
@ -114,7 +114,7 @@ class GuestAgent:
if command not in supported:
raise GuestAgentCommandNotSupportedError(command)
def guest_exec( # noqa: PLR0913
def guest_exec(
self,
path: str,
args: list[str] | None = None,
@ -199,7 +199,7 @@ class GuestAgent:
sleep(poll_interval)
now = time()
if now - start_time > self.timeout:
raise GuestAgentTimeoutError(self.timeout)
raise GuestAgentTimeoutExpired(self.timeout)
log.debug(
'Polling command pid=%s finished, time taken: %s seconds',
pid,

62
compute/instance/instance.py
View File

@ -25,7 +25,7 @@ import libvirt
from lxml import etree
from lxml.builder import E
from compute.common import DeviceConfig, EntityConfig
from compute.abstract import DeviceConfig, EntityConfig
from compute.exceptions import (
GuestAgentCommandNotSupportedError,
InstanceError,
@ -141,6 +141,14 @@ class InstanceConfig(EntityConfig):
)
)
xml.append(self._gen_cpu_xml(self.cpu))
xml.append(
E.clock(
E.timer(name='rtc', tickpolicy='catchup'),
E.timer(name='pit', tickpolicy='delay'),
E.timer(name='hpet', present='no'),
offset='utc',
)
)
os = E.os(E.type('hvm', machine=self.machine, arch=self.arch))
for dev in self.boot.order:
os.append(E.boot(dev=dev))
@ -159,7 +167,7 @@ class InstanceConfig(EntityConfig):
devices.append(E.emulator(str(self.emulator)))
for interface in self.network_interfaces:
devices.append(self._gen_network_interface_xml(interface))
devices.append(E.graphics(type='vnc', port='-1', autoport='yes'))
devices.append(E.graphics(type='vnc', autoport='yes'))
devices.append(E.input(type='tablet', bus='usb'))
devices.append(
E.channel(
@ -171,6 +179,7 @@ class InstanceConfig(EntityConfig):
type='unix',
)
)
devices.append(E.serial(E.target(port='0'), type='pty'))
devices.append(
E.console(E.target(type='serial', port='0'), type='pty')
)
@ -212,10 +221,30 @@ class Instance:
:param domain: libvirt domain object
"""
self.domain = domain
self.connection = domain.connect()
self.name = domain.name()
self.guest_agent = GuestAgent(domain)
self._domain = domain
self._connection = domain.connect()
self._name = domain.name()
self._guest_agent = GuestAgent(domain)
@property
def connection(self) -> libvirt.virConnect:
"""Libvirt connection object."""
return self._connection
@property
def domain(self) -> libvirt.virDomain:
"""Libvirt domain object."""
return self._domain
@property
def name(self) -> str:
"""Instance name."""
return self._name
@property
def guest_agent(self) -> GuestAgent:
""":class:`GuestAgent` object."""
return self._guest_agent
def _expand_instance_state(self, state: int) -> str:
states = {
@ -279,6 +308,9 @@ class Instance:
def get_max_vcpus(self) -> int:
"""Maximum vCPUs number for domain."""
if not self.is_running():
xml = etree.fromstring(self.dump_xml(inactive=True))
return int(xml.xpath('/domain/vcpu/text()')[0])
return self.domain.maxVcpus()
def start(self) -> None:
@ -324,7 +356,6 @@ class Instance:
:param method: Method used to shutdown instance
"""
if not self.is_running():
log.warning('Instance is not running, nothing to do')
return
methods = {
'SOFT': libvirt.VIR_DOMAIN_SHUTDOWN_GUEST_AGENT,
@ -737,13 +768,24 @@ class Instance:
:param with_volumes: If True delete local volumes with instance.
"""
log.info("Shutdown instance '%s'", self.name)
self.shutdown(method='HARD')
disks = self.list_disks(persistent=True)
log.debug('Disks list: %s', disks)
for disk in disks:
if with_volumes and disk.type == 'file':
volume = self.connection.storageVolLookupByPath(disk.source)
log.debug('Delete volume: %s', volume.path())
try:
volume = self.connection.storageVolLookupByPath(
disk.source
)
except libvirt.libvirtError as e:
if e.get_error_code() == libvirt.VIR_ERR_NO_STORAGE_VOL:
log.warning(
"Volume '%s' not found, skipped",
disk.source,
)
continue
log.info('Delete volume: %s', volume.path())
volume.delete()
log.debug('Undefine instance')
log.info('Undefine instance')
self.domain.undefine()

12
compute/instance/schemas.py
View File

@ -22,7 +22,7 @@ from pathlib import Path
from pydantic import ValidationError, validator
from pydantic.error_wrappers import ErrorWrapper
from compute.common import EntityModel
from compute.abstract import EntityModel
from compute.utils.units import DataUnit
@ -109,6 +109,15 @@ class BootOptionsSchema(EntityModel):
order: tuple
class CloudInitSchema(EntityModel):
"""Cloud-init config model."""
user_data: str | None = None
meta_data: str | None = None
vendor_data: str | None = None
network_config: str | None = None
class InstanceSchema(EntityModel):
"""Compute instance model."""
@ -127,6 +136,7 @@ class InstanceSchema(EntityModel):
volumes: list[VolumeSchema]
network_interfaces: list[NetworkInterfaceSchema]
image: str | None = None
cloud_init: CloudInitSchema | None = None
@validator('name')
def _check_name(cls, value: str) -> str: # noqa: N805