ge/python-compute
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

various improvements

Browse Source
This commit is contained in:
ge
2023-07-28 01:01:32 +03:00
parent 754c608826
commit 42ad91fa83
10 changed files with 322 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
node_agent/config.py
View File

@ -17,13 +17,14 @@ class ConfigLoader(UserDict):
file = NODEAGENT_CONFIG_FILE or NODEAGENT_DEFAULT_CONFIG_FILE
self.file = Path(file)
self.data = self._load()
# todo: load deafult configuration
def _load(self):
try:
with open(self.file, 'rb') as config:
return tomllib.load(config)
# todo: schema validation
# todo: config schema validation
except (OSError, ValueError) as readerr:
raise ConfigLoadError('Cannot read config file: %s: %s', (self.file, readerr)) from readerr
raise ConfigLoadError(f'Cannot read config file: {self.file}: {readerr}') from readerr
except tomllib.TOMLDecodeError as tomlerr:
raise ConfigLoadError('Bad TOML syntax in config file: %s: %s', (self.file, tomlerr)) from tomlerr
raise ConfigLoadError(f'Bad TOML syntax in config file: {self.file}: {tomlerr}') from tomlerr

4
node_agent/main.py
View File

@ -23,8 +23,8 @@ class LibvirtSession(AbstractContextManager):
return libvirt.open(connection_uri)
except libvirt.libvirtError as err:
raise LibvirtSessionError(
'Failed to open connection to the hypervisor: %s' % err
)
f'Failed to open connection to the hypervisor: {err}'
) from err
def close(self) -> None:
self.session.close()

6
node_agent/utils/vmctl.py
View File

@ -17,6 +17,7 @@ import sys
import pathlib
import logging
import libvirt
from docopt import docopt
sys.path.append('/home/ge/Code/node-agent')
@ -38,13 +39,14 @@ def cli():
args = docopt(__doc__)
config = pathlib.Path(args['--config']) or None
loglvl = args['--loglvl'].upper()
machine = args['<machine>']
if loglvl in levels:
logging.basicConfig(level=levels[loglvl])
with LibvirtSession(config) as session:
try:
vm = VirtualMachine(session, args['<machine>'])
vm = VirtualMachine(session, machine)
if args['status']:
print(vm.status)
if args['is-running']:
@ -58,7 +60,7 @@ def cli():
if args['shutdown']:
vm.shutdown(force=args['--force'], sigkill=args['sigkill'])
except VMNotFound as nferr:
sys.exit(f'{Color.RED}VM {args["<machine>"]} not found.{Color.NONE}')
sys.exit(f'{Color.RED}VM {machine} not found.{Color.NONE}')
except VMError as vmerr:
sys.exit(f'{Color.RED}{vmerr}{Color.NONE}')

15
node_agent/utils/vmexec.py
View File

@ -35,6 +35,7 @@ def cli():
args = docopt(__doc__)
config = pathlib.Path(args['--config']) or None
loglvl = args['--loglvl'].upper()
machine = args['<machine>']
if loglvl in levels:
logging.basicConfig(level=levels[loglvl])
@ -44,7 +45,7 @@ def cli():
cmd = args['<command>']
try:
ga = QemuAgent(session, args['<machine>'])
ga = QemuAgent(session, machine)
exited, exitcode, stdout, stderr = ga.shellexec(
cmd,
executable=shell,
@ -63,14 +64,15 @@ def cli():
sys.exit(errmsg)
except VMNotFound as err:
sys.exit(
f'{Color.RED}VM {args["<machine>"]} not found.{Color.NONE}'
f'{Color.RED}VM {machine} not found.{Color.NONE}'
)
if not exited:
print(
Color.YELLOW
+'[NOTE: command may still running]'
+ Color.NONE
+ Color.NONE,
file=sys.stderr
)
else:
if exitcode == 0:
@ -80,13 +82,14 @@ def cli():
print(
exitcolor
+ f'[command exited with exit code {exitcode}]'
+ Color.NONE
+ Color.NONE,
file=sys.stderr
)
if stderr:
print(Color.RED + stderr.strip() + Color.NONE)
print(Color.RED + stderr.strip() + Color.NONE, file=sys.stderr)
if stdout:
print(Color.GREEN + stdout.strip() + Color.NONE)
print(Color.GREEN + stdout.strip() + Color.NONE, file=sys.stdout)
if __name__ == '__main__':
cli()

14
node_agent/vm/ga.py
View File

@ -14,8 +14,8 @@ from .base import VMBase
logger = logging.getLogger(__name__)
DEFAULT_WAIT_TIMEOUT = 60 # seconds
POLL_INTERVAL = 0.3
QEMU_TIMEOUT = 60 # seconds
POLL_INTERVAL = 0.3 # also seconds
class QemuAgent(VMBase):
@ -33,7 +33,7 @@ class QemuAgent(VMBase):
_get_cmd_result()
Intended for long-time commands. This function loops and every
POLL_INTERVAL calls 'guest-exec-status' for specified guest PID.
Polling ends on command exited or on timeout.
Polling ends if command exited or on timeout.
_return_tuple()
This method transforms JSON command output to tuple and decode
base64 encoded strings optionally.
@ -46,7 +46,7 @@ class QemuAgent(VMBase):
flags: int | None = None
):
super().__init__(session, name)
self.timeout = timeout or DEFAULT_WAIT_TIMEOUT # timeout for guest agent
self.timeout = timeout or QEMU_TIMEOUT # timeout for guest agent
self.flags = flags or libvirt_qemu.VIR_DOMAIN_QEMU_MONITOR_COMMAND_DEFAULT
def execute(
@ -56,7 +56,7 @@ class QemuAgent(VMBase):
capture_output: bool = False,
decode_output: bool = False,
wait: bool = True,
timeout: int = DEFAULT_WAIT_TIMEOUT,
timeout: int = QEMU_TIMEOUT,
):
"""
Execute command on guest and return output if capture_output is True.
@ -99,7 +99,7 @@ class QemuAgent(VMBase):
capture_output: bool = False,
decode_output: bool = False,
wait: bool = True,
timeout: int = DEFAULT_WAIT_TIMEOUT,
timeout: int = QEMU_TIMEOUT,
):
"""
Execute command on guest with selected shell. /bin/sh by default.
@ -139,7 +139,7 @@ class QemuAgent(VMBase):
pid: int,
decode_output: bool = False,
wait: bool = True,
timeout: int = DEFAULT_WAIT_TIMEOUT,
timeout: int = QEMU_TIMEOUT,
):
"""Get executed command result. See GuestAgent.execute() for info."""
exited = exitcode = stdout = stderr = None

86
node_agent/vm/main.py
View File

@ -13,7 +13,7 @@ class VirtualMachine(VMBase):
@property
def name(self):
return self.domain.name()
return self.domname
@property
def status(self) -> str:
@ -21,7 +21,12 @@ class VirtualMachine(VMBase):
Return VM state: 'running', 'shutoff', etc. Reference:
https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainState
"""
state = self.domain.info()[0]
try:
# libvirt returns list [state: int, reason: int]
# https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainGetState
state = self.domain.state()[0]
except libvirt.libvirtError as err:
raise VMError(f'Cannot fetch VM status vm={self.domname}: {err}') from err
match state:
case libvirt.VIR_DOMAIN_NOSTATE:
return 'nostate'
@ -48,6 +53,16 @@ class VirtualMachine(VMBase):
return False
return True
@property
def is_autostart(self) -> bool:
"""Return True if VM autostart is enabled, else return False."""
try:
if self.domain.autostart() == 1:
return True
return False
except libvirt.libvirtError as err:
raise VMError(f'Cannot get autostart status vm={self.domname}: {err}') from err
def start(self) -> None:
"""Start defined VM."""
logger.info('Starting VM: vm=%s', self.domname)
@ -57,9 +72,7 @@ class VirtualMachine(VMBase):
try:
ret = self.domain.create()
except libvirt.libvirtError as err:
raise VMError(err) from err
if ret != 0:
raise VMError('Cannot start VM: vm=%s exit_code=%s', self.domname, ret)
raise VMError(f'Cannot start vm={self.domname} return_code={ret}: {err}') from err
def shutdown(self, force=False, sigkill=False) -> None:
"""
@ -71,41 +84,56 @@ class VirtualMachine(VMBase):
flags = libvirt.VIR_DOMAIN_DESTROY_DEFAULT
else:
flags = libvirt.VIR_DOMAIN_DESTROY_GRACEFUL
if force:
ret = self.domain.destroyFlags(flags=flags)
else:
# Normal VM shutdown via ACPI signal, OS may ignore this.
ret = self.domain.shutdown()
if ret != 0:
try:
if force:
self.domain.destroyFlags(flags=flags)
else:
# Normal VM shutdown via ACPI signal, OS may ignore this.
self.domain.shutdown()
except libvirt.libvirtError as err:
raise VMError(
f'Cannot shutdown VM, try force or sigkill: %s', self.domname
)
f'Cannot shutdown vm={self.domname} '
f'force={force} sigkill={sigkill}: {err}'
) from err
def reset(self):
"""
Copypaste from libvirt doc::
Copypaste from libvirt doc:
Reset a domain immediately without any guest OS shutdown.
Reset emulates the power reset button on a machine, where all
hardware sees the RST line set and reinitializes internal state.
Reset a domain immediately without any guest OS shutdown.
Reset emulates the power reset button on a machine, where all
hardware sees the RST line set and reinitializes internal state.
Note that there is a risk of data loss caused by reset without any
guest OS shutdown.
Note that there is a risk of data loss caused by reset without any
guest OS shutdown.
"""
ret = self.domian.reset()
if ret != 0:
raise VMError('Cannot reset VM: %s', self.domname)
try:
self.domian.reset()
except libvirt.libvirtError as err:
raise VMError(f'Cannot reset vm={self.domname}: {err}') from err
def reboot(self) -> None:
"""Send ACPI signal to guest OS to reboot. OS may ignore this."""
ret = self.domain.reboot()
if ret != 0:
raise VMError('Cannot reboot: %s', self.domname)
try:
self.domain.reboot()
except libvirt.libvirtError as err:
raise VMError(f'Cannot reboot vm={self.domname}: {err}') from err
def set_autostart(self) -> None:
ret = self.domain.autostart()
if ret != 0:
raise VMError('Cannot set : %s', self.domname)
def autostart(self, enabled: bool) -> None:
"""
Configure VM to be automatically started when the host machine boots.
"""
if enabled:
autostart_flag = 1
else:
autostart_flag = 0
try:
self.domain.setAutostart(autostart_flag)
except libvirt.libvirtError as err:
raise VMError(
f'Cannot set autostart vm={self.domname} '
f'autostart={autostart_flag}: {err}'
) from err
def vcpu_set(self, count: int):
pass

247
node_agent/xml.py
View File

@ -1,91 +1,79 @@
import pathlib
from pathlib import Path
from lxml import etree
from lxml.etree import Element, SubElement, QName, tostring
from lxml.builder import E
class NewXML:
def __init__(
class XMLConstructor:
"""
The XML constructor. This class builds XML configs for libvirtd.
Features:
- Generate basic virtual machine XML. See gen_domain_xml()
- Generate virtual disk XML. See gen_volume_xml()
- Add arbitrary metadata to XML from special structured dict
"""
def __init__(self, xml: str | None = None):
self.xml_string = xml
self.xml = None
@property
def domain_xml(self):
return self.xml
def gen_domain_xml(
self,
name: str,
title: str,
memory: int,
vcpus: int,
cpu_vendor: str,
cpu_model: str,
volume_path: str,
desc: str | None = None,
show_boot_menu: bool = False,
):
memory: int,
volume: Path,
desc: str = ""
) -> None:
"""
Initialise basic XML using lxml E-Factory. Ref:
- https://lxml.de/tutorial.html#the-e-factory
- https://libvirt.org/formatdomain.html
Generate default domain XML configuration for virtual machines.
See https://lxml.de/tutorial.html#the-e-factory for details.
"""
DOMAIN = E.domain
NAME = E.name
TITLE = E.title
DESCRIPTION = E.description
METADATA = E.metadata
MEMORY = E.memory
CURRENTMEMORY = E.currentMemory
VCPU = E.vcpu
OS = E.os
OS_TYPE = E.type
OS_BOOT = E.boot
FEATURES = E.features
ACPI = E.acpi
APIC = E.apic
CPU = E.cpu
CPU_VENDOR = E.vendor
CPU_MODEL = E.model
ON_POWEROFF = E.on_poweroff
ON_REBOOT = E.on_reboot
ON_CRASH = E.on_crash
DEVICES = E.devices
EMULATOR = E.emulator
DISK = E.disk
DISK_DRIVER = E.driver
DISK_SOURCE = E.source
DISK_TARGET = E.target
INTERFACE = E.interface
GRAPHICS = E.graphics
self.domain = DOMAIN(
NAME(name),
TITLE(title),
DESCRIPTION(desc or ""),
METADATA(),
MEMORY(str(memory), unit='MB'),
CURRENTMEMORY(str(memory), unit='MB'),
VCPU(str(vcpus), placement='static'),
OS(
OS_TYPE('hvm', arch='x86_64'),
OS_BOOT(dev='cdrom'),
OS_BOOT(dev='hd'),
self.xml = E.domain(
E.name(name),
E.title(title),
E.description(desc),
E.metadata(),
E.memory(str(memory), unit='MB'),
E.currentMemory(str(memory), unit='MB'),
E.vcpu(str(vcpus), placement='static'),
E.os(
E.type('hvm', arch='x86_64'),
E.boot(dev='cdrom'),
E.boot(dev='hd'),
),
FEATURES(
ACPI(),
APIC(),
E.features(
E.acpi(),
E.apic(),
),
CPU(
CPU_VENDOR(cpu_vendor),
CPU_MODEL(cpu_model, fallback='forbid'),
E.cpu(
E.vendor(cpu_vendor),
E.model(cpu_model, fallback='forbid'),
E.topology(sockets='1', dies='1', cores=str(vcpus), threads='1'),
mode='custom',
match='exact',
check='partial',
),
ON_POWEROFF('destroy'),
ON_REBOOT('restart'),
ON_CRASH('restart'),
DEVICES(
EMULATOR('/usr/bin/qemu-system-x86_64'),
DISK(
DISK_DRIVER(name='qemu', type='qcow2', cache='writethrough'),
DISK_SOURCE(file=volume_path),
DISK_TARGET(dev='vda', bus='virtio'),
E.on_poweroff('destroy'),
E.on_reboot('restart'),
E.on_crash('restart'),
E.pm(
E('suspend-to-mem', enabled='no'),
E('suspend-to-disk', enabled='no'),
),
E.devices(
E.emulator('/usr/bin/qemu-system-x86_64'),
E.disk(
E.driver(name='qemu', type='qcow2', cache='writethrough'),
E.source(file=volume),
E.target(dev='vda', bus='virtio'),
type='file',
device='disk',
),
@ -93,23 +81,106 @@ class NewXML:
type='kvm',
)
def add_volume(self, options: dict, params: dict):
"""Add disk device to domain."""
DISK = E.disk
DISK_DRIVER = E.driver
DISK_SOURCE = E.source
DISK_TARGET = E.target
def gen_volume_xml(
self,
device_name: str,
file: Path,
bus: str = 'virtio',
cache: str = 'writethrough',
disktype: str = 'file',
):
return E.disk(
E.driver(name='qemu', type='qcow2', cache=cache),
E.source(file=file),
E.target(dev=device_name, bus=bus),
type=disktype,
device='disk'
)
x = NewXML(
name='1',
title='first',
memory=2048,
vcpus=4,
cpu_vendor='Intel',
cpu_model='Broadwell',
volume_path='/srv/vm-volumes/5031077f-f9ea-410b-8d84-ae6e79f8cde0.qcow2',
)
def add_volume(self):
raise NotImplementedError()
# x.add_volume()
# print(x.domain)
print(etree.tostring(x.domain, pretty_print=True).decode().strip())
def add_meta(self, data: dict, namespace: str, nsprefix: str) -> None:
"""
Add metadata to domain. See:
https://libvirt.org/formatdomain.html#general-metadata
"""
metadata = metadata_old = self.xml.xpath('/domain/metadata')[0]
metadata.append(
self.construct_xml(
data,
namespace=namespace,
nsprefix=nsprefix,
)
)
self.xml.replace(metadata_old, metadata)
def remove_meta(self, namespace: str):
"""Remove metadata by namespace."""
raise NotImplementedError()
def construct_xml(
self,
tag: dict,
namespace: str | None = None,
nsprefix: str | None = None,
root: Element = None,
) -> Element:
"""
Shortly this recursive function transforms dictonary to XML.
Return etree.Element built from dict with following structure::
{
'name': 'devices', # tag name
'text': '', # optional key
'values': { # optional key, must be a dict of key-value pairs
'type': 'disk'
},
children: [] # optional key, must be a list of dicts
}
Child elements must have the same structure. Infinite `children` nesting
is allowed.
"""
use_ns = False
if isinstance(namespace, str) and isinstance(nsprefix, str):
use_ns = True
# Create element
if root is None:
if use_ns:
element = Element(
QName(namespace, tag['name']),
nsmap={nsprefix: namespace},
)
else:
element = Element(tag['name'])
else:
if use_ns:
element = SubElement(
root,
QName(namespace, tag['name']),
)
else:
element = SubElement(root, tag['name'])
# Fill up element with content
if 'text' in tag.keys():
element.text = tag['text']
if 'values' in tag.keys():
for key in tag['values'].keys():
element.set(str(key), str(tag['values'][key]))
if 'children' in tag.keys():
for child in tag['children']:
element.append(
self.construct_xml(
child,
namespace=namespace,
nsprefix=nsprefix,
root=element,
)
)
return element
def to_string(self):
return tostring(
self.xml, pretty_print=True, encoding='utf-8'
).decode().strip()