From e8133af3929b2fecd61d1c8683bf79c3e93745b2 Mon Sep 17 00:00:00 2001 From: ge Date: Sun, 27 Aug 2023 23:42:56 +0300 Subject: [PATCH] upd --- .gitignore | 3 +- README.md | 140 ++++------- config.toml | 30 +-- node_agent/cli/vmctl.py | 52 ++++ node_agent/config.py | 6 +- node_agent/session.py | 18 ++ node_agent/utils/__old_xml.py | 212 ++++++++++++++++ node_agent/utils/xml.py | 232 +++++------------- node_agent/vm/__init__.py | 6 +- node_agent/vm/base.py | 35 ++- node_agent/vm/exceptions.py | 2 +- node_agent/vm/{ga.py => guest_agent.py} | 33 ++- node_agent/vm/hardware.py | 81 ++++++ node_agent/vm/installer.py | 94 +++++++ node_agent/vm/{main.py => virtual_machine.py} | 87 +++++-- node_agent/volume/storage_pool.py | 9 + node_agent/volume/volume.py | 23 ++ 17 files changed, 722 insertions(+), 341 deletions(-) create mode 100644 node_agent/utils/__old_xml.py rename node_agent/vm/{ga.py => guest_agent.py} (82%) create mode 100644 node_agent/vm/hardware.py create mode 100644 node_agent/vm/installer.py rename node_agent/vm/{main.py => virtual_machine.py} (57%) create mode 100644 node_agent/volume/storage_pool.py create mode 100644 node_agent/volume/volume.py diff --git a/.gitignore b/.gitignore index 2d25954..145fa54 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,7 @@ __pycache__/ *.pyc *~ -domain.xml -domgen.py +dom* na dist/ P@ssw0rd diff --git a/README.md b/README.md index c3afecd..cceae22 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# Node Agent +# Compute Node Agent -Агент для работы на ворк-нодах. +Агент для работы на ворк-нодах. В этом репозитории развивается базовая библиотека для взаимодействия с libvirt и выполнения основных операций. # Как это должно выглядеть @@ -18,117 +18,54 @@ - `python3-docopt` 0.6.2 - `python3-libvirt` 9.0.0 (актуальная новее) +`docopt` скорее всего будет выброшен в будущем, так как интерфейс CLI сильно усложнится. + Минимальная поддерживаемая версия Python — `3.11`, потому, что можем. -# Классы +# API -Весь пакет разбит на модули, а основной функционал на классы. +Кодовая база растёт, необходимо автоматически генерировать документацию в README её больше небудет. -## `ConfigLoader` +В структуре проекта сейчас бардак, многое будет переосмыслено и переделано позже. Основная цель на текущем этапе — получить минимально работающий код, с помощью которого возможно выполнить установку виртуальной машины и как-то управлять ею. -Наследуется от `UserDict`. Принимает в конструктор путь до файла, после чего экземпляром `ConfigLoader` можно пользоваться как обычным словарём. Вызывается внутри `LibvirtSession` при инициализации. +Базовые сущности: -## `LibvirtSession` - -Устанавливает сессию с libvirtd и создаёт объект virConnect. Класс умеет принимать в конструктор один аргумент — путь до файла конфигурации, но его можно опустить. +- `LivbirtSession` - обёртка над объектом `libvirt.virConnect`. +- `VirtualMachine` - класс для работы с доменами, через него выполняется большинство действий. +- `VirtualMachineInstaller` - класс для установки ВМ, выполняет кучу проверок, генерирует XML конфиг и т.п. +- `QemuAgent` - понятно что это. +- `ConfigLoader` - загрузчик TOML-конфига, возможно будет выброшен на мороз. ```python -from node_agent import LibvirtSession +from na import LibvirtSession +from na.vm import VirtualMachineInstaller -session = LibvirtSession() + +session = LibvirtSession('config.toml') +compute = VirtualMachineInstaller(session).install( + name='devuan', + vcpus=4, + vcpu_mode='host-model', + memory=2048, +) +print(compute) session.close() ``` -Также этот класс является контекстным менеджером и его можно использвоать так: - -```python -from node_agent import LibvirtSession, VirtualMachine - -with LibvirtSession() as session: - vm = VirtualMachine(session, 'имя_вм') - vm.status -``` - -## `VirtualMachine` - -Класс для базового управления виртуалкой. В конструктор принимает объект LibvirtSession и создаёт объект `virDomain`. - -## `QemuAgent` - -Класс для работы с агентом на гостях. Инициализируется аналогично `VirtualMachine`. Его можно считать законченным. Он умеет: - -- Выполнять шелл команды через метод `shellexec()`. -- Выполнять любые команды QEMU через `execute()`. - -Также способен: - -- Поллить выполнение команды. То есть можно дождаться вывода долгой команды. -- Декодировать base64 вывод STDERR и STDOUT если надо. -- Отправлять данные на STDIN. - -## `XMLConstructor` - -Класс для генерации XML конфигов для либвирта и редактирования XML. Пока умеет очень мало и требует перепиливания. Возможно стоит разбить его на несколько классов. Пример работы с ним: - -```python -from node_agent.xml import XMLConstructor - -domain_xml = XMLConstructor() -domain_xml.gen_domain_xml( - name='13', - title='', - vcpus=2, - cpu_vendor='Intel', - cpu_model='Broadwell', - memory=2048, - volume='/srv/vm-volumes/ef0bcd68-02c2-4f31-ae96-14d2bda5a97b.qcow2', -) -tags_meta = { - 'name': 'tags', - 'children': [ - {'name': 'god_mode'}, - {'name': 'service'} - ] -} -domain_xml.add_meta(tags_meta, namespace='http://half-it-stack.org/xmlns/tags-meta', nsprefix='tags') -print(domain_xml.to_string()) -``` - -В итоге должен получиться какой-то конфиг для ВМ. - -Имеет метод `construct_xml()`, который позволяет привести словарь Python в XML элемент (обхект `lxml.etree.Element`). Пример: - -```python ->>> from lxml.etree import tostring ->>> from na.xml import XMLConstructor ->>> xml = XMLConstructor() ->>> tag = { -... 'name': 'mytag', -... 'values': { -... 'firstname': 'John', -... 'lastname': 'Doe' -... }, -... 'text': 'Hello!', -... 'children': [{'name': 'okay'}] -... } ->>> element = xml.construct_xml(tag) ->>> print(tostring(element).decode()) -'Hello!' ->>> -``` - -Функция рекурсивная, так что теоретически можно положить бесконечное число вложенных элементов в `children`. С аргументами `namespace` и `nsprefix` будет сгенерирован XML с неймспейсом, Ваш кэп. - # TODO - [ ] Установка ВМ - [x] Конструктор XML (базовый) + - [x] Автоматический выбор модели процессора - [ ] Метод создания дисков - - [ ] Дефайн, запуск и автостарт ВМ + - [x] Дефайн, запуск и автостарт ВМ + - [ ] Работа со StoragePool + - [ ] Создание блочных устройств + - [ ] Подключение/отключение устройств - [ ] Управление дисками - [ ] Удаление ВМ -- [ ] Изменение CPU -- [ ] Изменение RAM +- [x] Изменение CPU +- [x] Изменение RAM - [ ] Миграция ВМ между нодами - [x] Работа с qemu-ga - [x] Управление питанием @@ -138,11 +75,20 @@ print(domain_xml.to_string()) - [ ] SSH-ключи - [ ] Сеть - [ ] Создание снапшотов +- [ ] Поддержка выделения гарантированной доли CPU # Заметки -xml.py наверное лучше реализовать через lxml.objectify: https://stackoverflow.com/questions/47304314/adding-child-element-to-xml-in-python +## Будущее этой библиотеки -???: https://www.geeksforgeeks.org/reading-and-writing-xml-files-in-python/ +Либа -Минимальный рабочий XML: https://access.redhat.com/documentation/ru-ru/red_hat_enterprise_linux/6/html/virtualization_administration_guide/section-libvirt-dom-xml-example +## Failover + +В перспективе для ВМ с сетевыми дисками возможно организовать Failover решение — ВМ будет автоматически запускаться на другой ноде из пула при отключении оригинальной ноды. Технически это можно реализовать как создание ВМ с аналогичными характеристиками на другой ноде с подключением к тому же самому сетевому диску. Сразу нужно отметить, что для реализации: + +- Нужно где-то хранить и регулярно обновлять информацию о конфигурации ВМ для воссоздания ВМ +- Нужно иметь "плавающие адреса", чтобы переключить трафик на новую ноду +- Необходимо выполнять failover по чётким критериям: нода полностью недоступна более X времени, маунт сетевого диска отвалился и т.п. +- Как быть с целостностью данных на сетевом диске? При аварии на ноде, данные могли быть повреждены, тогда failover на тот же диск ничего не даст. +- Сетевой диск должен быть зарезервирован средствами распределённой ФС diff --git a/config.toml b/config.toml index b6ec88f..d684204 100644 --- a/config.toml +++ b/config.toml @@ -1,35 +1,29 @@ -[general] -# Наверное стоит создавать локи в виде файлов во время операций с ВМ -# /var/node-agent/locks/vms/{vm} -locks_dir = '/var/node-agent/locks' - [libvirt] -uri = 'qemu:///system' +uri = 'qemu:///session' [logging] level = 'INFO' +driver = 'file' file = '/var/log/node-agent.log' -[volumes] - -[[volumes.pools]] +[[storages.pools]] name = 'ssd-nvme' enabled = true default = true path = '/srv/vm-volumes/ssd-nvme' -[[volumes.pools]] +[[storages.pools]] name = 'hdd' enabled = true path = '/srv/vm-volumes/hdd' -[vms.defaults] -# Какие-то значения по-умолчанию, используемые при создании/работе с ВМ -# Эти параметры также будут аффектить на CLI утилиты -autostart = true # ставить виртуалки в автостарт после установки -start = true # запускать ВМ после установки +[[storages.pools]] +name = 'images' +enabled = true +path = '/srv/vm-images/vendor' + +[virtual_machine.defaults] +autostart = true +start = true cpu_vendor = 'Intel' cpu_model = 'Broadwell' - -[vms.images] -path = '/srv/vm-images' diff --git a/node_agent/cli/vmctl.py b/node_agent/cli/vmctl.py index 09b99de..0664da3 100644 --- a/node_agent/cli/vmctl.py +++ b/node_agent/cli/vmctl.py @@ -5,10 +5,14 @@ Usage: na-vmctl [options] status na-vmctl [options] is-running na-vmctl [options] start na-vmctl [options] shutdown [-f|--force] [-9|--sigkill] + na-vmctl [options] set-vcpus + na-vmctl [options] set-memory + na-vmctl [options] list [-a|--all] Options: -c, --config Config file [default: /etc/node-agent/config.yaml] -l, --loglvl Logging level + -a, --all List all machines including inactive -f, --force Force action. On shutdown calls graceful destroy() -9, --sigkill Send SIGKILL to QEMU process. Not affects without --force """ @@ -35,6 +39,44 @@ class Color: NONE = '\033[0m' +class Table: + """Print table. Example:: + + t = Table() + t.header(['KEY', 'VALUE']) # header is optional + t.row(['key 1', 'value 1']) + t.row(['key 2', 'value 2']) + t.rows( + [ + ['key 3', 'value 3'], + ['key 4', 'value 4'] + ] + ) + t.print() + + """ + + def __init__(self, whitespace: str = '\t'): + self.__rows = [] + self.__whitespace = whitespace + + def header(self, columns: list): + self.__rows.insert(0, [str(col) for col in columns]) + + def row(self, row: list): + self.__rows.append([str(col) for col in row]) + + def rows(self, rows: list): + for row in rows: + self.row(row) + + def print(self): + widths = [max(map(len, col)) for col in zip(*self.__rows)] + for row in self.__rows: + print(self.__whitespace.join( + (val.ljust(width) for val, width in zip(row, widths)))) + + def cli(): args = docopt(__doc__) config = pathlib.Path(args['--config']) or None @@ -49,6 +91,16 @@ def cli(): with LibvirtSession(config) as session: try: + if args['list']: + vms = session.list_domains() + table = Table() + table.header(['NAME', 'STATE', 'AUTOSTART']) + for vm_ in vms: + vm_ = VirtualMachine(vm_) + table.row([vm_.name, vm_.status, vm_.is_autostart]) + table.print() + sys.exit() + vm = VirtualMachine(session, machine) if args['status']: print(vm.status) diff --git a/node_agent/config.py b/node_agent/config.py index 16e48bb..05c1cb4 100644 --- a/node_agent/config.py +++ b/node_agent/config.py @@ -8,7 +8,7 @@ NODEAGENT_CONFIG_FILE = os.getenv('NODEAGENT_CONFIG_FILE') NODEAGENT_DEFAULT_CONFIG_FILE = '/etc/node-agent/config.toml' -class ConfigLoadError(Exception): +class ConfigLoaderError(Exception): """Bad config file syntax, unreachable file or bad config schema.""" @@ -27,11 +27,11 @@ class ConfigLoader(UserDict): return tomllib.load(config) # todo: config schema validation except tomllib.TOMLDecodeError as tomlerr: - raise ConfigLoadError( + raise ConfigLoaderError( f'Bad TOML syntax in config file: {self.file}: {tomlerr}' ) from tomlerr except (OSError, ValueError) as readerr: - raise ConfigLoadError( + raise ConfigLoaderError( f'Cannot read config file: {self.file}: {readerr}') from readerr def reload(self): diff --git a/node_agent/session.py b/node_agent/session.py index 6d8ac5f..a049158 100644 --- a/node_agent/session.py +++ b/node_agent/session.py @@ -31,3 +31,21 @@ class LibvirtSession(AbstractContextManager): def close(self) -> None: self.session.close() + + def list_domains(self): + return self.session.listAllDomains() + + def get_domain(self, name: str) -> libvirt.virDomain: + try: + return self.session.lookupByName(name) + except libvirt.libvirtError as err: + if err.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: + raise VMNotFound(name) + else: + raise LibvirtSessionError(err) + + def get_storage_pool(self, name: str) -> libvirt.virStoragePool: + try: + return self.session.storagePoolLookupByName(name) + except libvirt.libvirtError as err: + raise LibvirtSessionError(err) diff --git a/node_agent/utils/__old_xml.py b/node_agent/utils/__old_xml.py new file mode 100644 index 0000000..5e8b23f --- /dev/null +++ b/node_agent/utils/__old_xml.py @@ -0,0 +1,212 @@ +from pathlib import Path + +from lxml.builder import E +from lxml.etree import Element, QName, SubElement, tostring, fromstring + + +XPATH_DOM_NAME = '/domain/name' +XPATH_DOM_TITLE = '/domain/title' +XPATH_DOM_DESCRIPTION = '/domain/description' +XPATH_DOM_METADATA = '/domain/metadata' +XPATH_DOM_MEMORY = '/domain/memory' +XPATH_DOM_CURRENT_MEMORY = '/domain/currentMemory' +XPATH_DOM_VCPU = '/domain/vcpu' +XPATH_DOM_OS = '/domian/os' +XPATH_DOM_CPU = '/domain/cpu' + + +class Reader: + + def __init__(xml: str): + self.xml = xml + self.el = fromstring(self.xml) + + def get_domcaps_machine(self): + return self.el.xpath('/domainCapabilities/machine')[0].text + + def get_domcaps_cpus(self): + # mode can be: custom, host-model, host-passthrough + return self.el.xpath('/domainCapabilities/cpu/mode[@name="custom"]')[0] + + +class Constructor: + """ + The XML constructor. This class builds XML configs for libvirt. + Features: + - Generate basic virtual machine XML. See gen_domain_xml() + - Generate virtual disk XML. See gen_volume_xml() + - Add arbitrary metadata to XML from special structured dict + """ + + def __init__(self, xml: str | None = None): + self.xml_string = xml + self.xml = None + + @property + def domain_xml(self): + return self.xml + + def gen_domain_xml(self, + name: str, + title: str, + vcpus: int, + vcpu_vendor: str, + vcpu_model: str, + mac_addr: str, + memory: int, + volume: Path, + vcpu_features: dict | None = None, + desc: str = "") -> None: + """ + Generate default domain XML configuration for virtual machines. + See https://lxml.de/tutorial.html#the-e-factory for details. + """ + domain_xml = E.domain( + E.name(name), + E.title(title), + E.description(desc), + E.metadata(), + E.memory(str(memory), unit='MB'), + E.currentMemory(str(memory), unit='MB'), + E.vcpu(str(vcpus), placement='static'), + E.os( + E.type('hvm', arch='x86_64'), + E.boot(dev='cdrom'), + E.boot(dev='hd'), + ), + E.features( + E.acpi(), + E.apic(), + ), + E.cpu( + E.vendor(vcpu_vendor), + E.model(vcpu_model, fallback='forbid'), + E.topology(sockets='1', dies='1', cores=str(vcpus), + threads='1'), + mode='custom', + match='exact', + check='partial', + ), + E.on_poweroff('destroy'), + E.on_reboot('restart'), + E.on_crash('restart'), + E.pm( + E('suspend-to-mem', enabled='no'), + E('suspend-to-disk', enabled='no'), + ), + E.devices( + E.emulator('/usr/bin/qemu-system-x86_64'), + E.disk( + E.driver(name='qemu', type='qcow2', cache='writethrough'), + E.source(file=volume), + E.target(dev='vda', bus='virtio'), + type='file', + device='disk', + ), + E.interface( + E.source(network='default'), + E.mac(address=mac_addr), + type='network', + ), + E.graphics( + E.listen(type='address'), + type='vnc', port='-1', autoport='yes' + ), + E.video( + E.model(type='vga', vram='16384', heads='1', primary='yes'), + E.address(type='pci', domain='0x0000', bus='0x00', + slot='0x02', function='0x0'), + ), + ), + type='kvm', + ) + return self.to_string(domain_xml) + + def gen_volume_xml(self, + device_name: str, + file: Path, + bus: str = 'virtio', + cache: str = 'writethrough', + disktype: str = 'file'): + disk_xml = E.disk(E.driver(name='qemu', type='qcow2', cache=cache), + E.source(file=file), + E.target(dev=device_name, bus=bus), + type=disktype, + device='disk') + return self.to_string(disk_xml) + + def add_volume(self): + raise NotImplementedError() + + def add_meta(self, data: dict, namespace: str, nsprefix: str) -> None: + """ + Add metadata to domain. See: + https://libvirt.org/formatdomain.html#general-metadata + """ + metadata = metadata_old = self.xml.xpath('/domain/metadata')[0] + metadata.append( + self.construct_xml( + data, + namespace=namespace, + nsprefix=nsprefix, + )) + self.xml.replace(metadata_old, metadata) + + def remove_meta(self, namespace: str): + """Remove metadata by namespace.""" + raise NotImplementedError() + + def construct_xml(self, + tag: dict, + namespace: str | None = None, + nsprefix: str | None = None, + root: Element = None) -> Element: + """ + Shortly this recursive function transforms dictonary to XML. + Return etree.Element built from dict with following structure:: + + { + 'name': 'device', # tag name + 'text': '', # optional key + 'values': { # optional key, must be a dict of key-value pairs + 'type': 'disk' + }, + children: [] # optional key, must be a list of dicts + } + + Child elements must have the same structure. Infinite `children` nesting + is allowed. + """ + use_ns = False + if isinstance(namespace, str) and isinstance(nsprefix, str): + use_ns = True + # Create element + if root is None: + if use_ns: + element = Element(QName(namespace, tag['name']), + nsmap={nsprefix: namespace}) + else: + element = Element(tag['name']) + else: + if use_ns: + element = SubElement(root, QName(namespace, tag['name'])) + else: + element = SubElement(root, tag['name']) + # Fill up element with content + if 'text' in tag.keys(): + element.text = tag['text'] + if 'values' in tag.keys(): + for key in tag['values'].keys(): + element.set(str(key), str(tag['values'][key])) + if 'children' in tag.keys(): + for child in tag['children']: + element.append( + self.construct_xml(child, + namespace=namespace, + nsprefix=nsprefix, + root=element)) + return element + + def to_string(self): + return (tostring(self.xml, pretty_print=True, + encoding='utf-8').decode().strip()) diff --git a/node_agent/utils/xml.py b/node_agent/utils/xml.py index 3e9d4e1..862f162 100644 --- a/node_agent/utils/xml.py +++ b/node_agent/utils/xml.py @@ -1,54 +1,21 @@ from pathlib import Path from lxml.builder import E -from lxml.etree import Element, QName, SubElement, tostring - -from .mac import random_mac +from lxml.etree import Element, QName, SubElement, tostring, fromstring -XPATH_DOMAIN_NAME = '/domain/name' -XPATH_DOMAIN_TITLE = '/domain/title' -XPATH_DOMAIN_DESCRIPTION = '/domain/description' -XPATH_DOMAIN_METADATA = '/domain/metadata' -XPATH_DOMAIN_MEMORY = '/domain/memory' -XPATH_DOMAIN_CURRENT_MEMORY = '/domain/currentMemory' -XPATH_DOMAIN_VCPU = '/domain/vcpu' -XPATH_DOMAIN_OS = '/domian/os' -XPATH_DOMAIN_CPU = '/domain/cpu' - - -class XMLConstructor: +class Constructor: """ - The XML constructor. This class builds XML configs for libvirtd. - Features: - - Generate basic virtual machine XML. See gen_domain_xml() - - Generate virtual disk XML. See gen_volume_xml() - - Add arbitrary metadata to XML from special structured dict + The XML constructor. This class builds XML configs for libvirt. """ - def __init__(self, xml: str | None = None): - self.xml_string = xml - self.xml = None - - @property - def domain_xml(self): - return self.xml - - def gen_domain_xml(self, - name: str, - title: str, - vcpus: int, - vcpu_vendor: str, - vcpu_model: str, - memory: int, - volume: Path, - vcpu_features: dict | None = None, - desc: str = "") -> None: + def gen_domain_xml(self, name: str, title: str, desc: str, memory: int, + vcpus: int, domain_type: str, machine: str, arch: str, + boot_order: tuple, cpu: str, mac: str) -> str: """ - Generate default domain XML configuration for virtual machines. - See https://lxml.de/tutorial.html#the-e-factory for details. + Return basic libvirt domain configuration. """ - self.xml = E.domain( + domain = E.domain( E.name(name), E.title(title), E.description(desc), @@ -56,142 +23,55 @@ class XMLConstructor: E.memory(str(memory), unit='MB'), E.currentMemory(str(memory), unit='MB'), E.vcpu(str(vcpus), placement='static'), - E.os( - E.type('hvm', arch='x86_64'), - E.boot(dev='cdrom'), - E.boot(dev='hd'), - ), - E.features( - E.acpi(), - E.apic(), - ), - E.cpu( - E.vendor(vcpu_vendor), - E.model(vcpu_model, fallback='forbid'), - E.topology(sockets='1', dies='1', cores=str(vcpus), - threads='1'), - mode='custom', - match='exact', - check='partial', - ), - E.on_poweroff('destroy'), - E.on_reboot('restart'), - E.on_crash('restart'), - E.pm( - E('suspend-to-mem', enabled='no'), - E('suspend-to-disk', enabled='no'), - ), - E.devices( - E.emulator('/usr/bin/qemu-system-x86_64'), - E.disk( - E.driver(name='qemu', type='qcow2', cache='writethrough'), - E.source(file=volume), - E.target(dev='vda', bus='virtio'), - type='file', - device='disk', - ), - E.interface( - E.source(network='default'), - E.mac(address=random_mac()), - type='network', - ), - E.graphics( - E.listen(type='address'), - type='vnc', port='-1', autoport='yes' - ), - E.video( - E.model(type='vga', vram='16384', heads='1', primary='yes'), - E.address(type='pci', domain='0x0000', bus='0x00', - slot='0x02', function='0x0'), - ), - ), - type='kvm', + type='kvm' ) + os = E.os(E.type(domain_type, machine=machine, arch=arch)) + for dev in boot_order: + os.append(E.boot(dev=dev)) + domain.append(os) + domain.append(E.features(E.acpi(), E.apic())) + domain.append(fromstring(cpu)) + domain.append(E.on_poweroff('destroy')) + domain.append(E.on_reboot('restart')) + domain.append(E.on_crash('restart')) + domain.append(E.pm( + E('suspend-to-mem', enabled='no'), + E('suspend-to-disk', enabled='no')) + ) + devices = E.devices() + devices.append(E.emulator('/usr/bin/qemu-system-x86_64')) + devices.append(E.interface( + E.source(network='default'), + E.mac(address=mac), + type='network') + ) + devices.append(E.graphics(type='vnc', port='-1', autoport='yes')) + devices.append(E.input(type='tablet', bus='usb')) + devices.append(E.channel( + E.source(mode='bind'), + E.target(type='virtio', name='org.qemu.guest_agent.0'), + E.address(type='virtio-serial', controller='0', bus='0', port='1'), + type='unix') + ) + devices.append(E.console( + E.target(type='serial', port='0'), + type='pty') + ) + devices.append(E.video( + E.model(type='vga', vram='16384', heads='1', primary='yes')) + ) + domain.append(devices) + return tostring(domain, encoding='unicode', pretty_print=True).strip() - def gen_volume_xml(self, - device_name: str, - file: Path, - bus: str = 'virtio', - cache: str = 'writethrough', - disktype: str = 'file'): - return E.disk(E.driver(name='qemu', type='qcow2', cache=cache), - E.source(file=file), - E.target(dev=device_name, bus=bus), - type=disktype, - device='disk') - - def add_volume(self): - raise NotImplementedError() - - def add_meta(self, data: dict, namespace: str, nsprefix: str) -> None: + def gen_volume_xml(self, dev: str, mode: str) -> str: """ - Add metadata to domain. See: - https://libvirt.org/formatdomain.html#general-metadata + Todo: No hardcode + https://libvirt.org/formatdomain.html#hard-drives-floppy-disks-cdroms """ - metadata = metadata_old = self.xml.xpath('/domain/metadata')[0] - metadata.append( - self.construct_xml( - data, - namespace=namespace, - nsprefix=nsprefix, - )) - self.xml.replace(metadata_old, metadata) - - def remove_meta(self, namespace: str): - """Remove metadata by namespace.""" - raise NotImplementedError() - - def construct_xml(self, - tag: dict, - namespace: str | None = None, - nsprefix: str | None = None, - root: Element = None) -> Element: - """ - Shortly this recursive function transforms dictonary to XML. - Return etree.Element built from dict with following structure:: - - { - 'name': 'devices', # tag name - 'text': '', # optional key - 'values': { # optional key, must be a dict of key-value pairs - 'type': 'disk' - }, - children: [] # optional key, must be a list of dicts - } - - Child elements must have the same structure. Infinite `children` nesting - is allowed. - """ - use_ns = False - if isinstance(namespace, str) and isinstance(nsprefix, str): - use_ns = True - # Create element - if root is None: - if use_ns: - element = Element(QName(namespace, tag['name']), - nsmap={nsprefix: namespace}) - else: - element = Element(tag['name']) - else: - if use_ns: - element = SubElement(root, QName(namespace, tag['name'])) - else: - element = SubElement(root, tag['name']) - # Fill up element with content - if 'text' in tag.keys(): - element.text = tag['text'] - if 'values' in tag.keys(): - for key in tag['values'].keys(): - element.set(str(key), str(tag['values'][key])) - if 'children' in tag.keys(): - for child in tag['children']: - element.append( - self.construct_xml(child, - namespace=namespace, - nsprefix=nsprefix, - root=element)) - return element - - def to_string(self): - return (tostring(self.xml, pretty_print=True, - encoding='utf-8').decode().strip()) + volume = E.disk(type='file', device='disk') + volume.append(E.driver(name='qemu', type='qcow2', cache='writethrough')) + volume.append(E.source(file=path)) + volume.append(E.target(dev=dev, bus='virtio')) + if mode.lower() == 'ro': + volume.append(E.readonly()) + return tostring(volume, encoding='unicode', pretty_print=True).strip() diff --git a/node_agent/vm/__init__.py b/node_agent/vm/__init__.py index c190a25..5de936b 100644 --- a/node_agent/vm/__init__.py +++ b/node_agent/vm/__init__.py @@ -1,3 +1,5 @@ from .exceptions import * -from .ga import QemuAgent -from .main import VirtualMachine +from .guest_agent import QemuAgent +from .virtual_machine import VirtualMachine +from .installer import VirtualMachineInstaller +from .hardware import vCPUMode, vCPUTopology diff --git a/node_agent/vm/base.py b/node_agent/vm/base.py index 5338d61..66ae358 100644 --- a/node_agent/vm/base.py +++ b/node_agent/vm/base.py @@ -1,22 +1,31 @@ import libvirt -from .exceptions import VMError, VMNotFound +from .exceptions import VMError class VirtualMachineBase: - def __init__(self, session: 'LibvirtSession', name: str): - self.domname = name - self.session = session.session # virConnect object - self.config = session.config # ConfigLoader object - self.domain = self._get_domain(name) + def __init__(self, domain: libvirt.virDomain): + self.domain = domain + self.domain_name = self._get_domain_name() + self.domain_info = self._get_domain_info() - def _get_domain(self, name: str) -> libvirt.virDomain: - """Get virDomain object by name to manipulate with domain.""" + def _get_domain_name(self): try: - domain = self.session.lookupByName(name) - if domain is not None: - return domain - raise VMNotFound(name) + return self.domain.name() except libvirt.libvirtError as err: - raise VMError(err) from err + raise VMError(f'Cannot get domain name: {err}') from err + + def _get_domain_info(self): + # https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainInfo + try: + info = self.domain.info() + return { + 'state': info[0], + 'max_memory': info[1], + 'memory': info[2], + 'nproc': info[3], + 'cputime': info[4] + } + except libvirt.libvirtError as err: + raise VMError(f'Cannot get domain info: {err}') from err diff --git a/node_agent/vm/exceptions.py b/node_agent/vm/exceptions.py index e99e480..680bfab 100644 --- a/node_agent/vm/exceptions.py +++ b/node_agent/vm/exceptions.py @@ -8,7 +8,7 @@ class VMError(Exception): class VMNotFound(Exception): - def __init__(self, domain, message='VM not found: {domain}'): + def __init__(self, domain, message='VM not found vm={domain}'): self.domain = domain self.message = message.format(domain=domain) super().__init__(self.message) diff --git a/node_agent/vm/ga.py b/node_agent/vm/guest_agent.py similarity index 82% rename from node_agent/vm/ga.py rename to node_agent/vm/guest_agent.py index 52b31dd..0a48b1d 100644 --- a/node_agent/vm/ga.py +++ b/node_agent/vm/guest_agent.py @@ -12,8 +12,9 @@ from .exceptions import QemuAgentError logger = logging.getLogger(__name__) -QEMU_TIMEOUT = 60 # seconds -POLL_INTERVAL = 0.3 # also seconds +# Note that if no QEMU_TIMEOUT libvirt cannot connect to agent +QEMU_TIMEOUT = 60 # in seconds +POLL_INTERVAL = 0.3 # also in seconds class QemuAgent(VirtualMachineBase): @@ -28,12 +29,9 @@ class QemuAgent(VirtualMachineBase): must be passed as string. Wraps execute() method. """ - def __init__(self, - session: 'LibvirtSession', - name: str, - timeout: int | None = None, + def __init__(self, domain: libvirt.virDomain, timeout: int | None = None, flags: int | None = None): - super().__init__(session, name) + super().__init__(domain) self.timeout = timeout or QEMU_TIMEOUT # timeout for guest agent self.flags = flags or libvirt_qemu.VIR_DOMAIN_QEMU_MONITOR_COMMAND_DEFAULT @@ -110,7 +108,11 @@ class QemuAgent(VirtualMachineBase): ) def _execute(self, command: dict): - logging.debug('Execute command: vm=%s cmd=%s', self.domname, command) + logging.debug('Execute command: vm=%s cmd=%s', self.domain_name, + command) + if self.domain_info['state'] != libvirt.VIR_DOMAIN_RUNNING: + raise GuestAgentError( + f'Cannot execute command: vm={self.domain_name} is not running') try: return libvirt_qemu.qemuAgentCommand( self.domain, # virDomain object @@ -119,7 +121,9 @@ class QemuAgent(VirtualMachineBase): self.flags, ) except libvirt.libvirtError as err: - raise QemuAgentError(err) from err + raise QemuAgentError( + f'Cannot execute command on vm={self.domain_name}: {err}' + ) from err def _get_cmd_result( self, pid: int, decode_output: bool = False, wait: bool = True, @@ -131,7 +135,8 @@ class QemuAgent(VirtualMachineBase): output = json.loads(self._execute(cmd)) return self._return_tuple(output, decode=decode_output) - logger.debug('Start polling command pid=%s', pid) + logger.debug('Start polling command pid=%s on vm=%s', pid, + self.domain_name) start_time = time() while True: output = json.loads(self._execute(cmd)) @@ -141,10 +146,12 @@ class QemuAgent(VirtualMachineBase): now = time() if now - start_time > timeout: raise QemuAgentError( - f'Polling command pid={pid} took longer than {timeout} seconds.' + f'Polling command pid={pid} on vm={self.domain_name} ' + f'took longer than {timeout} seconds.' ) - logger.debug('Polling command pid=%s finished, time taken: %s seconds', - pid, int(time() - start_time)) + logger.debug('Polling command pid=%s on vm=%s finished, ' + 'time taken: %s seconds', + pid, self.domain_name, int(time() - start_time)) return self._return_tuple(output, decode=decode_output) def _return_tuple(self, output: dict, decode: bool = False): diff --git a/node_agent/vm/hardware.py b/node_agent/vm/hardware.py new file mode 100644 index 0000000..ef0c1a6 --- /dev/null +++ b/node_agent/vm/hardware.py @@ -0,0 +1,81 @@ +import textwrap +from enum import Enum +from collections import UserDict + +import libvirt +from lxml.etree import SubElement, fromstring, tostring + + +class Boot(Enum): + BIOS = 'bios' + UEFI = 'uefi' + + +class vCPUMode(Enum): + HOST_MODEL = 'host-model' + HOST_PASSTHROUGTH = 'host-passthrougth' + CUSTOM = 'custom' + MAXIMUM = 'maximum' + + +class DomainCapabilities: + + def __init__(self, session: libvirt.virConnect): + self.session = session + self.domcaps = fromstring( + self.session.getDomainCapabilities()) + + @property + def arch(self): + return self.domcaps.xpath('/domainCapabilities/arch')[0].text + + @property + def virttype(self): + return self.domcaps.xpath('/domainCapabilities/domain')[0].text + + @property + def emulator(self): + return self.domcaps.xpath('/domainCapabilities/path')[0].text + + @property + def machine(self): + return self.domcaps.xpath('/domainCapabilities/machine')[0].text + + def best_cpu(self, mode: vCPUMode) -> str: + """ + See https://libvirt.org/html/libvirt-libvirt-host.html + #virConnectBaselineHypervisorCPU + """ + cpus = self.domcaps.xpath( + f'/domainCapabilities/cpu/mode[@name="{mode}"]')[0] + cpus.tag = 'cpu' + for attr in cpus.attrib.keys(): + del cpus.attrib[attr] + arch = SubElement(cpus, 'arch') + arch.text = self.arch + xmlcpus = tostring(cpus, encoding='unicode', pretty_print=True) + xml = self.session.baselineHypervisorCPU(self.emulator, + self.arch, self.machine, self.virttype, [xmlcpus]) + return textwrap.indent(xml, ' ' * 2) + + +class vCPUTopology(UserDict): + """ + CPU topology schema ``{'sockets': 1, 'cores': 4, 'threads': 1}``:: + + + """ + + def __init__(self, topology: dict): + super().__init__(self._validate(topology)) + + def _validate(self, topology: dict): + if isinstance(topology, dict): + if ['sockets', 'cores', 'threads'] != list(topology.keys()): + raise ValueError("Topology must have 'sockets', 'cores' " + "and 'threads' keys.") + for key in topology.keys(): + if not isinstance(topology[key], int): + raise TypeError(f"Key '{key}' must be 'int'") + return topology + raise TypeError("Topology must be a 'dict'") diff --git a/node_agent/vm/installer.py b/node_agent/vm/installer.py new file mode 100644 index 0000000..742675b --- /dev/null +++ b/node_agent/vm/installer.py @@ -0,0 +1,94 @@ +import re + +import libvirt + +from ..utils.xml import Constructor +from ..utils.mac import random_mac +from .hardware import DomainCapabilities, vCPUMode, vCPUTopology, Boot + + +class vCPUInfo: + pass + +class ImageVolume: + pass + +class CloudInitConfig: + pass + +class BootOrder: + pass + +class VirtualMachineInstaller: + def __init__(self, session: libvirt.virConnect): + self.session = session + self.info = {} + + def install( + self, + name: str | None = None, + title: str | None = None, + description: str = '', + os: str | None = None, + image: ImageVolume | None = None, + volumes: list['VolumeInfo'] | None = None, + vcpus: int = 0, + vcpu_info: vCPUInfo | None = None, + vcpu_mode: vCPUMode | None = None, + vcpu_topology: vCPUTopology | None = None, + memory: int = 0, + boot: Boot = Boot.BIOS, + boot_menu: bool = False, + boot_order: BootOrder = ('cdrom', 'hd'), + cloud_init: CloudInitConfig | None = None): + """ + Install virtual machine with passed parameters. + """ + domcaps = DomainCapabilities(self.session.session) + name = self._validate_name(name) + if vcpu_topology is None: + vcpu_topology = vCPUTopology( + {'sockets': 1, 'cores': vcpus, 'threads': 1}) + self._validate_topology(vcpus, vcpu_topology) + if vcpu_info is None: + if not vcpu_mode: + vcpu_mode = vCPUMode.CUSTOM.value + xml_cpu = domcaps.best_cpu(vcpu_mode) + else: + raise NotImplementedError('Custom CPU not implemented') + xml_domain = Constructor().gen_domain_xml( + name=name, + title=title if title else name, + desc=description if description else '', + vcpus=vcpus, + memory=memory, + domain_type='hvm', + machine=domcaps.machine, + arch=domcaps.arch, + boot_order=('cdrom', 'hd'), + cpu=xml_cpu, + mac=random_mac() + ) + return xml_domain + + def _validate_name(self, name): + if name is None: + raise ValueError("'name' cannot be empty") + if isinstance(name, str): + if not re.match(r"^[a-z0-9_]+$", name, re.I): + raise ValueError( + "'name' can contain only letters, numbers " + "and underscore.") + return name.lower() + raise TypeError(f"'name' must be 'str', not {type(name)}") + + def _validate_topology(self, vcpus, topology): + sockets = topology['sockets'] + cores = topology['cores'] + threads = topology['threads'] + if sockets * cores * threads == vcpus: + return + raise ValueError("CPU topology must match the number of 'vcpus'") + + def _define(self, xml: str): + self.session.defineXML(xml) diff --git a/node_agent/vm/main.py b/node_agent/vm/virtual_machine.py similarity index 57% rename from node_agent/vm/main.py rename to node_agent/vm/virtual_machine.py index dc473d9..32967d4 100644 --- a/node_agent/vm/main.py +++ b/node_agent/vm/virtual_machine.py @@ -13,7 +13,7 @@ class VirtualMachine(VirtualMachineBase): @property def name(self): - return self.domname + return self.domain_name @property def status(self) -> str: @@ -27,7 +27,7 @@ class VirtualMachine(VirtualMachineBase): state = self.domain.state()[0] except libvirt.libvirtError as err: raise VMError( - f'Cannot fetch VM status vm={self.domname}: {err}') from err + f'Cannot fetch VM status vm={self.domain_name}: {err}') from err STATES = { libvirt.VIR_DOMAIN_NOSTATE: 'nostate', libvirt.VIR_DOMAIN_RUNNING: 'running', @@ -57,20 +57,21 @@ class VirtualMachine(VirtualMachineBase): return False except libvirt.libvirtError as err: raise VMError( - f'Cannot get autostart status vm={self.domname}: {err}' + f'Cannot get autostart status vm={self.domain_name}: {err}' ) from err def start(self) -> None: """Start defined VM.""" - logger.info('Starting VM: vm=%s', self.domname) + logger.info('Starting VM: vm=%s', self.domain_name) if self.is_running: - logger.debug('VM vm=%s is already started, nothing to do', - self.domname) + logger.warning('VM vm=%s is already started, nothing to do', + self.domain_name) return try: self.domain.create() except libvirt.libvirtError as err: - raise VMError(f'Cannot start vm={self.domname}: {err}') from err + raise VMError( + f'Cannot start vm={self.domain_name}: {err}') from err def shutdown(self, mode: str | None = None) -> None: """ @@ -78,7 +79,7 @@ class VirtualMachine(VirtualMachineBase): * GUEST_AGENT - use guest agent * NORMAL - use method choosen by hypervisor to shutdown machine * SIGTERM - send SIGTERM to QEMU process, destroy machine gracefully - * SIGKILL - send SIGKILL, this option may corrupt guest data! + * SIGKILL - send SIGKILL to QEMU process. May corrupt guest data! If mode is not passed use 'NORMAL' mode. """ MODES = { @@ -90,7 +91,7 @@ class VirtualMachine(VirtualMachineBase): if mode is None: mode = 'NORMAL' if not isinstance(mode, str): - raise ValueError(f'Mode must be a string, not {type(mode)}') + raise ValueError(f"Mode must be a 'str', not {type(mode)}") if mode.upper() not in MODES: raise ValueError(f"Unsupported mode: '{mode}'") try: @@ -99,7 +100,7 @@ class VirtualMachine(VirtualMachineBase): elif mode in ['SIGTERM', 'SIGKILL']: self.domain.destroyFlags(flags=MODES.get(mode)) except libvirt.libvirtError as err: - raise VMError(f'Cannot shutdown vm={self.domname} with ' + raise VMError(f'Cannot shutdown vm={self.domain_name} with ' f'mode={mode}: {err}') from err def reset(self) -> None: @@ -116,16 +117,18 @@ class VirtualMachine(VirtualMachineBase): try: self.domain.reset() except libvirt.libvirtError as err: - raise VMError(f'Cannot reset vm={self.domname}: {err}') from err + raise VMError( + f'Cannot reset vm={self.domain_name}: {err}') from err def reboot(self) -> None: """Send ACPI signal to guest OS to reboot. OS may ignore this.""" try: self.domain.reboot() except libvirt.libvirtError as err: - raise VMError(f'Cannot reboot vm={self.domname}: {err}') from err + raise VMError( + f'Cannot reboot vm={self.domain_name}: {err}') from err - def autostart(self, enable: bool) -> None: + def set_autostart(self, enable: bool) -> None: """ Configure VM to be automatically started when the host machine boots. """ @@ -136,13 +139,57 @@ class VirtualMachine(VirtualMachineBase): try: self.domain.setAutostart(autostart_flag) except libvirt.libvirtError as err: - raise VMError(f'Cannot set autostart vm={self.domname} ' + raise VMError(f'Cannot set autostart vm={self.domain_name} ' f'autostart={autostart_flag}: {err}') from err - def set_vcpus(self, count: int): + def set_vcpus(self, nvcpus: int, hotplug: bool = False): + """ + Set vCPUs for VM. If `hotplug` is True set vCPUs on running VM. + If VM is not running set `hotplug` to False. If `hotplug` is True + and VM is not currently running vCPUs will set in config and will + applied when machine boot. + + NB: Note that if this call is executed before the guest has + finished booting, the guest may fail to process the change. + """ + if nvcpus == 0: + raise VMError(f'Cannot set zero vCPUs vm={self.domain_name}') + if hotplug and self.domain_info['state'] == libvirt.VIR_DOMAIN_RUNNING: + flags = (libvirt.VIR_DOMAIN_AFFECT_LIVE + + libvirt.VIR_DOMAIN_AFFECT_CONFIG) + else: + flags = libvirt.VIR_DOMAIN_AFFECT_CONFIG + try: + self.domain.setVcpusFlags(nvcpus, flags=flags) + except libvirt.libvirtError as err: + raise VMError( + f'Cannot set vCPUs for vm={self.domain_name}: {err}') from err + + def set_memory(self, memory: int, hotplug: bool = False): + """ + Set momory for VM. `memory` must be passed in mebibytes. Internally + converted to kibibytes. If `hotplug` is True set memory for running + VM, else set memory in config and will applied when machine boot. + If `hotplug` is True and machine is not currently running set memory + in config. + """ + if memory == 0: + raise VMError(f'Cannot set zero memory vm={self.domain_name}') + if hotplug and self.domain_info['state'] == libvirt.VIR_DOMAIN_RUNNING: + flags = (libvirt.VIR_DOMAIN_AFFECT_LIVE + + libvirt.VIR_DOMAIN_AFFECT_CONFIG) + else: + flags = libvirt.VIR_DOMAIN_AFFECT_CONFIG + try: + self.domain.setVcpusFlags(memory * 1024, flags=flags) + except libvirt.libvirtError as err: + raise VMError( + f'Cannot set memory for vm={self.domain_name}: {err}') from err + + def attach_device(self, device: str): pass - def set_ram(self, count: int): + def detach_device(self, device: str): pass def list_ssh_keys(self, user: str): @@ -156,3 +203,11 @@ class VirtualMachine(VirtualMachineBase): def set_user_password(self, user: str): pass + + def dump_xml(self) -> str: + return self.domain.XMLDesc() + + def delete(self, delete_volumes: bool = False): + """Undefine VM.""" + self.shutdown(method='SIGTERM') + self.domain.undefine() diff --git a/node_agent/volume/storage_pool.py b/node_agent/volume/storage_pool.py new file mode 100644 index 0000000..96addce --- /dev/null +++ b/node_agent/volume/storage_pool.py @@ -0,0 +1,9 @@ +import libvirt + + +class StoragePool: + def __init__(self, pool: libvirt.virStoragePool): + self.pool = pool + + def create_volume(self): + pass diff --git a/node_agent/volume/volume.py b/node_agent/volume/volume.py new file mode 100644 index 0000000..6be8ce5 --- /dev/null +++ b/node_agent/volume/volume.py @@ -0,0 +1,23 @@ +import libvirt + + +class VolumeInfo: + """ + Volume info schema + {'type': 'local', 'system': True, 'size': 102400, 'mode': 'rw'} + """ + pass + + +class Volume: + def __init__(self, pool: libvirt.virStorageVol): + self.pool = pool + + def lookup_by_path(self): + pass + + def generate_xml(self): + pass + + def create(self): + pass