Merge pull request #1801 from GNS3/qemu-config-disk

QEMU config disk - enable QEMU config import/export

gns3server/compute/qemu/__init__.py

@@ -27,10 +27,13 @@ import re
 import subprocess
 
 from ...utils.asyncio import subprocess_check_output
+from ...utils.get_resource import get_resource
 from ..base_manager import BaseManager
+from ..error import NodeError, ImageMissingError
 from .qemu_error import QemuError
 from .qemu_vm import QemuVM
 from .utils.guest_cid import get_next_guest_cid
+from .utils.ziputils import unpack_zip
 
 import logging
 log = logging.getLogger(__name__)
@@ -45,6 +48,8 @@ class Qemu(BaseManager):
 
         super().__init__()
         self._guest_cid_lock = asyncio.Lock()
+        self.config_disk = "config.img"
+        self._init_config_disk()
 
     async def create_node(self, *args, **kwargs):
         """
@@ -343,3 +348,21 @@ class Qemu(BaseManager):
             log.info("Qemu disk '{}' extended by {} MB".format(path, extend))
         except (OSError, subprocess.SubprocessError) as e:
             raise QemuError("Could not update disk image {}:{}".format(path, e))
+
+    def _init_config_disk(self):
+        """
+        Initialize the default config disk
+        """
+
+        try:
+            self.get_abs_image_path(self.config_disk)
+        except (NodeError, ImageMissingError):
+            config_disk_zip = get_resource("compute/qemu/resources/{}.zip".format(self.config_disk))
+            if config_disk_zip and os.path.exists(config_disk_zip):
+                directory = self.get_images_directory()
+                try:
+                    unpack_zip(config_disk_zip, directory)
+                except OSError as e:
+                    log.warning("Config disk creation: {}".format(e))
+            else:
+                log.warning("Config disk: image '{}' missing".format(self.config_disk))

gns3server/compute/qemu/qemu_vm.py

@@ -26,6 +26,7 @@ import re
 import shlex
 import math
 import shutil
+import struct
 import asyncio
 import socket
 import gns3server
@@ -37,7 +38,9 @@ from gns3server.utils import parse_version, shlex_quote
 from gns3server.utils.asyncio import subprocess_check_output, cancellable_wait_run_in_executor
 from .qemu_error import QemuError
 from .utils.qcow2 import Qcow2, Qcow2Error
+from .utils.ziputils import pack_zip, unpack_zip
 from ..adapters.ethernet_adapter import EthernetAdapter
+from ..error import NodeError, ImageMissingError
 from ..nios.nio_udp import NIOUDP
 from ..nios.nio_tap import NIOTAP
 from ..base_node import BaseNode
@@ -100,10 +103,10 @@ class QemuVM(BaseNode):
         self._hdb_disk_image = ""
         self._hdc_disk_image = ""
         self._hdd_disk_image = ""
-        self._hda_disk_interface = "ide"
+        self._hda_disk_interface = "none"
-        self._hdb_disk_interface = "ide"
+        self._hdb_disk_interface = "none"
-        self._hdc_disk_interface = "ide"
+        self._hdc_disk_interface = "none"
-        self._hdd_disk_interface = "ide"
+        self._hdd_disk_interface = "none"
         self._cdrom_image = ""
         self._bios_image = ""
         self._boot_priority = "c"
@@ -119,12 +122,28 @@ class QemuVM(BaseNode):
         self._kernel_command_line = ""
         self._legacy_networking = False
         self._replicate_network_connection_state = True
+        self._create_config_disk = False
         self._on_close = "power_off"
         self._cpu_throttling = 0  # means no CPU throttling
         self._process_priority = "low"
 
         self.mac_address = ""  # this will generate a MAC address
         self.adapters = 1  # creates 1 adapter by default
+
+        # config disk
+        self.config_disk_name = self.manager.config_disk
+        self.config_disk_image = ""
+        if self.config_disk_name:
+            if not shutil.which("mcopy"):
+                log.warning("Config disk: 'mtools' are not installed.")
+                self.config_disk_name = ""
+            else:
+                try:
+                    self.config_disk_image = self.manager.get_abs_image_path(self.config_disk_name)
+                except (NodeError, ImageMissingError):
+                    log.warning("Config disk: image '{}' missing".format(self.config_disk_name))
+                    self.config_disk_name = ""
+
         log.info('QEMU VM "{name}" [{id}] has been created'.format(name=self._name, id=self._id))
 
     @property
@@ -641,6 +660,30 @@ class QemuVM(BaseNode):
             log.info('QEMU VM "{name}" [{id}] has disabled network connection state replication'.format(name=self._name, id=self._id))
         self._replicate_network_connection_state = replicate_network_connection_state
 
+    @property
+    def create_config_disk(self):
+        """
+        Returns whether a config disk is automatically created on HDD disk interface (secondary slave)
+
+        :returns: boolean
+        """
+
+        return self._create_config_disk
+
+    @create_config_disk.setter
+    def create_config_disk(self, create_config_disk):
+        """
+        Sets whether a config disk is automatically created on HDD disk interface (secondary slave)
+
+        :param replicate_network_connection_state: boolean
+        """
+
+        if create_config_disk:
+            log.info('QEMU VM "{name}" [{id}] has enabled the config disk creation feature'.format(name=self._name, id=self._id))
+        else:
+            log.info('QEMU VM "{name}" [{id}] has disabled the config disk creation feature'.format(name=self._name, id=self._id))
+        self._create_config_disk = create_config_disk
+
     @property
     def on_close(self):
         """
@@ -1124,6 +1167,7 @@ class QemuVM(BaseNode):
             self._stop_cpulimit()
             if self.on_close != "save_vm_state":
                 await self._clear_save_vm_stated()
+            await self._export_config()
             await super().stop()
 
     async def _open_qemu_monitor_connection_vm(self, timeout=10):
@@ -1661,6 +1705,105 @@ class QemuVM(BaseNode):
         log.info("{} returned with {}".format(self._get_qemu_img(), retcode))
         return retcode
 
+    async def _mcopy(self, image, *args):
+        try:
+            # read offset of first partition from MBR
+            with open(image, "rb") as img_file:
+                mbr = img_file.read(512)
+            part_type, offset, signature = struct.unpack("<450xB3xL52xH", mbr)
+            if signature != 0xAA55:
+                raise OSError("mcopy failure: {}: invalid MBR".format(image))
+            if part_type not in (1, 4, 6, 11, 12, 14):
+                raise OSError("mcopy failure: {}: invalid partition type {:02X}"
+                              .format(image, part_type))
+            part_image = image + "@@{}S".format(offset)
+
+            process = await asyncio.create_subprocess_exec(
+                "mcopy", "-i", part_image, *args,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                cwd=self.working_dir)
+            (stdout, _) = await process.communicate()
+            retcode = process.returncode
+        except (OSError, subprocess.SubprocessError) as e:
+            raise OSError("mcopy failure: {}".format(e))
+        if retcode != 0:
+            stdout = stdout.decode("utf-8").rstrip()
+            if stdout:
+                raise OSError("mcopy failure: {}".format(stdout))
+            else:
+                raise OSError("mcopy failure: return code {}".format(retcode))
+
+    async def _export_config(self):
+        disk_name = getattr(self, "config_disk_name")
+        if not disk_name:
+            return
+        disk = os.path.join(self.working_dir, disk_name)
+        if not os.path.exists(disk):
+            return
+        config_dir = os.path.join(self.working_dir, "configs")
+        zip_file = os.path.join(self.working_dir, "config.zip")
+        try:
+            os.mkdir(config_dir)
+            await self._mcopy(disk, "-s", "-m", "-n", "--", "::/", config_dir)
+            if os.path.exists(zip_file):
+                os.remove(zip_file)
+            pack_zip(zip_file, config_dir)
+        except OSError as e:
+            log.warning("Can't export config: {}".format(e))
+            self.project.emit("log.warning", {"message": "{}: Can't export config: {}".format(self._name, e)})
+        shutil.rmtree(config_dir, ignore_errors=True)
+
+    async def _import_config(self):
+        disk_name = getattr(self, "config_disk_name")
+        zip_file = os.path.join(self.working_dir, "config.zip")
+        if not disk_name or not os.path.exists(zip_file):
+            return
+        config_dir = os.path.join(self.working_dir, "configs")
+        disk = os.path.join(self.working_dir, disk_name)
+        disk_tmp = disk + ".tmp"
+        try:
+            os.mkdir(config_dir)
+            shutil.copyfile(getattr(self, "config_disk_image"), disk_tmp)
+            unpack_zip(zip_file, config_dir)
+            config_files = [os.path.join(config_dir, fname)
+                            for fname in os.listdir(config_dir)]
+            if config_files:
+                await self._mcopy(disk_tmp, "-s", "-m", "-o", "--", *config_files, "::/")
+            os.replace(disk_tmp, disk)
+        except OSError as e:
+            log.warning("Can't import config: {}".format(e))
+            self.project.emit("log.warning", {"message": "{}: Can't import config: {}".format(self._name, e)})
+            if os.path.exists(disk_tmp):
+                os.remove(disk_tmp)
+                os.remove(zip_file)
+        shutil.rmtree(config_dir, ignore_errors=True)
+
+    def _disk_interface_options(self, disk, disk_index, interface, format=None):
+        options = []
+        extra_drive_options = ""
+        if format:
+            extra_drive_options += ",format={}".format(format)
+
+        if interface == "sata":
+            # special case, sata controller doesn't exist in Qemu
+            options.extend(["-device", 'ahci,id=ahci{}'.format(disk_index)])
+            options.extend(["-drive", 'file={},if=none,id=drive{},index={},media=disk{}'.format(disk, disk_index, disk_index, extra_drive_options)])
+            options.extend(["-device", 'ide-drive,drive=drive{},bus=ahci{}.0,id=drive{}'.format(disk_index, disk_index, disk_index)])
+        elif interface == "nvme":
+            options.extend(["-drive", 'file={},if=none,id=drive{},index={},media=disk{}'.format(disk, disk_index, disk_index, extra_drive_options)])
+            options.extend(["-device", 'nvme,drive=drive{},serial={}'.format(disk_index, disk_index)])
+        elif interface == "scsi":
+            options.extend(["-device", 'virtio-scsi-pci,id=scsi{}'.format(disk_index)])
+            options.extend(["-drive", 'file={},if=none,id=drive{},index={},media=disk{}'.format(disk, disk_index, disk_index, extra_drive_options)])
+            options.extend(["-device", 'scsi-hd,drive=drive{}'.format(disk_index)])
+        #elif interface == "sd":
+        #    options.extend(["-drive", 'file={},id=drive{},index={}{}'.format(disk, disk_index, disk_index, extra_drive_options)])
+        #    options.extend(["-device", 'sd-card,drive=drive{},id=drive{}'.format(disk_index, disk_index, disk_index)])
+        else:
+            options.extend(["-drive", 'file={},if={},index={},media=disk,id=drive{}{}'.format(disk, interface, disk_index, disk_index, extra_drive_options)])
+        return options
+
     async def _disk_options(self):
         options = []
         qemu_img_path = self._get_qemu_img()
@@ -1669,13 +1812,16 @@ class QemuVM(BaseNode):
 
         for disk_index, drive in enumerate(drives):
             disk_image = getattr(self, "_hd{}_disk_image".format(drive))
-            interface = getattr(self, "hd{}_disk_interface".format(drive))
-
             if not disk_image:
                 continue
 
-            disk_name = "hd" + drive
+            interface = getattr(self, "hd{}_disk_interface".format(drive))
+            # fail-safe: use "ide" if there is a disk image and no interface type has been explicitly configured
+            if interface == "none":
+                interface = "ide"
+                setattr(self, "hd{}_disk_interface".format(drive), interface)
 
+            disk_name = "hd" + drive
             if not os.path.isfile(disk_image) or not os.path.exists(disk_image):
                 if os.path.islink(disk_image):
                     raise QemuError("{} disk image '{}' linked to '{}' is not accessible".format(disk_name, disk_image, os.path.realpath(disk_image)))
@@ -1725,23 +1871,29 @@ class QemuVM(BaseNode):
             else:
                 disk = disk_image
 
-            if interface == "sata":
+            options.extend(self._disk_interface_options(disk, disk_index, interface))
-                # special case, sata controller doesn't exist in Qemu
+
-                options.extend(["-device", 'ahci,id=ahci{}'.format(disk_index)])
+        # config disk
-                options.extend(["-drive", 'file={},if=none,id=drive{},index={},media=disk'.format(disk, disk_index, disk_index)])
+        disk_image = getattr(self, "config_disk_image")
-                options.extend(["-device", 'ide-drive,drive=drive{},bus=ahci{}.0,id=drive{}'.format(disk_index, disk_index, disk_index)])
+        if disk_image and self._create_config_disk:
-            elif interface == "nvme":
+            if getattr(self, "_hdd_disk_image"):
-                options.extend(["-drive", 'file={},if=none,id=drive{},index={},media=disk'.format(disk, disk_index, disk_index)])
+                log.warning("Config disk: blocked by disk image 'hdd'")
-                options.extend(["-device", 'nvme,drive=drive{},serial={}'.format(disk_index, disk_index)])
-            elif interface == "scsi":
-                options.extend(["-device", 'virtio-scsi-pci,id=scsi{}'.format(disk_index)])
-                options.extend(["-drive", 'file={},if=none,id=drive{},index={},media=disk'.format(disk, disk_index, disk_index)])
-                options.extend(["-device", 'scsi-hd,drive=drive{}'.format(disk_index)])
-            #elif interface == "sd":
-            #    options.extend(["-drive", 'file={},id=drive{},index={}'.format(disk, disk_index, disk_index)])
-            #    options.extend(["-device", 'sd-card,drive=drive{},id=drive{}'.format(disk_index, disk_index, disk_index)])
             else:
-                options.extend(["-drive", 'file={},if={},index={},media=disk,id=drive{}'.format(disk, interface, disk_index, disk_index)])
+                disk_name = getattr(self, "config_disk_name")
+                disk = os.path.join(self.working_dir, disk_name)
+                if self.hdd_disk_interface == "none":
+                    # use the HDA interface type if none has been configured for HDD
+                    self.hdd_disk_interface = getattr(self, "hda_disk_interface", "none")
+                await self._import_config()
+                disk_exists = os.path.exists(disk)
+                if not disk_exists:
+                    try:
+                        shutil.copyfile(disk_image, disk)
+                        disk_exists = True
+                    except OSError as e:
+                        log.warning("Could not create '{}' disk image: {}".format(disk_name, e))
+                if disk_exists:
+                    options.extend(self._disk_interface_options(disk, 3, self.hdd_disk_interface, "raw"))
 
         return options
 

gns3server/compute/qemu/utils/ziputils.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2020 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import time
+import shutil
+import zipfile
+
+def pack_zip(filename, root_dir=None, base_dir=None):
+    """Create a zip archive"""
+
+    if filename[-4:].lower() == ".zip":
+        filename = filename[:-4]
+    shutil.make_archive(filename, 'zip', root_dir, base_dir)
+
+def unpack_zip(filename, extract_dir=None):
+    """Unpack a zip archive"""
+
+    dirs = []
+    if not extract_dir:
+        extract_dir = os.getcwd()
+
+    try:
+        with zipfile.ZipFile(filename, 'r') as zfile:
+            for zinfo in zfile.infolist():
+                fname = os.path.join(extract_dir, zinfo.filename)
+                date_time = time.mktime(zinfo.date_time + (0, 0, -1))
+                zfile.extract(zinfo, extract_dir)
+
+                # update timestamp
+                if zinfo.is_dir():
+                    dirs.append((fname, date_time))
+                else:
+                    os.utime(fname, (date_time, date_time))
+        # update timestamp of directories
+        for fname, date_time in reversed(dirs):
+            os.utime(fname, (date_time, date_time))
+    except zipfile.BadZipFile:
+        raise shutil.ReadError("%s is not a zip file" % filename)

gns3server/schemas/qemu.py

@@ -206,6 +206,10 @@ QEMU_CREATE_SCHEMA = {
             "description": "Replicate the network connection state for links in Qemu",
             "type": ["boolean", "null"],
         },
+        "create_config_disk": {
+            "description": "Automatically create a config disk on HDD disk interface (secondary slave)",
+            "type": ["boolean", "null"],
+        },
         "on_close": {
             "description": "Action to execute on the VM is closed",
             "enum": ["power_off", "shutdown_signal", "save_vm_state"],
@@ -412,6 +416,10 @@ QEMU_UPDATE_SCHEMA = {
             "description": "Replicate the network connection state for links in Qemu",
             "type": ["boolean", "null"],
         },
+        "create_config_disk": {
+            "description": "Automatically create a config disk on HDD disk interface (secondary slave)",
+            "type": ["boolean", "null"],
+        },
         "on_close": {
             "description": "Action to execute on the VM is closed",
             "enum": ["power_off", "shutdown_signal", "save_vm_state"],
@@ -631,6 +639,10 @@ QEMU_OBJECT_SCHEMA = {
             "description": "Replicate the network connection state for links in Qemu",
             "type": "boolean",
         },
+        "create_config_disk": {
+            "description": "Automatically create a config disk on HDD disk interface (secondary slave)",
+            "type": ["boolean", "null"],
+        },
         "on_close": {
             "description": "Action to execute on the VM is closed",
             "enum": ["power_off", "shutdown_signal", "save_vm_state"],
@@ -704,6 +716,7 @@ QEMU_OBJECT_SCHEMA = {
                  "kernel_command_line",
                  "legacy_networking",
                  "replicate_network_connection_state",
+                 "create_config_disk",
                  "on_close",
                  "cpu_throttling",
                  "process_priority",

gns3server/schemas/qemu_template.py

@@ -123,7 +123,7 @@ QEMU_TEMPLATE_PROPERTIES = {
     "hda_disk_interface": {
         "description": "QEMU hda interface",
         "enum": ["ide", "sata", "nvme", "scsi", "sd", "mtd", "floppy", "pflash", "virtio", "none"],
-        "default": "ide"
+        "default": "none"
     },
     "hdb_disk_image": {
         "description": "QEMU hdb disk image path",
@@ -133,7 +133,7 @@ QEMU_TEMPLATE_PROPERTIES = {
     "hdb_disk_interface": {
         "description": "QEMU hdb interface",
         "enum": ["ide", "sata", "nvme", "scsi", "sd", "mtd", "floppy", "pflash", "virtio", "none"],
-        "default": "ide"
+        "default": "none"
     },
     "hdc_disk_image": {
         "description": "QEMU hdc disk image path",
@@ -143,7 +143,7 @@ QEMU_TEMPLATE_PROPERTIES = {
     "hdc_disk_interface": {
         "description": "QEMU hdc interface",
         "enum": ["ide", "sata", "nvme", "scsi", "sd", "mtd", "floppy", "pflash", "virtio", "none"],
-        "default": "ide"
+        "default": "none"
     },
     "hdd_disk_image": {
         "description": "QEMU hdd disk image path",
@@ -153,7 +153,7 @@ QEMU_TEMPLATE_PROPERTIES = {
     "hdd_disk_interface": {
         "description": "QEMU hdd interface",
         "enum": ["ide", "sata", "nvme", "scsi", "sd", "mtd", "floppy", "pflash", "virtio", "none"],
-        "default": "ide"
+        "default": "none"
     },
     "cdrom_image": {
         "description": "QEMU cdrom image path",
@@ -190,6 +190,11 @@ QEMU_TEMPLATE_PROPERTIES = {
         "type": "boolean",
         "default": True
     },
+    "create_config_disk": {
+        "description": "Automatically create a config disk on HDD disk interface (secondary slave)",
+        "type": "boolean",
+        "default": True
+    },
     "on_close": {
         "description": "Action to execute on the VM is closed",
         "enum": ["power_off", "shutdown_signal", "save_vm_state"],

tests/compute/qemu/test_qemu_vm.py

@@ -337,6 +337,7 @@ def test_set_qemu_path_kvm_binary(vm, fake_qemu_binary):
 
 async def test_set_platform(compute_project, manager):
 
+    manager.config_disk = None  # avoids conflict with config.img support
     with patch("shutil.which", return_value="/bin/qemu-system-x86_64") as which_mock:
         with patch("gns3server.compute.qemu.QemuVM._check_qemu_path"):
             vm = QemuVM("test", "00010203-0405-0607-0809-0a0b0c0d0e0f", compute_project, manager, platform="x86_64")

tests/compute/test_manager.py

@@ -18,7 +18,7 @@
 import uuid
 import os
 import pytest
-from unittest.mock import patch
+from unittest.mock import patch, MagicMock
 from tests.utils import asyncio_patch
 
 from gns3server.compute.vpcs import VPCS
@@ -41,6 +41,7 @@ async def vpcs(loop, port_manager):
 async def qemu(loop, port_manager):
 
     Qemu._instance = None
+    Qemu._init_config_disk = MagicMock()  # do not create the config.img image
     qemu = Qemu.instance()
     qemu.port_manager = port_manager
     return qemu

tests/conftest.py

@@ -117,8 +117,8 @@ def images_dir(config):
 
     path = config.get_section_config("Server").get("images_path")
     os.makedirs(path, exist_ok=True)
-    os.makedirs(os.path.join(path, "QEMU"))
+    os.makedirs(os.path.join(path, "QEMU"), exist_ok=True)
-    os.makedirs(os.path.join(path, "IOU"))
+    os.makedirs(os.path.join(path, "IOU"), exist_ok=True)
     return path
 
 

tests/handlers/api/compute/test_qemu.py

@@ -277,7 +277,8 @@ async def test_images(compute_api, fake_qemu_vm):
 
     response = await compute_api.get("/qemu/images")
     assert response.status == 200
-    assert response.json == [{"filename": "linux载.img", "path": "linux载.img", "md5sum": "c4ca4238a0b923820dcc509a6f75849b", "filesize": 1}]
+    assert response.json == [{'filename': 'config.img', 'filesize': 1048576, 'md5sum': '0ab49056760ae1db6c25376446190b47', 'path': 'config.img'},
+                             {"filename": "linux载.img", "path": "linux载.img", "md5sum": "c4ca4238a0b923820dcc509a6f75849b", "filesize": 1}]
 
 
 async def test_upload_image(compute_api, tmpdir):

tests/handlers/api/controller/test_template.py

@@ -669,13 +669,13 @@ async def test_qemu_template_create(controller_api):
                          "default_name_format": "{name}-{0}",
                          "first_port_name": "",
                          "hda_disk_image": "IOSvL2-15.2.4.0.55E.qcow2",
-                         "hda_disk_interface": "ide",
+                         "hda_disk_interface": "none",
                          "hdb_disk_image": "",
-                         "hdb_disk_interface": "ide",
+                         "hdb_disk_interface": "none",
                          "hdc_disk_image": "",
-                         "hdc_disk_interface": "ide",
+                         "hdc_disk_interface": "none",
                          "hdd_disk_image": "",
-                         "hdd_disk_interface": "ide",
+                         "hdd_disk_interface": "none",
                          "initrd": "",
                          "kernel_command_line": "",
                          "kernel_image": "",