Import/Export support for IOU nvrams.

gns3server/handlers/api/iou_handler.py

@@ -24,7 +24,7 @@ from ...schemas.iou import IOU_CREATE_SCHEMA
 from ...schemas.iou import IOU_UPDATE_SCHEMA
 from ...schemas.iou import IOU_OBJECT_SCHEMA
 from ...schemas.iou import IOU_CAPTURE_SCHEMA
-from ...schemas.iou import IOU_INITIAL_CONFIG_SCHEMA
+from ...schemas.iou import IOU_CONFIGS_SCHEMA
 from ...schemas.iou import IOU_LIST_VMS_SCHEMA
 from ...modules.iou import IOU
 
@@ -59,11 +59,15 @@ class IOUHandler:
 
         for name, value in request.json.items():
             if hasattr(vm, name) and getattr(vm, name) != value:
-                if name == "initial_config_content" and (vm.initial_config_content and len(vm.initial_config_content) > 0):
+                if name == "startup_config_content" and (vm.startup_config_content and len(vm.startup_config_content) > 0):
+                    continue
+                if name == "private_config_content" and (vm.private_config_content and len(vm.private_config_content) > 0):
                     continue
                 setattr(vm, name, value)
-        if "initial_config_content" in request.json:
+        if "startup_config_content" in request.json:
-            vm.initial_config = request.json.get("initial_config_content")
+            vm.startup_config = request.json.get("startup_config_content")
+        if "private_config_content" in request.json:
+            vm.private_config = request.json.get("private_config_content")
         response.set_status(201)
         response.json(vm)
 
@@ -111,8 +115,10 @@ class IOUHandler:
         for name, value in request.json.items():
             if hasattr(vm, name) and getattr(vm, name) != value:
                 setattr(vm, name, value)
-        if "initial_config_content" in request.json:
+        if "startup_config_content" in request.json:
-            vm.initial_config = request.json.get("initial_config_content")
+            vm.startup_config = request.json.get("startup_config_content")
+        if "private_config_content" in request.json:
+            vm.private_config = request.json.get("private_config_content")
         response.json(vm)
 
     @classmethod
@@ -301,21 +307,56 @@ class IOUHandler:
         response.set_status(204)
 
     @Route.get(
-        r"/projects/{project_id}/iou/vms/{vm_id}/initial_config",
+        r"/projects/{project_id}/iou/vms/{vm_id}/configs",
         status_codes={
-            200: "Initial config retrieved",
+            200: "Configs retrieved",
             400: "Invalid request",
             404: "Instance doesn't exist"
         },
-        output=IOU_INITIAL_CONFIG_SCHEMA,
+        output=IOU_CONFIGS_SCHEMA,
-        description="Retrieve the initial config content")
+        description="Retrieve the startup and private configs content")
-    def show_initial_config(request, response):
+    def get_configs(request, response):
 
         iou_manager = IOU.instance()
-        vm = iou_manager.get_vm(request.match_info["vm_id"],
+        vm = iou_manager.get_vm(request.match_info["vm_id"], project_id=request.match_info["project_id"])
-                                project_id=request.match_info["project_id"])
+
+        startup_config_content, private_config_content = vm.extract_configs()
+        result = {}
+        if startup_config_content:
+            result["startup_config_content"] = startup_config_content.decode("utf-8", errors='replace')
+        else:
+            # nvram doesn't exists if the VM has not been started at least once
+            # in this case just use the startup-config file
+            startup_config_content = vm.startup_config_content
+            if startup_config_content:
+                result["startup_config_content"] = startup_config_content
+
+        if private_config_content:
+            result["private_config_content"] = private_config_content.decode("utf-8", errors='replace')
+        else:
+            # nvram doesn't exists if the VM has not been started at least once
+            # in this case just use the private-config file
+            private_config_content = vm.private_config_content
+            if private_config_content:
+                result["private_config_content"] = private_config_content
+
+        response.set_status(200)
+        response.json(result)
+
+    @Route.post(
+        r"/projects/{project_id}/iou/vms/{vm_id}/configs/save",
+        status_codes={
+            200: "Configs saved",
+            400: "Invalid request",
+            404: "Instance doesn't exist"
+        },
+        description="Save the startup and private configs content")
+    def save_configs(request, response):
+
+        iou_manager = IOU.instance()
+        vm = iou_manager.get_vm(request.match_info["vm_id"], project_id=request.match_info["project_id"])
+        vm.save_configs()
         response.set_status(200)
-        response.json({"content": vm.initial_config_content})
 
     @Route.get(
         r"/iou/vms",

gns3server/modules/iou/__init__.py

@@ -68,6 +68,19 @@ class IOU(BaseManager):
         yield from super().close_vm(vm_id, *args, **kwargs)
         return vm
 
+    @asyncio.coroutine
+    def project_committed(self, project):
+        """
+        Called when a project has been committed.
+
+        :param project: Project instance
+        """
+
+        # save the configs when the project is committed
+        for vm in self._vms.copy().values():
+            if vm.project.id == project.id:
+                vm.save_configs()
+
     def get_application_id(self, vm_id):
         """
         Get an unique application identifier for IOU.

gns3server/modules/iou/iou_vm.py

@@ -33,6 +33,7 @@ import configparser
 import struct
 import hashlib
 import glob
+import binascii
 
 from .iou_error import IOUError
 from ..adapters.ethernet_adapter import EthernetAdapter
@@ -41,6 +42,8 @@ from ..nios.nio_udp import NIOUDP
 from ..nios.nio_tap import NIOTAP
 from ..nios.nio_generic_ethernet import NIOGenericEthernet
 from ..base_vm import BaseVM
+from .utils.iou_import import nvram_import
+from .utils.iou_export import nvram_export
 from .ioucon import start_ioucon
 import gns3server.utils.asyncio
 
@@ -82,7 +85,8 @@ class IOUVM(BaseVM):
         self.serial_adapters = 2  # one adapter = 4 interfaces
         self._use_default_iou_values = True  # for RAM & NVRAM values
         self._nvram = 128  # Kilobytes
-        self._initial_config = ""
+        self._startup_config = ""
+        self._private_config = ""
         self._ram = 256  # Megabytes
         self._l1_keepalives = False  # used to overcome the always-up Ethernet interfaces (not supported by all IOSes).
 
@@ -106,6 +110,7 @@ class IOUVM(BaseVM):
                         self.manager.port_manager.release_udp_port(nio.lport, self._project)
 
         yield from self.stop()
+        self.save_configs()
 
     @property
     def path(self):
@@ -208,7 +213,8 @@ class IOUVM(BaseVM):
                        "ram": self._ram,
                        "nvram": self._nvram,
                        "l1_keepalives": self._l1_keepalives,
-                       "initial_config": self.relative_initial_config_file,
+                       "startup_config": self.relative_startup_config_file,
+                       "private_config": self.relative_private_config_file,
                        "iourc_path": self.iourc_path,
                        "use_default_iou_values": self._use_default_iou_values}
 
@@ -316,10 +322,10 @@ class IOUVM(BaseVM):
         :param new_name: name
         """
 
-        if self.initial_config_file:
+        if self.startup_config_file:
-            content = self.initial_config_content
+            content = self.startup_config_content
             content = content.replace(self._name, new_name)
-            self.initial_config_content = content
+            self.startup_config_content = content
 
         super(IOUVM, IOUVM).name.__set__(self, new_name)
 
@@ -422,6 +428,36 @@ class IOUVM(BaseVM):
                                                                                                          self.iourc_path,
                                                                                                          hostname))
 
+    def _push_configs_to_nvram(self):
+        """
+        Push the startup-config and private-config content to the NVRAM.
+        """
+
+        startup_config_content = self.startup_config_content
+        if startup_config_content:
+            nvram_file = os.path.join(self.working_dir, "nvram_{:05d}".format(self.application_id))
+            try:
+                if not os.path.exists(nvram_file):
+                    open(nvram_file, "a").close()
+                with open(nvram_file, "rb") as file:
+                    nvram_content = file.read()
+            except OSError as e:
+                raise IOUError("Cannot read nvram file {}: {}".format(nvram_file, e))
+
+            startup_config_content = startup_config_content.encode("utf-8")
+            private_config_content = self.private_config_content
+            if private_config_content is not None:
+                private_config_content = private_config_content.encode("utf-8")
+            try:
+                nvram_content = nvram_import(nvram_content, startup_config_content, private_config_content, self.nvram)
+            except ValueError as e:
+                raise IOUError("Cannot push configs to nvram {}: {}".format(nvram_file, e))
+            try:
+                with open(nvram_file, "wb") as file:
+                    file.write(nvram_content)
+            except OSError as e:
+                raise IOUError("Cannot write nvram file {}: {}".format(nvram_file, e))
+
     @asyncio.coroutine
     def start(self):
         """
@@ -450,6 +486,8 @@ class IOUVM(BaseVM):
                 raise IOUError("iouyap is necessary to start IOU")
 
             self._create_netmap_config()
+            self._push_configs_to_nvram()
+
             # created a environment variable pointing to the iourc file.
             env = os.environ.copy()
 
@@ -747,9 +785,11 @@ class IOUVM(BaseVM):
             command.extend(["-m", str(self._ram)])
         command.extend(["-L"])  # disable local console, use remote console
 
-        initial_config_file = self.initial_config_file
+        # do not let IOU create the NVRAM anymore
-        if initial_config_file:
+        #startup_config_file = self.startup_config_file
-            command.extend(["-c", os.path.basename(initial_config_file)])
+        #if startup_config_file:
+        #    command.extend(["-c", os.path.basename(startup_config_file)])
+
         if self._l1_keepalives:
             yield from self._enable_l1_keepalives(command)
         command.extend([str(self.application_id)])
@@ -962,12 +1002,12 @@ class IOUVM(BaseVM):
             log.warn("could not determine if layer 1 keepalive messages are supported by {}: {}".format(os.path.basename(self._path), e))
 
     @property
-    def initial_config_content(self):
+    def startup_config_content(self):
         """
-        Returns the content of the current initial-config file.
+        Returns the content of the current startup-config file.
         """
 
-        config_file = self.initial_config_file
+        config_file = self.startup_config_file
         if config_file is None:
             return None
 
@@ -975,64 +1015,190 @@ class IOUVM(BaseVM):
             with open(config_file, "rb") as f:
                 return f.read().decode("utf-8", errors="replace")
         except OSError as e:
-            raise IOUError("Can't read configuration file '{}': {}".format(config_file, e))
+            raise IOUError("Can't read startup-config file '{}': {}".format(config_file, e))
 
-    @initial_config_content.setter
+    @startup_config_content.setter
-    def initial_config_content(self, initial_config):
+    def startup_config_content(self, startup_config):
         """
-        Update the initial config
+        Update the startup config
 
-        :param initial_config: content of the initial configuration file
+        :param startup_config: content of the startup configuration file
         """
 
         try:
-            initial_config_path = os.path.join(self.working_dir, "initial-config.cfg")
+            startup_config_path = os.path.join(self.working_dir, "startup-config.cfg")
 
-            if initial_config is None:
+            if startup_config is None:
-                initial_config = ''
+                startup_config = ''
 
-            # We disallow erasing the initial config file
+            # We disallow erasing the startup config file
-            if len(initial_config) == 0 and os.path.exists(initial_config_path):
+            if len(startup_config) == 0 and os.path.exists(startup_config_path):
                 return
 
-            with open(initial_config_path, 'w+', encoding='utf-8') as f:
+            with open(startup_config_path, 'w+', encoding='utf-8') as f:
-                if len(initial_config) == 0:
+                if len(startup_config) == 0:
                     f.write('')
                 else:
-                    initial_config = initial_config.replace("%h", self._name)
+                    startup_config = startup_config.replace("%h", self._name)
-                    f.write(initial_config)
+                    f.write(startup_config)
         except OSError as e:
-            raise IOUError("Can't write initial configuration file '{}': {}".format(initial_config_path, e))
+            raise IOUError("Can't write startup-config file '{}': {}".format(startup_config_path, e))
 
     @property
-    def initial_config_file(self):
+    def private_config_content(self):
         """
-        Returns the initial config file for this IOU VM.
+        Returns the content of the current private-config file.
+        """
+
+        config_file = self.private_config_file
+        if config_file is None:
+            return None
+
+        try:
+            with open(config_file, "rb") as f:
+                return f.read().decode("utf-8", errors="replace")
+        except OSError as e:
+            raise IOUError("Can't read private-config file '{}': {}".format(config_file, e))
+
+    @private_config_content.setter
+    def private_config_content(self, private_config):
+        """
+        Update the private config
+
+        :param private_config: content of the private configuration file
+        """
+
+        try:
+            private_config_path = os.path.join(self.working_dir, "private-config.cfg")
+
+            if private_config is None:
+                private_config = ''
+
+            # We disallow erasing the startup config file
+            if len(private_config) == 0 and os.path.exists(private_config_path):
+                return
+
+            with open(private_config_path, 'w+', encoding='utf-8') as f:
+                if len(private_config) == 0:
+                    f.write('')
+                else:
+                    private_config = private_config.replace("%h", self._name)
+                    f.write(private_config)
+        except OSError as e:
+            raise IOUError("Can't write private-config file '{}': {}".format(private_config_path, e))
+
+    @property
+    def startup_config_file(self):
+        """
+        Returns the startup-config file for this IOU VM.
 
         :returns: path to config file. None if the file doesn't exist
         """
 
-        path = os.path.join(self.working_dir, 'initial-config.cfg')
+        path = os.path.join(self.working_dir, 'startup-config.cfg')
         if os.path.exists(path):
             return path
         else:
             return None
 
     @property
-    def relative_initial_config_file(self):
+    def private_config_file(self):
         """
-        Returns the initial config file relative to the project directory.
+        Returns the private-config file for this IOU VM.
-        It's compatible with pre 1.3 projects.
 
         :returns: path to config file. None if the file doesn't exist
         """
 
-        path = os.path.join(self.working_dir, 'initial-config.cfg')
+        path = os.path.join(self.working_dir, 'private-config.cfg')
         if os.path.exists(path):
-            return 'initial-config.cfg'
+            return path
         else:
             return None
 
+    @property
+    def relative_startup_config_file(self):
+        """
+        Returns the startup-config file relative to the project directory.
+        It's compatible with pre 1.3 projects.
+
+        :returns: path to startup-config file. None if the file doesn't exist
+        """
+
+        path = os.path.join(self.working_dir, 'startup-config.cfg')
+        if os.path.exists(path):
+            return 'startup-config.cfg'
+        else:
+            return None
+
+    @property
+    def relative_private_config_file(self):
+        """
+        Returns the private-config file relative to the project directory.
+
+        :returns: path to private-config file. None if the file doesn't exist
+        """
+
+        path = os.path.join(self.working_dir, 'private-config.cfg')
+        if os.path.exists(path):
+            return 'private-config.cfg'
+        else:
+            return None
+
+    def extract_configs(self):
+        """
+        Gets the contents of the config files
+        startup-config and private-config from NVRAM.
+
+        :returns: tuple (startup-config, private-config)
+        """
+
+        nvram_file = os.path.join(self.working_dir, "nvram_{:05d}".format(self.application_id))
+        if not os.path.exists(nvram_file):
+            return None, None
+        try:
+            with open(nvram_file, "rb") as file:
+                nvram_content = file.read()
+        except OSError as e:
+            log.warning("Cannot read nvram file {}: {}".format(nvram_file, e))
+            return None, None
+
+        try:
+            startup_config_content, private_config_content = nvram_export(nvram_content)
+        except ValueError as e:
+            log.warning("Could not export configs from nvram file".format(nvram_file, e))
+            return None, None
+
+        return startup_config_content, private_config_content
+
+    def save_configs(self):
+        """
+        Saves the startup-config and private-config to files.
+        """
+
+        if self.startup_config_content or self.private_config_content:
+            startup_config_content, private_config_content = self.extract_configs()
+            if startup_config_content:
+                config_path = os.path.join(self.working_dir, "startup-config.cfg")
+                try:
+                    config = startup_config_content.decode("utf-8", errors="replace")
+                    config = "!\n" + config.replace("\r", "")
+                    with open(config_path, "wb") as f:
+                        log.info("saving startup-config to {}".format(config_path))
+                        f.write(config.encode("utf-8"))
+                except (binascii.Error, OSError) as e:
+                    raise IOUError("Could not save the startup configuration {}: {}".format(config_path, e))
+
+            if private_config_content:
+                config_path = os.path.join(self.working_dir, "private-config.cfg")
+                try:
+                    config = private_config_content.decode("utf-8", errors="replace")
+                    config = "!\n" + config.replace("\r", "")
+                    with open(config_path, "wb") as f:
+                        log.info("saving private-config to {}".format(config_path))
+                        f.write(config.encode("utf-8"))
+                except (binascii.Error, OSError) as e:
+                    raise IOUError("Could not save the private configuration {}: {}".format(config_path, e))
+
     @asyncio.coroutine
     def start_capture(self, adapter_number, port_number, output_file, data_link_type="DLT_EN10MB"):
         """

gns3server/modules/iou/utils/iou_export.py

@@ -0,0 +1,239 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# To use python v2.7 change the first line to:
+#!/usr/bin/env python
+
+# Copyright (C) 2015 Bernhard Ehlers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# This utility is a stripped down version of dynamips' nvram_export,
+# ported from C to Python, see https://github.com/GNS3/dynamips
+# nvram_export is (c) 2013 Flávio J. Saraiva
+
+"""
+iou_export exports startup/private configuration from IOU NVRAM file.
+
+usage: iou_export [-h] NVRAM startup-config [private-config]
+
+positional arguments:
+  NVRAM           NVRAM file
+  startup-config  startup configuration
+  private-config  private configuration
+
+optional arguments:
+  -h, --help      show this help message and exit
+"""
+
+import argparse
+import sys
+from array import array
+
+
+# Uncompress data in .Z file format.
+# Ported from dynamips' fs_nvram.c to python
+# Adapted from 7zip's ZDecoder.cpp, which is licensed under LGPL 2.1.
+def uncompress_LZC(data):
+    LZC_NUM_BITS_MIN = 9
+    LZC_NUM_BITS_MAX = 16
+
+    in_data  = array('B', data)
+    in_len   = len(in_data)
+    out_data = array('B')
+
+    if in_len == 0:
+        return out_data.tostring()
+    if in_len < 3:
+        raise ValueError('invalid length')
+    if in_data[0] != 0x1F or in_data[1] != 0x9D:
+        raise ValueError('invalid header')
+
+    maxbits   = in_data[2] & 0x1F
+    numItems  = 1 << maxbits
+    blockMode = (in_data[2] & 0x80) != 0
+    if maxbits < LZC_NUM_BITS_MIN or maxbits > LZC_NUM_BITS_MAX:
+        raise ValueError('not supported')
+
+    parents  = array('H', [0] * numItems)
+    suffixes = array('B', [0] * numItems)
+    stack    = array('B', [0] * numItems)
+
+    in_pos  = 3
+    numBits = LZC_NUM_BITS_MIN
+    head    = 256
+    if blockMode:
+        head += 1
+
+    needPrev   = 0
+    bitPos     = 0
+    numBufBits = 0
+
+    parents[256]  = 0
+    suffixes[256] = 0
+
+    buf_extend = array('B', [0] * 3)
+
+    while True:
+        # fill buffer, when empty
+        if numBufBits == bitPos:
+            buf_len = min(in_len - in_pos, numBits)
+            buf = in_data[in_pos:in_pos+buf_len] + buf_extend
+            numBufBits = buf_len << 3
+            bitPos = 0
+            in_pos += buf_len
+
+        # extract next symbol
+        bytePos = bitPos >> 3
+        symbol = buf[bytePos] | buf[bytePos + 1] << 8 | buf[bytePos + 2] << 16
+        symbol >>= bitPos & 7
+        symbol &= (1 << numBits) - 1
+        bitPos += numBits
+
+        # check for special conditions: end, bad data, re-initialize dictionary
+        if bitPos > numBufBits:
+            break
+        if symbol >= head:
+            raise ValueError('invalid data')
+        if blockMode and symbol == 256:
+            numBufBits = bitPos = 0
+            numBits = LZC_NUM_BITS_MIN
+            head = 257
+            needPrev = 0
+            continue
+
+        # convert symbol to string
+        cur = symbol
+        i = len(stack)
+        while cur >= 256:
+            i -= 1
+            stack[i] = suffixes[cur]
+            cur = parents[cur]
+        i -= 1
+        stack[i] = cur
+        if needPrev:
+            suffixes[head - 1] = cur
+            if symbol == head - 1:
+                stack[-1] = cur
+        out_data.extend(stack[i:])
+
+        # update parents, check for numBits change
+        if head < numItems:
+            needPrev = 1
+            parents[head] = symbol
+            head += 1
+            if head > (1 << numBits):
+                if numBits < maxbits:
+                    numBufBits = bitPos = 0
+                    numBits += 1
+        else:
+            needPrev = 0
+
+    return out_data.tostring()
+
+
+# extract 16 bit unsigned int from data
+def get_uint16(data, off):
+    return data[off] << 8 | data[off+1]
+
+
+# extract 32 bit unsigned int from data
+def get_uint32(data, off):
+    return data[off] << 24 | data[off+1] << 16 | data[off+2] << 8 | data[off+3]
+
+
+# export IOU NVRAM
+def nvram_export(nvram):
+    nvram = array('B', nvram)
+
+    # extract startup config
+    offset = 0
+    if len(nvram) < offset + 36:
+        raise ValueError('invalid length')
+    if get_uint16(nvram, offset + 0) != 0xABCD:
+        raise ValueError('no startup config')
+    format = get_uint16(nvram, offset + 2)
+    length = get_uint32(nvram, offset + 16)
+    offset += 36
+    if len(nvram) < offset + length:
+        raise ValueError('invalid length')
+    startup = nvram[offset:offset+length].tostring()
+
+    # compressed startup config
+    if format == 2:
+        try:
+            startup = uncompress_LZC(startup)
+        except ValueError as err:
+            raise ValueError('uncompress startup: ' + str(err))
+
+    offset += length
+    # alignment to multiple of 4
+    offset = (offset+3) & ~3
+    # check for additonal offset of 4
+    if len(nvram) >= offset + 8 and \
+       get_uint16(nvram, offset + 4) == 0xFEDC and \
+       get_uint16(nvram, offset + 6) == 1:
+        offset += 4
+
+    # extract private config
+    private = None
+    if len(nvram) >= offset + 16 and get_uint16(nvram, offset + 0) == 0xFEDC:
+        length = get_uint32(nvram, offset + 12)
+        offset += 16
+        if len(nvram) >= offset + length:
+            private = nvram[offset:offset+length].tostring()
+
+    return (startup, private)
+
+
+if __name__ == '__main__':
+    # Main program
+
+    parser = argparse.ArgumentParser(description='%(prog)s exports startup/private configuration from IOU NVRAM file.')
+    parser.add_argument('nvram', metavar='NVRAM',
+                        help='NVRAM file')
+    parser.add_argument('startup', metavar='startup-config',
+                        help='startup configuration')
+    parser.add_argument('private', metavar='private-config', nargs='?',
+                        help='private configuration')
+    args = parser.parse_args()
+
+    try:
+        fd = open(args.nvram, 'rb')
+        nvram = fd.read()
+        fd.close()
+    except (IOError, OSError) as err:
+        sys.stderr.write("Error reading file: {}\n".format(err))
+        sys.exit(1)
+
+    try:
+        startup, private = nvram_export(nvram)
+    except ValueError as err:
+        sys.stderr.write("nvram_export: {}\n".format(err))
+        sys.exit(3)
+
+    try:
+        fd = open(args.startup, 'wb')
+        fd.write(startup)
+        fd.close()
+        if args.private is not None:
+            if private is None:
+                sys.stderr.write("Warning: No private config\n")
+            else:
+                fd = open(args.private, 'wb')
+                fd.write(private)
+                fd.close()
+    except (IOError, OSError) as err:
+        sys.stderr.write("Error writing file: {}\n".format(err))
+        sys.exit(1)

gns3server/modules/iou/utils/iou_import.py

@@ -0,0 +1,241 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# To use python v2.7 change the first line to:
+#!/usr/bin/env python
+
+# Copyright (C) 2015 Bernhard Ehlers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+iou_import imports startup/private configuration into IOU NVRAM file.
+
+usage: iou_import [-h] [-c size] NVRAM startup-config [private-config]
+
+positional arguments:
+  NVRAM                 NVRAM file
+  startup-config        startup configuration
+  private-config        private configuration
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -c size, --create size
+                        create NVRAM file, size in kByte
+"""
+
+import argparse
+import sys
+from array import array
+
+
+# extract 16 bit unsigned int from data
+def get_uint16(data, off):
+    return data[off] << 8 | data[off+1]
+
+
+# extract 32 bit unsigned int from data
+def get_uint32(data, off):
+    return data[off] << 24 | data[off+1] << 16 | data[off+2] << 8 | data[off+3]
+
+
+# insert 16 bit unsigned int into data
+def put_uint16(data, off, value):
+    data[off]   = (value >> 8) & 0xff
+    data[off+1] = value & 0xff
+
+
+# insert 32 bit unsigned int into data
+def put_uint32(data, off, value):
+    data[off]   = (value >> 24) & 0xff
+    data[off+1] = (value >> 16) & 0xff
+    data[off+2] = (value >> 8) & 0xff
+    data[off+3] = value & 0xff
+
+
+# calculate padding
+def padding(off, ios):
+    pad = (4 - off % 4) % 4             # padding to alignment of 4
+    if ios <= 0x0F00 and pad != 0:      # add 4 if IOS <= 15.0
+        pad += 4
+    return pad
+
+
+# update checksum
+def checksum(data, start, end):
+    put_uint16(data, start + 4, 0)      # set checksum to 0
+
+    chk = 0
+    idx = start
+    while idx < end-1:
+        chk += get_uint16(data, idx)
+        idx += 2
+    if idx < end:
+        chk += data[idx] << 8
+
+    while chk >> 16:
+        chk = (chk & 0xffff) + (chk >> 16)
+
+    chk = chk ^ 0xffff
+    put_uint16(data, start + 4, chk)    # set checksum
+
+
+# import IOU NVRAM
+def nvram_import(nvram, startup, private, size):
+    BASE_ADDRESS = 0x10000000
+    DEFAULT_IOS  = 0x0F04               # IOS 15.4
+
+    if size is None:
+        nvram = array('B', nvram)
+    else:
+        nvram = array('B', [0] * (size*1024))
+
+    # check nvram size
+    nvram_len = len(nvram)
+    if nvram_len < 8*1024 or nvram_len > 1024*1024 or nvram_len % 1024 != 0:
+        raise ValueError('invalid length')
+    nvram_len = nvram_len // 2
+
+    # get size of current config
+    config_len = 0
+    ios = None
+    try:
+        if get_uint16(nvram, 0) == 0xABCD:
+            ios = get_uint16(nvram, 6)
+            config_len = 36 + get_uint32(nvram, 16)
+            config_len += padding(config_len, ios)
+            if get_uint16(nvram, config_len) == 0xFEDC:
+                config_len += 16 + get_uint32(nvram, config_len + 12)
+    except IndexError:
+        raise ValueError('unknown nvram format')
+    if config_len > nvram_len:
+        raise ValueError('unknown nvram format')
+
+    # calculate max. config size
+    max_config = nvram_len - 2*1024             # reserve 2k for files
+    idx = max_config
+    empty_sector = array('B', [0] * 1024)
+    while True:
+        idx -= 1024
+        if idx < config_len:
+            break
+        # if valid file header:
+        if get_uint16(nvram, idx+0) == 0xDCBA and \
+           get_uint16(nvram, idx+4) < 8 and \
+           get_uint16(nvram, idx+6) <= 992:
+            max_config = idx
+        elif nvram[idx:idx+1024] != empty_sector:
+            break
+
+    # import startup config
+    startup = array('B', startup)
+    if ios is None:
+        # Target IOS version is unknown. As some IOU don't work nicely with
+        # the padding of a different version, the startup config is padded
+        # with '\n' to the alignment of 4.
+        ios = DEFAULT_IOS
+        startup.extend([ord('\n')] * ((4 - len(startup) % 4) % 4))
+    new_nvram = array('B', [0] * 36)                            # startup hdr
+    put_uint16(new_nvram,  0, 0xABCD)                           # magic
+    put_uint16(new_nvram,  2, 1)                                # raw data
+    put_uint16(new_nvram,  6, ios)                              # IOS version
+    put_uint32(new_nvram,  8, BASE_ADDRESS+36)                  # start address
+    put_uint32(new_nvram, 12, BASE_ADDRESS+36 + len(startup))   # end address
+    put_uint32(new_nvram, 16, len(startup))                     # length
+    new_nvram.extend(startup)
+    new_nvram.extend([0] * padding(len(new_nvram), ios))
+
+    # import private config
+    if private is None:
+        private = array('B')
+    else:
+        private = array('B', private)
+    offset = len(new_nvram)
+    new_nvram.extend([0] * 16)                                  # private hdr
+    put_uint16(new_nvram,  0 + offset, 0xFEDC)                  # magic
+    put_uint16(new_nvram,  2 + offset, 1)                       # raw data
+    put_uint32(new_nvram,  4 + offset,
+               BASE_ADDRESS + offset + 16)                      # start address
+    put_uint32(new_nvram,  8 + offset,
+               BASE_ADDRESS + offset + 16 + len(private))       # end address
+    put_uint32(new_nvram, 12 + offset, len(private))            # length
+    new_nvram.extend(private)
+
+    # add rest
+    if len(new_nvram) > max_config:
+        raise ValueError('NVRAM size too small')
+    new_nvram.extend([0] * (max_config - len(new_nvram)))
+    new_nvram.extend(nvram[max_config:])
+
+    checksum(new_nvram, 0, nvram_len)
+
+    return new_nvram.tostring()
+
+
+if __name__ == '__main__':
+    # Main program
+
+    def check_size(string):
+        try:
+            value = int(string)
+        except ValueError:
+            raise argparse.ArgumentTypeError('invalid int value: ' + string)
+        if value < 8 or value > 1024:
+            raise argparse.ArgumentTypeError('size must be 8..1024')
+        return value
+
+    parser = argparse.ArgumentParser(description='%(prog)s imports startup/private configuration into IOU NVRAM file.')
+    parser.add_argument('-c', '--create', metavar='size', type=check_size,
+                        help='create NVRAM file, size in kByte')
+    parser.add_argument('nvram', metavar='NVRAM',
+                        help='NVRAM file')
+    parser.add_argument('startup', metavar='startup-config',
+                        help='startup configuration')
+    parser.add_argument('private', metavar='private-config', nargs='?',
+                        help='private configuration')
+    args = parser.parse_args()
+
+    try:
+        if args.create is None:
+            fd = open(args.nvram, 'rb')
+            nvram = fd.read()
+            fd.close()
+        else:
+            nvram = None
+        fd = open(args.startup, 'rb')
+        startup = fd.read()
+        fd.close()
+        if args.private is None:
+            private = None
+        else:
+            fd = open(args.private, 'rb')
+            private = fd.read()
+            fd.close()
+    except (IOError, OSError) as err:
+        sys.stderr.write("Error reading file: {}\n".format(err))
+        sys.exit(1)
+
+    try:
+        nvram = nvram_import(nvram, startup, private, args.create)
+    except ValueError as err:
+        sys.stderr.write("nvram_import: {}\n".format(err))
+        sys.exit(3)
+
+    try:
+        fd = open(args.nvram, 'wb')
+        fd.write(nvram)
+        fd.close()
+    except (IOError, OSError) as err:
+        sys.stderr.write("Error writing file: {}\n".format(err))
+        sys.exit(1)

gns3server/schemas/iou.py

@@ -70,12 +70,20 @@ IOU_CREATE_SCHEMA = {
             "description": "Use default IOU values",
             "type": ["boolean", "null"]
         },
-        "initial_config": {
+        "startup_config": {
-            "description": "Path to the initial configuration of IOU",
+            "description": "Path to the startup-config of IOU",
             "type": ["string", "null"]
         },
-        "initial_config_content": {
+        "private_config": {
-            "description": "Initial configuration of IOU",
+            "description": "Path to the private-config of IOU",
+            "type": ["string", "null"]
+        },
+        "startup_config_content": {
+            "description": "Startup-config of IOU",
+            "type": ["string", "null"]
+        },
+        "private_config_content": {
+            "description": "Private-config of IOU",
             "type": ["string", "null"]
         },
         "iourc_content": {
@@ -127,8 +135,12 @@ IOU_UPDATE_SCHEMA = {
             "description": "Always up ethernet interface",
             "type": ["boolean", "null"]
         },
-        "initial_config_content": {
+        "startup_config_content": {
-            "description": "Initial configuration of IOU",
+            "description": "Startup-config of IOU",
+            "type": ["string", "null"]
+        },
+        "private_config_content": {
+            "description": "Private-config of IOU",
             "type": ["string", "null"]
         },
         "use_default_iou_values": {
@@ -197,8 +209,12 @@ IOU_OBJECT_SCHEMA = {
             "description": "Always up ethernet interface",
             "type": "boolean"
         },
-        "initial_config": {
+        "startup_config": {
-            "description": "Path of the initial config content relative to project directory",
+            "description": "Path of the startup-config content relative to project directory",
+            "type": ["string", "null"]
+        },
+        "private_config": {
+            "description": "Path of the private-config content relative to project directory",
             "type": ["string", "null"]
         },
         "use_default_iou_values": {
@@ -211,7 +227,8 @@ IOU_OBJECT_SCHEMA = {
         }
     },
     "additionalProperties": False,
-    "required": ["name", "vm_id", "console", "project_id", "path", "serial_adapters", "ethernet_adapters", "ram", "nvram", "l1_keepalives", "initial_config", "use_default_iou_values"]
+    "required": ["name", "vm_id", "console", "project_id", "path", "serial_adapters", "ethernet_adapters",
+                 "ram", "nvram", "l1_keepalives", "startup_config", "private_config", "use_default_iou_values"]
 }
 
 IOU_CAPTURE_SCHEMA = {
@@ -234,18 +251,23 @@ IOU_CAPTURE_SCHEMA = {
     "required": ["capture_file_name", "data_link_type"]
 }
 
-IOU_INITIAL_CONFIG_SCHEMA = {
+IOU_CONFIGS_SCHEMA = {
     "$schema": "http://json-schema.org/draft-04/schema#",
-    "description": "Request validation to get the initial configuration file",
+    "description": "Request validation to get the startup and private configuration file",
     "type": "object",
     "properties": {
-        "content": {
+        "startup_config_content": {
-            "description": "Content of the initial configuration file",
+            "description": "Content of the startup configuration file",
-            "type": ["string", "null"]
+            "type": ["string", "null"],
+            "minLength": 1,
+        },
+        "private_config_content": {
+            "description": "Content of the private configuration file",
+            "type": ["string", "null"],
+            "minLength": 1,
         },
     },
     "additionalProperties": False,
-    "required": ["content"]
 }
 
 IOU_LIST_VMS_SCHEMA = {

tests/handlers/api/test_iou.py

@@ -52,10 +52,10 @@ def vm(server, project, base_params):
     return response.json
 
 
-def initial_config_file(project, vm):
+def startup_config_file(project, vm):
     directory = os.path.join(project.path, "project-files", "iou", vm["vm_id"])
     os.makedirs(directory, exist_ok=True)
-    return os.path.join(directory, "initial-config.cfg")
+    return os.path.join(directory, "startup-config.cfg")
 
 
 def test_iou_create(server, project, base_params):
@@ -78,7 +78,7 @@ def test_iou_create_with_params(server, project, base_params):
     params["serial_adapters"] = 4
     params["ethernet_adapters"] = 0
     params["l1_keepalives"] = True
-    params["initial_config_content"] = "hostname test"
+    params["startup_config_content"] = "hostname test"
     params["use_default_iou_values"] = True
     params["iourc_content"] = "test"
 
@@ -94,31 +94,31 @@ def test_iou_create_with_params(server, project, base_params):
     assert response.json["l1_keepalives"] is True
     assert response.json["use_default_iou_values"] is True
 
-    assert "initial-config.cfg" in response.json["initial_config"]
+    assert "startup-config.cfg" in response.json["startup_config"]
-    with open(initial_config_file(project, response.json)) as f:
+    with open(startup_config_file(project, response.json)) as f:
         assert f.read() == "hostname test"
 
     assert "iourc" in response.json["iourc_path"]
 
 
-def test_iou_create_initial_config_already_exist(server, project, base_params):
+def test_iou_create_startup_config_already_exist(server, project, base_params):
-    """We don't erase an initial config if already exist at project creation"""
+    """We don't erase a startup-config if already exist at project creation"""
 
     vm_id = str(uuid.uuid4())
-    initial_config_file_path = initial_config_file(project, {'vm_id': vm_id})
+    startup_config_file_path = startup_config_file(project, {'vm_id': vm_id})
-    with open(initial_config_file_path, 'w+') as f:
+    with open(startup_config_file_path, 'w+') as f:
         f.write("echo hello")
 
     params = base_params
     params["vm_id"] = vm_id
-    params["initial_config_content"] = "hostname test"
+    params["startup_config_content"] = "hostname test"
 
     response = server.post("/projects/{project_id}/iou/vms".format(project_id=project.id), params, example=True)
     assert response.status == 201
     assert response.route == "/projects/{project_id}/iou/vms"
 
-    assert "initial-config.cfg" in response.json["initial_config"]
+    assert "startup-config.cfg" in response.json["startup_config"]
-    with open(initial_config_file(project, response.json)) as f:
+    with open(startup_config_file(project, response.json)) as f:
         assert f.read() == "echo hello"
 
 
@@ -172,7 +172,7 @@ def test_iou_update(server, vm, tmpdir, free_console_port, project):
         "ethernet_adapters": 4,
         "serial_adapters": 0,
         "l1_keepalives": True,
-        "initial_config_content": "hostname test",
+        "startup_config_content": "hostname test",
         "use_default_iou_values": True,
         "iourc_content": "test"
     }
@@ -186,8 +186,8 @@ def test_iou_update(server, vm, tmpdir, free_console_port, project):
     assert response.json["nvram"] == 2048
     assert response.json["l1_keepalives"] is True
     assert response.json["use_default_iou_values"] is True
-    assert "initial-config.cfg" in response.json["initial_config"]
+    assert "startup-config.cfg" in response.json["startup_config"]
-    with open(initial_config_file(project, response.json)) as f:
+    with open(startup_config_file(project, response.json)) as f:
         assert f.read() == "hostname test"
 
     assert "iourc" in response.json["iourc_path"]
@@ -294,22 +294,22 @@ def test_iou_stop_capture_not_started(server, vm, tmpdir):
             assert response.status == 409
 
 
-def test_get_initial_config_without_config_file(server, vm):
+def test_get_configs_without_configs_file(server, vm):
 
-    response = server.get("/projects/{project_id}/iou/vms/{vm_id}/initial_config".format(project_id=vm["project_id"], vm_id=vm["vm_id"]), example=True)
+    response = server.get("/projects/{project_id}/iou/vms/{vm_id}/configs".format(project_id=vm["project_id"], vm_id=vm["vm_id"]), example=True)
     assert response.status == 200
-    assert response.json["content"] == None
+    assert "startup_config" not in response.json
+    assert "private_config" not in response.json
 
+def test_get_configs_with_startup_config_file(server, project, vm):
 
-def test_get_initial_config_with_config_file(server, project, vm):
+    path = startup_config_file(project, vm)
-
-    path = initial_config_file(project, vm)
     with open(path, "w+") as f:
         f.write("TEST")
 
-    response = server.get("/projects/{project_id}/iou/vms/{vm_id}/initial_config".format(project_id=vm["project_id"], vm_id=vm["vm_id"]), example=True)
+    response = server.get("/projects/{project_id}/iou/vms/{vm_id}/configs".format(project_id=vm["project_id"], vm_id=vm["vm_id"]), example=True)
     assert response.status == 200
-    assert response.json["content"] == "TEST"
+    assert response.json["startup_config_content"] == "TEST"
 
 
 def test_vms(server, tmpdir, fake_iou_bin):

tests/modules/iou/test_iou_vm.py

@@ -87,11 +87,11 @@ def test_vm(project, manager):
     assert vm.id == "00010203-0405-0607-0809-0a0b0c0d0e0f"
 
 
-def test_vm_initial_config_content(project, manager):
+def test_vm_startup_config_content(project, manager):
     vm = IOUVM("test", "00010203-0405-0607-0808-0a0b0c0d0e0f", project, manager)
-    vm.initial_config_content = "hostname %h"
+    vm.startup_config_content = "hostname %h"
     assert vm.name == "test"
-    assert vm.initial_config_content == "hostname test"
+    assert vm.startup_config_content == "hostname test"
     assert vm.id == "00010203-0405-0607-0808-0a0b0c0d0e0f"
 
 
@@ -255,54 +255,44 @@ def test_build_command(vm, loop):
 
     assert loop.run_until_complete(asyncio.async(vm._build_command())) == [vm.path, "-L", str(vm.application_id)]
 
-
+def test_get_startup_config(vm):
-def test_build_command_initial_config(vm, loop):
-
-    filepath = os.path.join(vm.working_dir, "initial-config.cfg")
-    with open(filepath, "w+") as f:
-        f.write("service timestamps debug datetime msec\nservice timestamps log datetime msec\nno service password-encryption")
-
-    assert loop.run_until_complete(asyncio.async(vm._build_command())) == [vm.path, "-L", "-c", os.path.basename(vm.initial_config_file), str(vm.application_id)]
-
-
-def test_get_initial_config(vm):
 
     content = "service timestamps debug datetime msec\nservice timestamps log datetime msec\nno service password-encryption"
-    vm.initial_config = content
+    vm.startup_config = content
-    assert vm.initial_config == content
+    assert vm.startup_config == content
 
 
-def test_update_initial_config(vm):
+def test_update_startup_config(vm):
     content = "service timestamps debug datetime msec\nservice timestamps log datetime msec\nno service password-encryption"
-    vm.initial_config = content
+    vm.startup_config_content = content
-    filepath = os.path.join(vm.working_dir, "initial-config.cfg")
+    filepath = os.path.join(vm.working_dir, "startup-config.cfg")
     assert os.path.exists(filepath)
     with open(filepath) as f:
         assert f.read() == content
 
 
-def test_update_initial_config_empty(vm):
+def test_update_startup_config_empty(vm):
     content = "service timestamps debug datetime msec\nservice timestamps log datetime msec\nno service password-encryption"
-    vm.initial_config = content
+    vm.startup_config_content = content
-    filepath = os.path.join(vm.working_dir, "initial-config.cfg")
+    filepath = os.path.join(vm.working_dir, "startup-config.cfg")
     assert os.path.exists(filepath)
     with open(filepath) as f:
         assert f.read() == content
-    vm.initial_config = ""
+    vm.startup_config_content = ""
     with open(filepath) as f:
         assert f.read() == content
 
 
-def test_update_initial_config_content_hostname(vm):
+def test_update_startup_config_content_hostname(vm):
     content = "hostname %h\n"
     vm.name = "pc1"
-    vm.initial_config_content = content
+    vm.startup_config_content = content
-    with open(vm.initial_config_file) as f:
+    with open(vm.startup_config_file) as f:
         assert f.read() == "hostname pc1\n"
 
 
 def test_change_name(vm, tmpdir):
-    path = os.path.join(vm.working_dir, "initial-config.cfg")
+    path = os.path.join(vm.working_dir, "startup-config.cfg")
     vm.name = "world"
     with open(path, 'w+') as f:
         f.write("hostname world")

tests/modules/qemu/test_qemu_vm.py

@@ -41,8 +41,7 @@ def manager(port_manager):
 @pytest.fixture
 def fake_qemu_img_binary():
 
-    # Should not crash with unicode characters
+    bin_path = os.path.join(os.environ["PATH"], "qemu-img")
-    bin_path = os.path.join(os.environ["PATH"], "qemu-img\u62FF")
     with open(bin_path, "w+") as f:
         f.write("1")
     os.chmod(bin_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)

utils/__init__.py

@@ -1 +0,0 @@
-__author__ = 'grossmj'