Merge branch 'master' into 2.0

conf/gns3_server.conf

@@ -0,0 +1,61 @@
+[Server]
+; IP where the server listen for connections
+host = 0.0.0.0
+; HTTP port for controlling the servers
+port = 3080
+
+; Option to enable SSL encryption
+ssl = False
+certfile=/home/gns3/.config/GNS3/ssl/server.cert
+certkey=/home/gns3/.config/GNS3/ssl/server.key
+
+; Path where devices images are stored
+images_path = /home/gns3/GNS3/images
+; Path where user projects are stored
+projects_path = /home/gns3/GNS3/projects
+
+; Option to automatically send crash reports to the GNS3 team
+report_errors = True
+
+; First console port of the range allocated to devices
+console_start_port_range = 5000
+; Last console port of the range allocated to devices
+console_end_port_range = 10000
+; First port of the range allocated for inter-device communication. Two ports are allocated per link.
+udp_start_port_range = 10000
+; Last port of the range allocated for inter-device communication. Two ports are allocated per link
+udp_start_end_range = 20000
+; uBridge executable location, default: search in PATH
+;ubridge_path = ubridge
+
+; Option to enable HTTP authentication.
+auth = False
+; Username for HTTP authentication.
+user = gns3
+; Password for HTTP authentication.
+password = gns3
+
+[VPCS]
+; VPCS executable location, default: search in PATH
+;vpcs_path = vpcs
+
+[Dynamips]
+; Enable auxiliary console ports on IOS routers
+allocate_aux_console_ports = False
+mmap_support = True
+; Dynamips executable path, default: search in PATH
+;dynamips_path = dynamips
+sparse_memory_support = True
+ghost_ios_support = True
+
+[IOU]
+; iouyap executable path, default: search in PATH
+;iouyap_path = iouyap
+; Path of your .iourc file. If not provided, the file is searched in $HOME/.iourc
+iourc_path = /home/gns3/.iourc
+; Validate if the iourc license file is correct. If you turn this off and your licence is invalid IOU will not start and no errors will be shown.
+license_check = True
+
+[Qemu]
+; !! Remember to add the gns3 user to the KVM group, otherwise you will not have read / write permssions to /dev/kvm !!
+enable_kvm = True

gns3server/compute/base_node.py

@@ -452,11 +452,7 @@ class BaseNode:
         """
 
         path = self._manager.config.get_section_config("Server").get("ubridge_path", "ubridge")
-        if path == "ubridge" or path == "None":
-            path = shutil.which("ubridge")
-
-        if path is None or len(path) == 0:
-            raise NodeError("uBridge is not installed")
+        path = shutil.which(path)
         return path
 
     @asyncio.coroutine
@@ -481,7 +477,7 @@ class BaseNode:
         """
 
         if self.ubridge_path is None:
-            raise NodeError("uBridge is not available")
+            raise NodeError("uBridge is not available or path doesn't exist")
 
         if not self._manager.has_privileged_access(self.ubridge_path):
             raise NodeError("uBridge requires root access or capability to interact with network adapters")

gns3server/compute/dynamips/__init__.py

@@ -303,16 +303,16 @@ class Dynamips(BaseManager):
     def find_dynamips(self):
 
         # look for Dynamips
-        dynamips_path = self.config.get_section_config("Dynamips").get("dynamips_path")
-        if not dynamips_path:
-            dynamips_path = shutil.which("dynamips")
+        dynamips_path = self.config.get_section_config("Dynamips").get("dynamips_path", "dynamips")
+        if not os.path.isabs(dynamips_path):
+            dynamips_path = shutil.which(dynamips_path)
 
         if not dynamips_path:
             raise DynamipsError("Could not find Dynamips")
         if not os.path.isfile(dynamips_path):
             raise DynamipsError("Dynamips {} is not accessible".format(dynamips_path))
         if not os.access(dynamips_path, os.X_OK):
-            raise DynamipsError("Dynamips is not executable")
+            raise DynamipsError("Dynamips {} is not executable".format(dynamips_path))
 
         self._dynamips_path = dynamips_path
         return dynamips_path

gns3server/compute/iou/iou_vm.py

@@ -233,9 +233,11 @@ class IOUVM(BaseNode):
         :returns: path to IOUYAP
         """
 
-        path = self._config().get("iouyap_path", "iouyap")
-        if path == "iouyap":
-            path = shutil.which("iouyap")
+        search_path = self._config().get("iouyap_path", "iouyap")
+        path = shutil.which(search_path)
+        # shutil.which return None if the path doesn't exists
+        if not path:
+            return search_path
         return path
 
     @property

gns3server/compute/qemu/qcow2.py

@@ -61,7 +61,11 @@ class Qcow2:
         struct_format = ">IIQi"
         with open(self._path, 'rb') as f:
             content = f.read(struct.calcsize(struct_format))
-            self.magic, self.version, self.backing_file_offset, self.backing_file_size = struct.unpack_from(struct_format, content)
+            try:
+                self.magic, self.version, self.backing_file_offset, self.backing_file_size = struct.unpack_from(struct_format, content)
+
+            except struct.error:
+                raise Qcow2Error("Invalid file header for {}".format(self._path))
 
         if self.magic != 1363560955:  # The first 4 bytes contain the characters 'Q', 'F', 'I' followed by 0xfb.
             raise Qcow2Error("Invalid magic for {}".format(self._path))

gns3server/compute/virtualbox/__init__.py

@@ -66,7 +66,10 @@ class VirtualBox(BaseManager):
             elif sys.platform.startswith("darwin"):
                 vboxmanage_path = "/Applications/VirtualBox.app/Contents/MacOS/VBoxManage"
             else:
-                vboxmanage_path = shutil.which("vboxmanage")
+                vboxmanage_path = "vboxmanage"
+
+        if not os.path.abspath(vboxmanage_path):
+            vboxmanage_path = shutil.which(vboxmanage_path)
 
         if not vboxmanage_path:
             raise VirtualBoxError("Could not find VBoxManage")

gns3server/compute/virtualbox/virtualbox_vm.py

@@ -30,7 +30,7 @@ import asyncio
 
 from gns3server.utils import parse_version
 from gns3server.utils.telnet_server import TelnetServer
-from gns3server.utils.asyncio import wait_for_file_creation, wait_for_named_pipe_creation
+from gns3server.utils.asyncio import wait_for_file_creation, wait_for_named_pipe_creation, locked_coroutine
 from .virtualbox_error import VirtualBoxError
 from ..nios.nio_udp import NIOUDP
 from ..adapters.ethernet_adapter import EthernetAdapter
@@ -242,7 +242,7 @@ class VirtualBoxVM(BaseNode):
         if (yield from self.check_hw_virtualization()):
             self._hw_virtualization = True
 
-    @asyncio.coroutine
+    @locked_coroutine
     def stop(self):
         """
         Stops this VirtualBox VM.
@@ -943,8 +943,8 @@ class VirtualBoxVM(BaseNode):
 
         if self.ubridge and self.ubridge.is_running():
             yield from self._add_ubridge_udp_connection("VBOX-{}-{}".format(self._id, adapter_number),
-                                                         self._local_udp_tunnels[adapter_number][1],
-                                                         nio)
+                                                        self._local_udp_tunnels[adapter_number][1],
+                                                        nio)
         else:
             vm_state = yield from self._get_vm_state()
             if vm_state == "running":

gns3server/compute/vmware/__init__.py

@@ -39,6 +39,7 @@ from gns3server.compute.base_manager import BaseManager
 from gns3server.compute.vmware.vmware_vm import VMwareVM
 from gns3server.compute.vmware.vmware_error import VMwareError
 
+
 class VMware(BaseManager):
 
     _NODE_CLASS = VMwareVM
@@ -103,7 +104,10 @@ class VMware(BaseManager):
             elif sys.platform.startswith("darwin"):
                 vmrun_path = "/Applications/VMware Fusion.app/Contents/Library/vmrun"
             else:
-                vmrun_path = shutil.which("vmrun")
+                vmrun_path = "vmrun"
+
+        if not os.path.abspath(vmrun_path):
+            vmrun_path = shutil.which(vmrun_path)
 
         if not vmrun_path:
             raise VMwareError("Could not find VMware vmrun, please make sure it is installed")

gns3server/compute/vpcs/vpcs_vm.py

@@ -153,9 +153,11 @@ class VPCSVM(BaseNode):
         :returns: path to VPCS
         """
 
-        path = self._manager.config.get_section_config("VPCS").get("vpcs_path", "vpcs")
-        if path == "vpcs":
-            path = shutil.which("vpcs")
+        search_path = self._manager.config.get_section_config("VPCS").get("vpcs_path", "vpcs")
+        path = shutil.which(search_path)
+        # shutil.which return None if the path doesn't exists
+        if not path:
+            return search_path
         return path
 
     @BaseNode.name.setter

gns3server/controller/gns3vm/__init__.py

@@ -128,7 +128,7 @@ class GNS3VM:
         """
         The GNSVM is activated
         """
-        return self._settings["enable"]
+        return self._settings.get("enable", False)
 
     @property
     def auto_stop(self):

gns3server/handlers/upload_handler.py

@@ -0,0 +1,165 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2015 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import aiohttp
+import stat
+import io
+import tarfile
+import asyncio
+
+from ..config import Config
+from ..web.route import Route
+from ..utils.images import remove_checksum, md5sum
+
+
+class UploadHandler:
+
+    @classmethod
+    @Route.get(
+        r"/upload",
+        description="Manage upload of GNS3 images",
+        api_version=None
+    )
+    def index(request, response):
+        uploaded_files = []
+        try:
+            for root, _, files in os.walk(UploadHandler.image_directory()):
+                for filename in files:
+                    if not filename.startswith(".") and not filename.endswith(".md5sum"):
+                        image_file = os.path.join(root, filename)
+                        uploaded_files.append(image_file)
+        except OSError:
+            pass
+        iourc_path = os.path.join(os.path.expanduser("~/"), ".iourc")
+        if os.path.exists(iourc_path):
+            uploaded_files.append(iourc_path)
+        response.template("upload.html", files=uploaded_files)
+
+    @classmethod
+    @Route.post(
+        r"/upload",
+        description="Manage upload of GNS3 images",
+        api_version=None,
+        raw=True
+    )
+    def upload(request, response):
+        data = yield from request.post()
+
+        if not data["file"]:
+            response.redirect("/upload")
+            return
+
+        if data["type"] not in ["IOU", "IOURC", "QEMU", "IOS", "IMAGES", "PROJECTS"]:
+            raise aiohttp.web.HTTPForbidden(text="You are not authorized to upload this kind of image {}".format(data["type"]))
+
+        try:
+            if data["type"] == "IMAGES":
+                UploadHandler._restore_directory(data["file"], UploadHandler.image_directory())
+            elif data["type"] == "PROJECTS":
+                UploadHandler._restore_directory(data["file"], UploadHandler.project_directory())
+            else:
+                if data["type"] == "IOURC":
+                    destination_dir = os.path.expanduser("~/")
+                    destination_path = os.path.join(destination_dir, ".iourc")
+                else:
+                    destination_dir = os.path.join(UploadHandler.image_directory(), data["type"])
+                    destination_path = os.path.join(destination_dir, data["file"].filename)
+                os.makedirs(destination_dir, exist_ok=True)
+                remove_checksum(destination_path)
+                with open(destination_path, "wb+") as f:
+                    while True:
+                        chunk = data["file"].file.read(512)
+                        if not chunk:
+                            break
+                        f.write(chunk)
+                md5sum(destination_path)
+                st = os.stat(destination_path)
+                os.chmod(destination_path, st.st_mode | stat.S_IXUSR)
+        except OSError as e:
+            response.html("Could not upload file: {}".format(e))
+            response.set_status(200)
+            return
+        response.redirect("/upload")
+
+    @classmethod
+    @Route.get(
+        r"/backup/images.tar",
+        description="Backup GNS3 images",
+        api_version=None
+    )
+    def backup_images(request, response):
+        yield from UploadHandler._backup_directory(request, response, UploadHandler.image_directory())
+
+    @classmethod
+    @Route.get(
+        r"/backup/projects.tar",
+        description="Backup GNS3 projects",
+        api_version=None
+    )
+    def backup_projects(request, response):
+        yield from UploadHandler._backup_directory(request, response, UploadHandler.project_directory())
+
+    @staticmethod
+    def _restore_directory(file, directory):
+        """
+        Extract from HTTP stream the content of a tar
+        """
+        destination_path = os.path.join(directory, "archive.tar")
+        os.makedirs(directory, exist_ok=True)
+        with open(destination_path, "wb+") as f:
+            chunk = file.file.read()
+            f.write(chunk)
+        t = tarfile.open(destination_path)
+        t.extractall(directory)
+        t.close()
+        os.remove(destination_path)
+
+    @staticmethod
+    @asyncio.coroutine
+    def _backup_directory(request, response, directory):
+        """
+        Return a tar archive from a directory
+        """
+        response.content_type = 'application/x-gtar'
+        response.set_status(200)
+        response.enable_chunked_encoding()
+        # Very important: do not send a content length otherwise QT close the connection but curl can consume the Feed
+        response.content_length = None
+        response.start(request)
+
+        buffer = io.BytesIO()
+        with tarfile.open('arch.tar', 'w', fileobj=buffer) as tar:
+            for root, dirs, files in os.walk(directory):
+                for file in files:
+                    path = os.path.join(root, file)
+                    tar.add(os.path.join(root, file), arcname=os.path.relpath(path, directory))
+                    response.write(buffer.getvalue())
+                    yield from response.drain()
+                    buffer.truncate(0)
+                    buffer.seek(0)
+        yield from response.write_eof()
+
+    @staticmethod
+    def image_directory():
+        server_config = Config.instance().get_section_config("Server")
+        return os.path.expanduser(server_config.get("images_path", "~/GNS3/images"))
+
+    @staticmethod
+    def project_directory():
+        server_config = Config.instance().get_section_config("Server")
+        return os.path.expanduser(server_config.get("projects_path", "~/GNS3/projects"))

gns3server/utils/asyncio/__init__.py

@@ -127,3 +127,24 @@ def wait_for_named_pipe_creation(pipe_path, timeout=60):
         else:
             return
     raise asyncio.TimeoutError()
+
+
+def locked_coroutine(f):
+    """
+    Method decorator that replace asyncio.coroutine that warranty
+    that this specific method of this class instance will not we
+    executed twice at the same time
+    """
+    @asyncio.coroutine
+    def new_function(*args, **kwargs):
+
+        # In the instance of the class we will store
+        # a lock has an attribute.
+        lock_var_name = "__" + f.__name__ + "_lock"
+        if not hasattr(args[0], lock_var_name):
+            setattr(args[0], lock_var_name, asyncio.Lock())
+
+        with (yield from getattr(args[0], lock_var_name)):
+            return (yield from f(*args, **kwargs))
+
+    return new_function

init/gns3.service.systemd

@@ -3,12 +3,15 @@ Description=GNS3 server
 
 [Service]
 Type=forking
-Environment=statedir=/var/cache/gns3
-PIDFile=/var/run/gns3.pid
-ExecStart=/usr/local/bin/gns3server --log /var/log/gns3.log \
- --pid /var/run/gns3.pid --daemon
-Restart=on-abort
 User=gns3
+Group=gns3
+PermissionsStartOnly=true
+ExecStartPre=/bin/mkdir -p /var/log/gns3 /var/run/gns3
+ExecStartPre=/bin/chown -R gns3:gns3 /var/log/gns3 /var/run/gns3
+ExecStart=/usr/local/bin/gns3server --log /var/log/gns3/gns3.log \
+ --pid /var/run/gns3/gns3.pid --daemon
+Restart=on-abort
+PIDFile=/var/run/gns3/gns3.pid
 
 [Install]
 WantedBy=multi-user.target

tests/utils/test_asyncio.py

@@ -21,7 +21,7 @@ import pytest
 import sys
 from unittest.mock import MagicMock
 
-from gns3server.utils.asyncio import wait_run_in_executor, subprocess_check_output, wait_for_process_termination
+from gns3server.utils.asyncio import wait_run_in_executor, subprocess_check_output, wait_for_process_termination, locked_coroutine
 
 
 def test_wait_run_in_executor(loop):
@@ -67,3 +67,26 @@ def test_wait_for_process_termination(loop):
     exec = wait_for_process_termination(process, timeout=0.5)
     with pytest.raises(asyncio.TimeoutError):
         loop.run_until_complete(asyncio.async(exec))
+
+
+def test_lock_decorator(loop):
+    """
+    The test check if the the second call to method_to_lock wait for the
+    first call to finish
+    """
+
+    class TestLock:
+
+        def __init__(self):
+            self._test_val = 0
+
+        @locked_coroutine
+        def method_to_lock(self):
+            res = self._test_val
+            yield from asyncio.sleep(0.1)
+            self._test_val += 1
+            return res
+
+    i = TestLock()
+    res = set(loop.run_until_complete(asyncio.gather(i.method_to_lock(), i.method_to_lock())))
+    assert res == set((0, 1,))  # We use a set to test this to avoid order issue