Merge pull request #2428 from GNS3/bugfix/2427

Symbolic links support for project export/import

gns3server/controller/export_project.py

@@ -70,14 +70,15 @@ async def export_project(zstream, project, temporary_dir, include_images=False,
             files = [f for f in files if _is_exportable(os.path.join(root, f), include_snapshots)]
             for file in files:
                 path = os.path.join(root, file)
-                # check if we can export the file
+                if not os.path.islink(path):
-                try:
+                    try:
-                    open(path).close()
+                        # check if we can export the file
-                except OSError as e:
+                        open(path).close()
-                    msg = "Could not export file {}: {}".format(path, e)
+                    except OSError as e:
-                    log.warning(msg)
+                        msg = "Could not export file {}: {}".format(path, e)
-                    project.emit_notification("log.warning", {"message": msg})
+                        log.warning(msg)
-                    continue
+                        project.emit_notification("log.warning", {"message": msg})
+                        continue
                 # ignore the .gns3 file
                 if file.endswith(".gns3"):
                     continue
@@ -128,7 +129,7 @@ def _patch_mtime(path):
     if sys.platform.startswith("win"):
         # only UNIX type platforms
         return
-    st = os.stat(path)
+    st = os.stat(path, follow_symlinks=False)
     file_date = datetime.fromtimestamp(st.st_mtime)
     if file_date.year < 1980:
         new_mtime = file_date.replace(year=1980).timestamp()
@@ -144,10 +145,6 @@ def _is_exportable(path, include_snapshots=False):
     if include_snapshots is False and path.endswith("snapshots"):
         return False
 
-    # do not export symlinks
-    if os.path.islink(path):
-        return False
-
     # do not export directories of snapshots
     if include_snapshots is False and "{sep}snapshots{sep}".format(sep=os.path.sep) in path:
         return False

gns3server/controller/import_project.py

@@ -17,6 +17,7 @@
 
 import os
 import sys
+import stat
 import json
 import uuid
 import shutil
@@ -93,6 +94,7 @@ async def import_project(controller, project_id, stream, location=None, name=Non
     try:
         with zipfile.ZipFile(stream) as zip_file:
             await wait_run_in_executor(zip_file.extractall, path)
+            _create_symbolic_links(zip_file, path)
     except zipfile.BadZipFile:
         raise aiohttp.web.HTTPConflict(text="Cannot extract files from GNS3 project (invalid zip)")
 
@@ -174,6 +176,24 @@ async def import_project(controller, project_id, stream, location=None, name=Non
     project = await controller.load_project(dot_gns3_path, load=False)
     return project
 
+def _create_symbolic_links(zip_file, path):
+    """
+    Manually create symbolic links (if any) because ZipFile does not support it.
+
+    :param zip_file: ZipFile instance
+    :param path: project location
+    """
+
+    for zip_info in zip_file.infolist():
+        if stat.S_ISLNK(zip_info.external_attr >> 16):
+            symlink_target = zip_file.read(zip_info.filename).decode()
+            symlink_path = os.path.join(path, zip_info.filename)
+            try:
+                # remove the regular file and replace it by a symbolic link
+                os.remove(symlink_path)
+                os.symlink(symlink_target, symlink_path)
+            except OSError as e:
+                raise aiohttp.web.HTTPConflict(text=f"Cannot create symbolic link: {e}")
 
 def _move_node_file(path, old_id, new_id):
     """
@@ -257,6 +277,7 @@ async def _import_snapshots(snapshots_path, project_name, project_id):
                 with open(snapshot_path, "rb") as f:
                     with zipfile.ZipFile(f) as zip_file:
                         await wait_run_in_executor(zip_file.extractall, tmpdir)
+                        _create_symbolic_links(zip_file, tmpdir)
             except OSError as e:
                 raise aiohttp.web.HTTPConflict(text="Cannot open snapshot '{}': {}".format(os.path.basename(snapshot), e))
             except zipfile.BadZipFile:

gns3server/controller/project.py

@@ -1277,7 +1277,7 @@ class Project:
         new_project_id = str(uuid.uuid4())
         new_project_path = p_work.joinpath(new_project_id)
         # copy dir
-        await wait_run_in_executor(shutil.copytree, self.path, new_project_path.as_posix())
+        await wait_run_in_executor(shutil.copytree, self.path, new_project_path.as_posix(), symlinks=True, ignore_dangling_symlinks=True)
         log.info("Project content copied from '{}' to '{}' in {}s".format(self.path, new_project_path, time.time() - t0))
         topology = json.loads(new_project_path.joinpath('{}.gns3'.format(self.name)).read_bytes())
         project_name = name or topology["name"]

gns3server/utils/asyncio/aiozipstream.py

@@ -161,14 +161,17 @@ class ZipFile(zipfile.ZipFile):
         self._comment = comment
         self._didModify = True
 
-    async def data_generator(self, path):
+    async def data_generator(self, path, islink=False):
 
-        async with aiofiles.open(path, "rb") as f:
+        if islink:
-            while True:
+            yield os.readlink(path).encode()
-                part = await f.read(self._chunksize)
+        else:
-                if not part:
+            async with aiofiles.open(path, "rb") as f:
-                    break
+                while True:
-                yield part
+                    part = await f.read(self._chunksize)
+                    if not part:
+                        break
+                    yield part
         return
 
     async def _run_in_executor(self, task, *args, **kwargs):
@@ -224,12 +227,13 @@ class ZipFile(zipfile.ZipFile):
             raise ValueError("either (exclusively) filename or iterable shall be not None")
 
         if filename:
-            st = os.stat(filename)
+            st = os.stat(filename, follow_symlinks=False)
             isdir = stat.S_ISDIR(st.st_mode)
+            islink = stat.S_ISLNK(st.st_mode)
             mtime = time.localtime(st.st_mtime)
             date_time = mtime[0:6]
         else:
-            st, isdir, date_time = None, False, time.localtime()[0:6]
+            st, isdir, islink, date_time = None, False, False, time.localtime()[0:6]
         # Create ZipInfo instance to store file information
         if arcname is None:
             arcname = filename
@@ -282,7 +286,7 @@ class ZipFile(zipfile.ZipFile):
 
         file_size = 0
         if filename:
-            async for buf in self.data_generator(filename):
+            async for buf in self.data_generator(filename, islink):
                 file_size = file_size + len(buf)
                 CRC = zipfile.crc32(buf, CRC) & 0xffffffff
                 if cmpr:

tests/controller/test_export_project.py

@@ -21,6 +21,7 @@ import json
 import pytest
 import aiohttp
 import zipfile
+import stat
 
 from pathlib import Path
 from unittest.mock import patch
@@ -116,6 +117,8 @@ async def test_export(tmpdir, project):
     with open(os.path.join(path, "project-files", "snapshots", "test"), 'w+') as f:
         f.write("WORLD")
 
+    os.symlink("/tmp/anywhere", os.path.join(path, "vm-1", "dynamips", "symlink"))
+
     with aiozipstream.ZipFile() as z:
         with patch("gns3server.compute.Dynamips.get_images_directory", return_value=str(tmpdir / "IOS"),):
             await export_project(z, project, str(tmpdir), include_images=False)
@@ -131,9 +134,12 @@ async def test_export(tmpdir, project):
         assert 'vm-1/dynamips/empty-dir/' in myzip.namelist()
         assert 'project-files/snapshots/test' not in myzip.namelist()
         assert 'vm-1/dynamips/test_log.txt' not in myzip.namelist()
-
         assert 'images/IOS/test.image' not in myzip.namelist()
 
+        assert 'vm-1/dynamips/symlink' in myzip.namelist()
+        zip_info = myzip.getinfo('vm-1/dynamips/symlink')
+        assert stat.S_ISLNK(zip_info.external_attr >> 16)
+
         with myzip.open("project.gns3") as myfile:
             topo = json.loads(myfile.read().decode())["topology"]
             assert topo["nodes"][0]["compute_id"] == "local"  # All node should have compute_id local after export

tests/controller/test_import_project.py

@@ -21,7 +21,11 @@ import json
 import zipfile
 
 from tests.utils import asyncio_patch, AsyncioMagicMock
+from unittest.mock import patch, MagicMock
 
+from gns3server.utils.asyncio import aiozipstream
+from gns3server.controller.project import Project
+from gns3server.controller.export_project import export_project
 from gns3server.controller.import_project import import_project, _move_files_to_compute
 from gns3server.version import __version__
 
@@ -106,6 +110,54 @@ async def test_import_project_override(tmpdir, controller):
     assert project.name == "test"
 
 
+async def write_file(path, z):
+
+    with open(path, 'wb') as f:
+        async for chunk in z:
+            f.write(chunk)
+
+
+async def test_import_project_containing_symlink(tmpdir, controller):
+
+    project = Project(controller=controller, name="test")
+    project.dump = MagicMock()
+    path = project.path
+
+    project_id = str(uuid.uuid4())
+    topology = {
+        "project_id": str(uuid.uuid4()),
+        "name": "test",
+        "auto_open": True,
+        "auto_start": True,
+        "topology": {
+        },
+        "version": "2.0.0"
+    }
+
+    with open(os.path.join(path, "project.gns3"), 'w+') as f:
+        json.dump(topology, f)
+
+    os.makedirs(os.path.join(path, "vm1", "dynamips"))
+    symlink_path = os.path.join(project.path, "vm1", "dynamips", "symlink")
+    symlink_target = "/tmp/anywhere"
+    os.symlink(symlink_target, symlink_path)
+
+    zip_path = str(tmpdir / "project.zip")
+    with aiozipstream.ZipFile() as z:
+        with patch("gns3server.compute.Dynamips.get_images_directory", return_value=str(tmpdir / "IOS"),):
+            await export_project(z, project, str(tmpdir), include_images=False)
+            await write_file(zip_path, z)
+
+    with open(zip_path, "rb") as f:
+        project = await import_project(controller, project_id, f)
+
+    assert project.name == "test"
+    assert project.id == project_id
+    symlink_path = os.path.join(project.path, "vm1", "dynamips", "symlink")
+    assert os.path.islink(symlink_path)
+    assert os.readlink(symlink_path) == symlink_target
+
+
 async def test_import_upgrade(tmpdir, controller):
     """
     Topology made for previous GNS3 version are upgraded during the process