From f35787918691c5bfa2c1bbbeb4ff83d5f403e0ea Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Mon, 25 Jul 2016 14:47:37 +0200 Subject: [PATCH] API for duplicate a project Ref https://github.com/GNS3/gns3-gui/issues/995 --- gns3server/controller/export_project.py | 48 ++++++++------ gns3server/controller/import_project.py | 65 +++++++++++-------- gns3server/controller/project.py | 59 ++++++++++++++++- .../api/controller/project_handler.py | 34 +++++++++- tests/controller/test_export_project.py | 42 ++++++++++++ tests/controller/test_import_project.py | 49 +++++++++++++- tests/controller/test_project.py | 36 +++++++++- tests/handlers/api/controller/test_project.py | 7 ++ tests/utils.py | 2 +- 9 files changed, 285 insertions(+), 57 deletions(-) diff --git a/gns3server/controller/export_project.py b/gns3server/controller/export_project.py index 102095ec..c3c86681 100644 --- a/gns3server/controller/export_project.py +++ b/gns3server/controller/export_project.py @@ -25,7 +25,7 @@ import zipstream @asyncio.coroutine -def export_project(project, temporary_dir, include_images=False): +def export_project(project, temporary_dir, include_images=False, keep_compute_id=False, allow_all_nodes=False): """ Export the project as zip. It's a ZipStream object. The file will be read chunk by chunk when you iterate on @@ -34,6 +34,8 @@ def export_project(project, temporary_dir, include_images=False): It will ignore some files like snapshots and :param temporary_dir: A temporary dir where to store intermediate data + :param keep_compute_id: If false replace all compute id by local it's the standard behavior for .gns3project to make them portable + :param allow_all_nodes: Allow all nodes type to be include in the zip even if not portable default False :returns: ZipStream object """ @@ -46,7 +48,7 @@ def export_project(project, temporary_dir, include_images=False): # First we process the .gns3 in order to be sure we don't have an error for file in os.listdir(project._path): if file.endswith(".gns3"): - _export_project_file(project, os.path.join(project._path, file), z, include_images) + _export_project_file(project, os.path.join(project._path, file), z, include_images, keep_compute_id, allow_all_nodes) for root, dirs, files in os.walk(project._path, topdown=True): files = [f for f in files if not _filter_files(os.path.join(root, f))] @@ -61,10 +63,10 @@ def export_project(project, temporary_dir, include_images=False): log.warn(msg) project.emit("log.warning", {"message": msg}) continue - if file.endswith(".gns3"): - pass - else: - z.write(path, os.path.relpath(path, project._path), compress_type=zipfile.ZIP_DEFLATED) + if file.endswith(".gns3"): + pass + else: + z.write(path, os.path.relpath(path, project._path), compress_type=zipfile.ZIP_DEFLATED) for compute in project.computes: if compute.id != "local": @@ -104,7 +106,7 @@ def _filter_files(path): return False -def _export_project_file(project, path, z, include_images): +def _export_project_file(project, path, z, include_images, keep_compute_id, allow_all_nodes): """ Take a project file (.gns3) and patch it for the export @@ -118,22 +120,26 @@ def _export_project_file(project, path, z, include_images): with open(path) as f: topology = json.load(f) - if "topology" in topology and "nodes" in topology["topology"]: - for node in topology["topology"]["nodes"]: - if node["node_type"] in ["virtualbox", "vmware", "cloud"]: - raise aiohttp.web.HTTPConflict(text="Topology with a {} could not be exported".format(node["node_type"])) - - node["compute_id"] = "local" # To make project portable all node by default run on local - - if "properties" in node and node["node_type"] != "Docker": - for prop, value in node["properties"].items(): - if prop.endswith("image"): - node["properties"][prop] = os.path.basename(value) - if include_images is True: - images.add(value) if "topology" in topology: - topology["topology"]["computes"] = [] # Strip compute informations because could contain secret info like password + if "nodes" in topology["topology"]: + for node in topology["topology"]["nodes"]: + if not allow_all_nodes and node["node_type"] in ["virtualbox", "vmware", "cloud"]: + raise aiohttp.web.HTTPConflict(text="Topology with a {} could not be exported".format(node["node_type"])) + + if not keep_compute_id: + node["compute_id"] = "local" # To make project portable all node by default run on local + + if "properties" in node and node["node_type"] != "Docker": + for prop, value in node["properties"].items(): + if prop.endswith("image"): + if not keep_compute_id: # If we keep the original compute we can keep the image path + node["properties"][prop] = os.path.basename(value) + if include_images is True: + images.add(value) + + if not keep_compute_id: + topology["topology"]["computes"] = [] # Strip compute informations because could contain secret info like password for image in images: _export_images(project, image, z) diff --git a/gns3server/controller/import_project.py b/gns3server/controller/import_project.py index 7c8b81c3..2c63f4da 100644 --- a/gns3server/controller/import_project.py +++ b/gns3server/controller/import_project.py @@ -34,7 +34,7 @@ Handle the import of project from a .gns3project @asyncio.coroutine -def import_project(controller, project_id, stream, location=None, name=None): +def import_project(controller, project_id, stream, location=None, name=None, keep_compute_id=False): """ Import a project contain in a zip file @@ -45,13 +45,9 @@ def import_project(controller, project_id, stream, location=None, name=None): :param stream: A io.BytesIO of the zipfile :param location: Parent directory for the project if None put in the default directory :param name: Wanted project name, generate one from the .gns3 if None + :param keep_compute_id: If true do not touch the compute id :returns: Project """ - if location: - projects_path = location - else: - projects_path = controller.projects_directory() - os.makedirs(projects_path, exist_ok=True) with zipfile.ZipFile(stream) as myzip: @@ -65,31 +61,42 @@ def import_project(controller, project_id, stream, location=None, name=None): except KeyError: raise aiohttp.web.HTTPConflict(text="Can't import topology the .gns3 is corrupted or missing") - path = os.path.join(projects_path, project_name) + if location: + path = location + else: + projects_path = controller.projects_directory() + path = os.path.join(projects_path, project_name) os.makedirs(path) myzip.extractall(path) topology = load_topology(os.path.join(path, "project.gns3")) topology["name"] = project_name - # For some VM type we move them to the GNS3 VM if it's not a Linux host - if not sys.platform.startswith("linux"): - vm_created = False + # Modify the compute id of the node depending of compute capacity + if not keep_compute_id: + # For some VM type we move them to the GNS3 VM if it's not a Linux host + if not sys.platform.startswith("linux"): + for node in topology["topology"]["nodes"]: + if node["node_type"] in ("docker", "qemu", "iou"): + node["compute_id"] = "vm" + else: + for node in topology["topology"]["nodes"]: + node["compute_id"] = "local" - for node in topology["topology"]["nodes"]: - if node["node_type"] in ("docker", "qemu", "iou"): - node["compute_id"] = "vm" + compute_created = set() + for node in topology["topology"]["nodes"]: - # Project created on the remote GNS3 VM? - if not vm_created: - compute = controller.get_compute("vm") - yield from compute.post("/projects", data={ - "name": project_name, - "project_id": project_id, - }) - vm_created = True + if node["compute_id"] != "local": + # Project created on the remote GNS3 VM? + if node["compute_id"] not in compute_created: + compute = controller.get_compute(node["compute_id"]) + yield from compute.post("/projects", data={ + "name": project_name, + "project_id": project_id, + }) + compute_created.add(node["compute_id"]) - yield from _move_files_to_compute(compute, project_id, path, os.path.join("project-files", node["node_type"], node["node_id"])) + yield from _move_files_to_compute(compute, project_id, path, os.path.join("project-files", node["node_type"], node["node_id"])) # And we dump the updated.gns3 dot_gns3_path = os.path.join(path, project_name + ".gns3") @@ -111,12 +118,14 @@ def _move_files_to_compute(compute, project_id, directory, files_path): """ Move the files to a remote compute """ - for (dirpath, dirnames, filenames) in os.walk(os.path.join(directory, files_path)): - for filename in filenames: - path = os.path.join(dirpath, filename) - dst = os.path.relpath(path, directory) - yield from _upload_file(compute, project_id, path, dst) - shutil.rmtree(os.path.join(directory, files_path)) + location = os.path.join(directory, files_path) + if os.path.exists(location): + for (dirpath, dirnames, filenames) in os.walk(location): + for filename in filenames: + path = os.path.join(dirpath, filename) + dst = os.path.relpath(path, directory) + yield from _upload_file(compute, project_id, path, dst) + shutil.rmtree(os.path.join(directory, files_path)) @asyncio.coroutine diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index 08153ff8..0ee37e99 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -17,9 +17,11 @@ import os import json +import uuid +import shutil import asyncio import aiohttp -import shutil +import tempfile from uuid import UUID, uuid4 @@ -29,12 +31,26 @@ from .topology import project_to_topology, load_topology from .udp_link import UDPLink from ..config import Config from ..utils.path import check_path_allowed, get_default_project_directory +from .export_project import export_project +from .import_project import import_project import logging log = logging.getLogger(__name__) +def open_required(func): + """ + Use this decorator to raise an error if the project is not opened + """ + + def wrapper(self, *args, **kwargs): + if self._status == "closed": + raise aiohttp.web.HTTPForbidden(text="The project is not opened") + return func(self, *args, **kwargs) + return wrapper + + class Project: """ A project inside a controller @@ -74,8 +90,13 @@ class Project: self._filename = filename else: self._filename = self.name + ".gns3" + self.reset() + # At project creation we write an empty .gns3 + if not os.path.exists(self._topology_file()): + self.dump() + def reset(self): """ Called when open/close a project. Cleanup internal stuff @@ -212,6 +233,7 @@ class Project: return self.update_allocated_node_name(new_name) return new_name + @open_required @asyncio.coroutine def add_node(self, compute, name, node_id, **kwargs): """ @@ -243,6 +265,7 @@ class Project: return node return self._nodes[node_id] + @open_required @asyncio.coroutine def delete_node(self, node_id): @@ -258,6 +281,7 @@ class Project: self.dump() self.controller.notification.emit("node.deleted", node.__json__()) + @open_required def get_node(self, node_id): """ Return the node or raise a 404 if the node is unknown @@ -281,6 +305,7 @@ class Project: """ return self._drawings + @open_required @asyncio.coroutine def add_drawing(self, drawing_id=None, **kwargs): """ @@ -296,6 +321,7 @@ class Project: return drawing return self._drawings[drawing_id] + @open_required def get_drawing(self, drawing_id): """ Return the Drawing or raise a 404 if the drawing is unknown @@ -305,6 +331,7 @@ class Project: except KeyError: raise aiohttp.web.HTTPNotFound(text="Drawing ID {} doesn't exist".format(drawing_id)) + @open_required @asyncio.coroutine def delete_drawing(self, drawing_id): drawing = self.get_drawing(drawing_id) @@ -312,6 +339,7 @@ class Project: self.dump() self.controller.notification.emit("drawing.deleted", drawing.__json__()) + @open_required @asyncio.coroutine def add_link(self, link_id=None): """ @@ -324,6 +352,7 @@ class Project: self.dump() return link + @open_required @asyncio.coroutine def delete_link(self, link_id): link = self.get_link(link_id) @@ -332,6 +361,7 @@ class Project: self.dump() self.controller.notification.emit("link.deleted", link.__json__()) + @open_required def get_link(self, link_id): """ Return the Link or raise a 404 if the link is unknown @@ -371,6 +401,7 @@ class Project: except OSError as e: log.warning(str(e)) + @open_required @asyncio.coroutine def delete(self): yield from self.close() @@ -406,6 +437,8 @@ class Project: return self.reset() + self._status = "opened" + path = self._topology_file() if os.path.exists(path): topology = load_topology(path)["topology"] @@ -424,7 +457,29 @@ class Project: for drawing_data in topology.get("drawings", []): drawing = yield from self.add_drawing(**drawing_data) - self._status = "opened" + + @open_required + @asyncio.coroutine + def duplicate(self, name=None, location=None): + """ + Duplicate a project + + It's the save as feature of the 1.X. It's implemented on top of the + export / import features. It will generate a gns3p and reimport it. + It's a little slower but we have only one implementation to maintain. + + :param name: Name of the new project. A new one will be generated in case of conflicts + :param location: Parent directory of the new project + """ + + with tempfile.TemporaryDirectory() as tmpdir: + zipstream = yield from export_project(self, tmpdir, keep_compute_id=True, allow_all_nodes=True) + with open(os.path.join(tmpdir, "project.gns3p"), "wb+") as f: + for data in zipstream: + f.write(data) + with open(os.path.join(tmpdir, "project.gns3p"), "rb") as f: + project = yield from import_project(self._controller, str(uuid.uuid4()), f, location=location, name=name, keep_compute_id=True) + return project def is_running(self): """ diff --git a/gns3server/handlers/api/controller/project_handler.py b/gns3server/handlers/api/controller/project_handler.py index 0e1fe238..107519fa 100644 --- a/gns3server/handlers/api/controller/project_handler.py +++ b/gns3server/handlers/api/controller/project_handler.py @@ -236,7 +236,7 @@ class ProjectHandler: project = controller.get_project(request.match_info["project_id"]) with tempfile.TemporaryDirectory() as tmp_dir: - datas = yield from export_project(project, tmp_dir, include_images=bool(request.GET.get("include_images", "0"))) + datas = yield from export_project(project, tmp_dir, include_images=bool(request.get("include_images", "0"))) # We need to do that now because export could failed and raise an HTTP error # that why response start need to be the later possible response.content_type = 'application/gns3project' @@ -285,6 +285,38 @@ class ProjectHandler: response.json(project) response.set_status(201) + @Route.post( + r"/projects/{project_id}/duplicate", + description="Duplicate a project", + parameters={ + "project_id": "Project UUID", + }, + input=PROJECT_CREATE_SCHEMA, + output=PROJECT_OBJECT_SCHEMA, + status_codes={ + 201: "Project duplicate", + 403: "The server is not the local server", + 404: "The project doesn't exist" + }) + def duplicate(request, response): + + controller = Controller.instance() + project = controller.get_project(request.match_info["project_id"]) + + if request.json.get("path"): + config = Config.instance() + if config.get_section_config("Server").getboolean("local", False) is False: + response.set_status(403) + return + location = request.json.get("path") + else: + location = None + + new_project = yield from project.duplicate(name=request.json.get("name"), location=location) + + response.json(new_project) + response.set_status(201) + @Route.get( r"/projects/{project_id}/files/{path:.+}", description="Get a file from a project. Beware you have warranty to be able to access only to file global to the project (for example README.txt)", diff --git a/tests/controller/test_export_project.py b/tests/controller/test_export_project.py index 916a79d6..39b186b3 100644 --- a/tests/controller/test_export_project.py +++ b/tests/controller/test_export_project.py @@ -200,6 +200,7 @@ def test_export_disallow_some_type(tmpdir, project, async_run): with pytest.raises(aiohttp.web.HTTPConflict): z = async_run(export_project(project, str(tmpdir))) + z = async_run(export_project(project, str(tmpdir), allow_all_nodes=True)) def test_export_fix_path(tmpdir, project, async_run): @@ -271,3 +272,44 @@ def test_export_with_images(tmpdir, project, async_run): with zipfile.ZipFile(str(tmpdir / 'zipfile.zip')) as myzip: myzip.getinfo("images/IOS/test.image") + + +def test_export_keep_compute_id(tmpdir, project, async_run): + """ + If we want to restore the same computes we could ask to keep them + in the file + """ + + with open(os.path.join(project.path, "test.gns3"), 'w+') as f: + data = { + "topology": { + "computes": [ + { + "compute_id": "6b7149c8-7d6e-4ca0-ab6b-daa8ab567be0", + "host": "127.0.0.1", + "name": "Remote 1", + "port": 8001, + "protocol": "http" + } + ], + "nodes": [ + { + "compute_id": "6b7149c8-7d6e-4ca0-ab6b-daa8ab567be0", + "node_type": "vpcs" + } + ] + } + } + json.dump(data, f) + + z = async_run(export_project(project, str(tmpdir), keep_compute_id=True)) + + with open(str(tmpdir / 'zipfile.zip'), 'wb') as f: + for data in z: + f.write(data) + + with zipfile.ZipFile(str(tmpdir / 'zipfile.zip')) as myzip: + with myzip.open("project.gns3") as myfile: + topo = json.loads(myfile.read().decode())["topology"] + assert topo["nodes"][0]["compute_id"] == "6b7149c8-7d6e-4ca0-ab6b-daa8ab567be0" + assert len(topo["computes"]) == 1 diff --git a/tests/controller/test_import_project.py b/tests/controller/test_import_project.py index b68f46d0..6f11babc 100644 --- a/tests/controller/test_import_project.py +++ b/tests/controller/test_import_project.py @@ -131,7 +131,7 @@ def test_import_with_images(tmpdir, async_run, controller): assert os.path.exists(path), path -def test_import_iou_non_linux(linux_platform, async_run, tmpdir, controller): +def test_import_iou_linux(linux_platform, async_run, tmpdir, controller): """ On non linux host IOU should be local """ @@ -224,6 +224,49 @@ def test_import_iou_non_linux(windows_platform, async_run, tmpdir, controller): assert topo["topology"]["nodes"][1]["compute_id"] == "local" +def test_import_keep_compute_id(windows_platform, async_run, tmpdir, controller): + """ + On linux host IOU should be moved to the GNS3 VM + """ + project_id = str(uuid.uuid4()) + controller._computes["vm"] = AsyncioMagicMock() + + topology = { + "project_id": str(uuid.uuid4()), + "name": "test", + "type": "topology", + "topology": { + "nodes": [ + { + "compute_id": "local", + "node_id": "0fd3dd4d-dc93-4a04-a9b9-7396a9e22e8b", + "node_type": "iou", + "properties": {} + } + ], + "links": [], + "computes": [], + "drawings": [] + }, + "revision": 5, + "version": "2.0.0" + } + + with open(str(tmpdir / "project.gns3"), 'w+') as f: + json.dump(topology, f) + + zip_path = str(tmpdir / "project.zip") + with zipfile.ZipFile(zip_path, 'w') as myzip: + myzip.write(str(tmpdir / "project.gns3"), "project.gns3") + + with open(zip_path, "rb") as f: + project = async_run(import_project(controller, project_id, f, keep_compute_id=True)) + + with open(os.path.join(project.path, "test.gns3")) as f: + topo = json.load(f) + assert topo["topology"]["nodes"][0]["compute_id"] == "local" + + def test_move_files_to_compute(tmpdir, async_run): project_id = str(uuid.uuid4()) @@ -261,11 +304,11 @@ def test_import_project_name_and_location(async_run, tmpdir, controller): myzip.write(str(tmpdir / "project.gns3"), "project.gns3") with open(zip_path, "rb") as f: - project = async_run(import_project(controller, project_id, f, name="hello", location=str(tmpdir / "test"))) + project = async_run(import_project(controller, project_id, f, name="hello", location=str(tmpdir / "hello"))) assert project.name == "hello" - assert os.path.exists(str(tmpdir / "test" / "hello" / "hello.gns3")) + assert os.path.exists(str(tmpdir / "hello" / "hello.gns3")) # A new project name is generated when you import twice the same name with open(zip_path, "rb") as f: diff --git a/tests/controller/test_project.py b/tests/controller/test_project.py index a82540f8..82e97b8b 100644 --- a/tests/controller/test_project.py +++ b/tests/controller/test_project.py @@ -203,7 +203,7 @@ def test_delete_node_delete_link(async_run, controller): controller.notification.emit.assert_any_call("link.deleted", link.__json__()) -def test_getVM(async_run, controller): +def test_get_node(async_run, controller): compute = MagicMock() project = Project(controller=controller, name="Test") @@ -217,6 +217,11 @@ def test_getVM(async_run, controller): with pytest.raises(aiohttp.web_exceptions.HTTPNotFound): project.get_node("test") + # Raise an error if the project is not opened + async_run(project.close()) + with pytest.raises(aiohttp.web.HTTPForbidden): + project.get_node(vm.id) + def test_addLink(async_run, project, controller): compute = MagicMock() @@ -339,3 +344,32 @@ def test_is_running(project, async_run, node): assert project.is_running() is False node._status = "started" assert project.is_running() is True + + +def test_duplicate(project, async_run, controller): + """ + Duplicate a project, the node should remain on the remote server + if they were on remote server + """ + compute = MagicMock() + compute.id = "remote" + compute.list_files = AsyncioMagicMock(return_value=[]) + controller._computes["remote"] = compute + + response = MagicMock() + response.json = {"console": 2048} + compute.post = AsyncioMagicMock(return_value=response) + + remote_vpcs = async_run(project.add_node(compute, "test", None, node_type="vpcs", properties={"startup_config": "test.cfg"})) + + # We allow node not allowed for standard import / export + remote_virtualbox = async_run(project.add_node(compute, "test", None, node_type="virtualbox", properties={"startup_config": "test.cfg"})) + + new_project = async_run(project.duplicate(name="Hello")) + assert new_project.id != project.id + assert new_project.name == "Hello" + + async_run(new_project.open()) + + assert new_project.get_node(remote_vpcs.id).compute.id == "remote" + assert new_project.get_node(remote_virtualbox.id).compute.id == "remote" diff --git a/tests/handlers/api/controller/test_project.py b/tests/handlers/api/controller/test_project.py index 77edb2b6..80427908 100644 --- a/tests/handlers/api/controller/test_project.py +++ b/tests/handlers/api/controller/test_project.py @@ -220,3 +220,10 @@ def test_import(http_controller, tmpdir, controller): with open(os.path.join(project.path, "demo")) as f: content = f.read() assert content == "hello" + + +def test_duplicate(http_controller, tmpdir, loop, project): + + response = http_controller.post("/projects/{project_id}/duplicate".format(project_id=project.id), {"name": "hello"}, example=True) + assert response.status == 201 + assert response.json["name"] == "hello" diff --git a/tests/utils.py b/tests/utils.py index 72c0152a..3c638d52 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -74,7 +74,7 @@ class AsyncioMagicMock(unittest.mock.MagicMock): """ :return_values: Array of return value at each call will return the next """ - if return_value: + if return_value is not None: future = asyncio.Future() future.set_result(return_value) kwargs["return_value"] = future