From 6f9f004ebb78e6684b96ba3d0a9253d94909be11 Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Thu, 14 May 2015 12:03:17 +0200 Subject: [PATCH] API for list & download files of a project --- gns3server/handlers/api/project_handler.py | 73 +++++++++++++++++++++- gns3server/modules/project.py | 40 ++++++++++++ gns3server/schemas/project.py | 23 +++++++ tests/conftest.py | 7 ++- tests/handlers/api/test_project.py | 40 +++++++++++- tests/modules/test_project.py | 26 ++++++++ 6 files changed, 205 insertions(+), 4 deletions(-) diff --git a/gns3server/handlers/api/project_handler.py b/gns3server/handlers/api/project_handler.py index 54fe3283..a5018db3 100644 --- a/gns3server/handlers/api/project_handler.py +++ b/gns3server/handlers/api/project_handler.py @@ -15,13 +15,16 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import aiohttp import asyncio import json +import os from ...web.route import Route -from ...schemas.project import PROJECT_OBJECT_SCHEMA, PROJECT_CREATE_SCHEMA, PROJECT_UPDATE_SCHEMA +from ...schemas.project import PROJECT_OBJECT_SCHEMA, PROJECT_CREATE_SCHEMA, PROJECT_UPDATE_SCHEMA, PROJECT_FILE_LIST_SCHEMA from ...modules.project_manager import ProjectManager from ...modules import MODULES +from ...utils.asyncio import wait_run_in_executor import logging log = logging.getLogger() @@ -198,3 +201,71 @@ class ProjectHandler: response.write("{\"action\": \"ping\"}\n".encode("utf-8")) project.stop_listen_queue(queue) ProjectHandler._notifications_listening -= 1 + + @classmethod + @Route.get( + r"/projects/{project_id}/files", + description="List files of a project", + parameters={ + "project_id": "The UUID of the project", + }, + status_codes={ + 200: "Return list of files", + 404: "The project doesn't exist" + }, + output=PROJECT_FILE_LIST_SCHEMA) + def list_files(request, response): + + pm = ProjectManager.instance() + project = pm.get_project(request.match_info["project_id"]) + files = yield from project.list_files() + response.json(files) + response.set_status(200) + + @classmethod + @Route.get( + r"/projects/{project_id}/files/{path:.+}", + description="Get a file of a project", + parameters={ + "project_id": "The UUID of the project", + }, + status_codes={ + 200: "Return the file", + 403: "Permission denied", + 404: "The file doesn't exist" + }) + def get_file(request, response): + + pm = ProjectManager.instance() + project = pm.get_project(request.match_info["project_id"]) + path = request.match_info["path"] + path = os.path.normpath(path) + + # Raise error if user try to escape + if path[0] == ".": + raise aiohttp.web.HTTPForbidden + path = os.path.join(project.path, path) + + response.content_type = "application/octet-stream" + response.set_status(200) + response.enable_chunked_encoding() + # Very important: do not send a content length otherwise QT close the connection but curl can consume the Feed + response.content_length = None + + try: + yield from wait_run_in_executor(ProjectHandler._read_file, path, request, response) + except FileNotFoundError: + raise aiohttp.web.HTTPNotFound() + except PermissionError: + raise aiohttp.web.HTTPForbidden + + @staticmethod + def _read_file(path, request, response): + + with open(path, "rb") as f: + response.start(request) + while True: + data = f.read(4096) + if not data: + break + response.write(data) diff --git a/gns3server/modules/project.py b/gns3server/modules/project.py index 85f4c7f5..fc191221 100644 --- a/gns3server/modules/project.py +++ b/gns3server/modules/project.py @@ -19,6 +19,7 @@ import aiohttp import os import shutil import asyncio +import hashlib from uuid import UUID, uuid4 from .port_manager import PortManager @@ -457,3 +458,42 @@ class Project: """Stop sending notification to this clients""" self._listeners.remove(queue) + + @asyncio.coroutine + def list_files(self): + """ + :returns: Array of files in project without temporary files. The files are dictionnary {"path": "test.bin", "md5sum": "aaaaa"} + """ + + files = [] + for (dirpath, dirnames, filenames) in os.walk(self.path): + for filename in filenames: + if not filename.endswith(".ghost"): + path = os.path.relpath(dirpath, self.path) + path = os.path.join(path, filename) + path = os.path.normpath(path) + file_info = {"path": path} + + try: + file_info["md5sum"] = yield from wait_run_in_executor(self._hash_file, os.path.join(dirpath, filename)) + except OSError: + continue + files.append(file_info) + + return files + + def _hash_file(self, path): + """ + Compute and md5 hash for file + + :returns: hexadecimal md5 + """ + + m = hashlib.md5() + with open(path, "rb") as f: + while True: + buf = f.read(128) + if not buf: + break + m.update(buf) + return m.hexdigest() diff --git a/gns3server/schemas/project.py b/gns3server/schemas/project.py index 3e9dfa6d..a07039b6 100644 --- a/gns3server/schemas/project.py +++ b/gns3server/schemas/project.py @@ -103,3 +103,26 @@ PROJECT_OBJECT_SCHEMA = { "additionalProperties": False, "required": ["location", "project_id", "temporary"] } + +PROJECT_FILE_LIST_SCHEMA = { + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "List files in the project", + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "path": { + "description": "File path", + "type": ["string"] + }, + "md5sum": { + "description": "MD5 hash of the file", + "type": ["string"] + }, + + }, + } + ], + "additionalProperties": False, +} diff --git a/tests/conftest.py b/tests/conftest.py index 74aee0cb..5e49a7da 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,6 +24,8 @@ import shutil import os import sys from aiohttp import web +from unittest.mock import patch + sys._called_from_test = True # Prevent execution of external binaries @@ -100,10 +102,11 @@ def server(request, loop, port_manager, monkeypatch): @pytest.fixture(scope="function") -def project(): +def project(tmpdir): """A GNS3 lab""" - return ProjectManager.instance().create_project(project_id="a1e920ca-338a-4e9f-b363-aa607b09dd80") + p = ProjectManager.instance().create_project(project_id="a1e920ca-338a-4e9f-b363-aa607b09dd80") + return p @pytest.fixture(scope="session") diff --git a/tests/handlers/api/test_project.py b/tests/handlers/api/test_project.py index 604ec756..432401b1 100644 --- a/tests/handlers/api/test_project.py +++ b/tests/handlers/api/test_project.py @@ -20,12 +20,14 @@ This test suite check /project endpoint """ import uuid +import os import asyncio import aiohttp from unittest.mock import patch from tests.utils import asyncio_patch from gns3server.handlers.api.project_handler import ProjectHandler +from gns3server.modules.project_manager import ProjectManager def test_create_project_with_path(server, tmpdir): @@ -175,6 +177,42 @@ def test_notification(server, project, loop): assert response.body == b'{"action": "ping"}\n{"action": "vm.created", "event": {"a": "b"}}\n' -def test_notification_invalid_id(server, project): +def test_notification_invalid_id(server): response = server.get("/projects/{project_id}/notifications".format(project_id=uuid.uuid4())) assert response.status == 404 + + +def test_list_files(server, project): + files = [ + { + "path": "test.txt", + "md5sum": "ad0234829205b9033196ba818f7a872b" + }, + { + "path": "vm-1/dynamips/test.bin", + "md5sum": "098f6bcd4621d373cade4e832627b4f6" + } + ] + with asyncio_patch("gns3server.modules.project.Project.list_files", return_value=files) as mock: + response = server.get("/projects/{project_id}/files".format(project_id=project.id), example=True) + assert response.status == 200 + assert response.json == files + + +def test_get_file(server, tmpdir): + + with patch("gns3server.config.Config.get_section_config", return_value={"project_directory": str(tmpdir)}): + project = ProjectManager.instance().create_project() + + with open(os.path.join(project.path, "hello"), "w+") as f: + f.write("world") + + response = server.get("/projects/{project_id}/files/hello".format(project_id=project.id), raw=True, example=True) + assert response.status == 200 + assert response.body == b"world" + + response = server.get("/projects/{project_id}/files/false".format(project_id=project.id), raw=True) + assert response.status == 404 + + response = server.get("/projects/{project_id}/files/../hello".format(project_id=project.id), raw=True) + assert response.status == 403 diff --git a/tests/modules/test_project.py b/tests/modules/test_project.py index c4eaee4b..7e268d59 100644 --- a/tests/modules/test_project.py +++ b/tests/modules/test_project.py @@ -229,3 +229,29 @@ def test_clean_project_directory(tmpdir): assert os.path.exists(str(project1)) assert os.path.exists(str(oldproject)) assert not os.path.exists(str(project2)) + + +def test_list_files(tmpdir, loop): + + with patch("gns3server.config.Config.get_section_config", return_value={"project_directory": str(tmpdir)}): + project = Project() + path = project.path + os.makedirs(os.path.join(path, "vm-1", "dynamips")) + with open(os.path.join(path, "vm-1", "dynamips", "test.bin"), "w+") as f: + f.write("test") + open(os.path.join(path, "vm-1", "dynamips", "test.ghost"), "w+").close() + with open(os.path.join(path, "test.txt"), "w+") as f: + f.write("test2") + + files = loop.run_until_complete(asyncio.async(project.list_files())) + + assert files == [ + { + "path": "test.txt", + "md5sum": "ad0234829205b9033196ba818f7a872b" + }, + { + "path": os.path.join("vm-1", "dynamips", "test.bin"), + "md5sum": "098f6bcd4621d373cade4e832627b4f6" + } + ]