From 1ce576c020758402eae2ec145c191a0dad8e85be Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Fri, 22 Apr 2016 16:22:03 +0200 Subject: [PATCH] Stream pcap from compute to controller to client --- gns3server/compute/project.py | 4 +- gns3server/controller/compute.py | 16 +++++ gns3server/controller/link.py | 7 +++ gns3server/controller/udp_link.py | 9 +++ gns3server/handlers/api/compute/__init__.py | 1 - .../handlers/api/compute/file_handler.py | 59 ------------------ .../handlers/api/compute/project_handler.py | 44 +++++++++++++ .../handlers/api/controller/link_handler.py | 40 ++++++++++++ gns3server/templates/project.html | 2 + tests/controller/test_compute.py | 8 +++ tests/controller/test_link.py | 1 + tests/controller/test_udp_link.py | 15 +++++ tests/handlers/api/compute/test_file.py | 62 ------------------- tests/handlers/api/compute/test_project.py | 19 ++++++ tests/handlers/api/controller/test_link.py | 9 +++ 15 files changed, 172 insertions(+), 124 deletions(-) delete mode 100644 gns3server/handlers/api/compute/file_handler.py delete mode 100644 tests/handlers/api/compute/test_file.py diff --git a/gns3server/compute/project.py b/gns3server/compute/project.py index 67c6c610..5140bbd2 100644 --- a/gns3server/compute/project.py +++ b/gns3server/compute/project.py @@ -284,7 +284,7 @@ class Project: """ A temporary directory. Will be clean at project open and close """ - return os.path.join(self._path, "project-files", "tmp") + return os.path.join(self._path, "tmp") def capture_working_directory(self): """ @@ -293,7 +293,7 @@ class Project: :returns: path to the directory """ - workdir = os.path.join(self._path, "project-files", "tmp", "captures") + workdir = os.path.join(self._path, "tmp", "captures") try: os.makedirs(workdir, exist_ok=True) except OSError as e: diff --git a/gns3server/controller/compute.py b/gns3server/controller/compute.py index 2c9fc367..97cafa41 100644 --- a/gns3server/controller/compute.py +++ b/gns3server/controller/compute.py @@ -125,6 +125,22 @@ class Compute: "connected": self._connected } + @asyncio.coroutine + def streamFile(self, project, path): + """ + Read file of a project and stream it + + :param project: A project object + :param path: The path of the file in the project + :returns: A file stream + """ + + url = self._getUrl("/projects/{}/stream/{}".format(project.id, path)) + response = yield from self._session.request("GET", url, auth=self._auth) + if response.status == 404: + raise aiohttp.web.HTTPNotFound(text="{} not found on compute".format(path)) + return response.content + @asyncio.coroutine def httpQuery(self, method, path, data=None): if not self._connected: diff --git a/gns3server/controller/link.py b/gns3server/controller/link.py index 432dfc99..219cefe5 100644 --- a/gns3server/controller/link.py +++ b/gns3server/controller/link.py @@ -70,6 +70,13 @@ class Link: """ raise NotImplementedError + @asyncio.coroutine + def read_pcap(self): + """ + Return a FileStream of the Pcap from the compute node + """ + raise NotImplementedError + def capture_file_name(self): """ :returns: File name for a capture on this link diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index 854ea931..f335786f 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -122,3 +122,12 @@ class UDPLink(Link): return vm raise aiohttp.web.HTTPConflict(text="Capture is not supported for this link") + + @asyncio.coroutine + def read_pcap(self): + """ + Return a FileStream of the Pcap from the compute node + """ + if self._capture_vm: + compute = self._capture_vm["vm"].compute + return compute.streamFile(self._project, "tmp/captures/" + self.capture_file_name()) diff --git a/gns3server/handlers/api/compute/__init__.py b/gns3server/handlers/api/compute/__init__.py index f0a74b38..ccd984e4 100644 --- a/gns3server/handlers/api/compute/__init__.py +++ b/gns3server/handlers/api/compute/__init__.py @@ -27,7 +27,6 @@ from .virtualbox_handler import VirtualBoxHandler from .vpcs_handler import VPCSHandler from .vmware_handler import VMwareHandler from .config_handler import ConfigHandler -from .file_handler import FileHandler from .version_handler import VersionHandler from .notification_handler import NotificationHandler diff --git a/gns3server/handlers/api/compute/file_handler.py b/gns3server/handlers/api/compute/file_handler.py deleted file mode 100644 index 02243a01..00000000 --- a/gns3server/handlers/api/compute/file_handler.py +++ /dev/null @@ -1,59 +0,0 @@ -# -# Copyright (C) 2015 GNS3 Technologies Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import asyncio -import aiohttp - -from ....web.route import Route -from ....schemas.file import FILE_STREAM_SCHEMA - - -class FileHandler: - - @classmethod - @Route.get( - r"/files/stream", - description="Stream a file from the server", - status_codes={ - 200: "File retrieved", - 404: "File doesn't exist", - 409: "Can't access to file" - }, - input=FILE_STREAM_SCHEMA - ) - def read(request, response): - response.enable_chunked_encoding() - - try: - with open(request.json.get("location"), "rb") as f: - loop = asyncio.get_event_loop() - response.content_type = "application/octet-stream" - response.set_status(200) - # Very important: do not send a content lenght otherwise QT close the connection but curl can consume the Feed - response.content_length = None - - response.start(request) - - while True: - data = yield from loop.run_in_executor(None, f.read, 16) - if len(data) == 0: - yield from asyncio.sleep(0.1) - else: - response.write(data) - except FileNotFoundError: - raise aiohttp.web.HTTPNotFound() - except OSError as e: - raise aiohttp.web.HTTPConflict(text=str(e)) diff --git a/gns3server/handlers/api/compute/project_handler.py b/gns3server/handlers/api/compute/project_handler.py index 24991ba3..ac673714 100644 --- a/gns3server/handlers/api/compute/project_handler.py +++ b/gns3server/handlers/api/compute/project_handler.py @@ -305,6 +305,50 @@ class ProjectHandler: except PermissionError: raise aiohttp.web.HTTPForbidden() + @classmethod + @Route.get( + r"/projects/{project_id}/stream/{path:.+}", + description="Stream a file from a project", + parameters={ + "project_id": "The UUID of the project", + }, + status_codes={ + 200: "Return the file", + 403: "Permission denied", + 404: "The file doesn't exist" + }) + def stream_file(request, response): + + pm = ProjectManager.instance() + project = pm.get_project(request.match_info["project_id"]) + path = request.match_info["path"] + path = os.path.normpath(path) + + # Raise error if user try to escape + if path[0] == ".": + raise aiohttp.web.HTTPForbidden + path = os.path.join(project.path, path) + + response.content_type = "application/octet-stream" + response.set_status(200) + response.enable_chunked_encoding() + # Very important: do not send a content length otherwise QT close the connection but curl can consume the Feed + response.content_length = None + + try: + with open(path, "rb") as f: + response.start(request) + while True: + data = f.read(4096) + if not data: + yield from asyncio.sleep(0.1) + yield from response.write(data) + + except FileNotFoundError: + raise aiohttp.web.HTTPNotFound() + except PermissionError: + raise aiohttp.web.HTTPForbidden() + @classmethod @Route.post( r"/projects/{project_id}/files/{path:.+}", diff --git a/gns3server/handlers/api/controller/link_handler.py b/gns3server/handlers/api/controller/link_handler.py index 329ba65b..58291a85 100644 --- a/gns3server/handlers/api/controller/link_handler.py +++ b/gns3server/handlers/api/controller/link_handler.py @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import aiohttp +import asyncio + from ....web.route import Route from ....schemas.link import LINK_OBJECT_SCHEMA, LINK_CAPTURE_SCHEMA from ....controller.project import Project @@ -113,3 +116,40 @@ class LinkHandler: yield from link.delete() response.set_status(204) response.json(link) + + @classmethod + @Route.get( + r"/projects/{project_id}/links/{link_id}/pcap", + parameters={ + "project_id": "UUID for the project", + "link_id": "UUID of the link" + }, + description="Get the pcap from the capture", + status_codes={ + 200: "Return the file", + 403: "Permission denied", + 404: "The file doesn't exist" + }) + def pcap(request, response): + + controller = Controller.instance() + project = controller.getProject(request.match_info["project_id"]) + link = project.getLink(request.match_info["link_id"]) + + content = yield from link.read_pcap() + if content is None: + raise aiohttp.web.HTTPNotFound(text="pcap file not found") + + response.content_type = "application/vnd.tcpdump.pcap" + response.set_status(200) + response.enable_chunked_encoding() + # Very important: do not send a content length otherwise QT close the connection but curl can consume the Feed + response.content_length = None + + response.start(request) + + while True: + chunk = yield from content.read(4096) + if not chunk: + yield from asyncio.sleep(0.1) + yield from response.write(chunk) diff --git a/gns3server/templates/project.html b/gns3server/templates/project.html index 253c935f..5a03d2bf 100644 --- a/gns3server/templates/project.html +++ b/gns3server/templates/project.html @@ -45,11 +45,13 @@ in futur GNS3 versions. ID Capture + PCAP {% for link in project.links.values() %} {{link.id}} {{link.capturing}} + Download {% endfor %} diff --git a/tests/controller/test_compute.py b/tests/controller/test_compute.py index babc7445..63aa8fc7 100644 --- a/tests/controller/test_compute.py +++ b/tests/controller/test_compute.py @@ -179,3 +179,11 @@ def test_json(compute): "user": "test", "connected": True } + + +def test_streamFile(project, async_run, compute): + response = MagicMock() + response.status = 200 + with asyncio_patch("aiohttp.ClientSession.request", return_value=response) as mock: + async_run(compute.streamFile(project, "test/titi")) + mock.assert_called_with("GET", "https://example.com:84/v2/compute/projects/{}/stream/test/titi".format(project.id), auth=None) diff --git a/tests/controller/test_link.py b/tests/controller/test_link.py index 4504c3ee..6c7610c1 100644 --- a/tests/controller/test_link.py +++ b/tests/controller/test_link.py @@ -18,6 +18,7 @@ import pytest from unittest.mock import MagicMock + from gns3server.controller.link import Link from gns3server.controller.vm import VM from gns3server.controller.compute import Compute diff --git a/tests/controller/test_udp_link.py b/tests/controller/test_udp_link.py index ca16a9d3..7bd21eb3 100644 --- a/tests/controller/test_udp_link.py +++ b/tests/controller/test_udp_link.py @@ -19,6 +19,7 @@ import pytest import asyncio import aiohttp from unittest.mock import MagicMock +from tests.utils import asyncio_patch from gns3server.controller.project import Project from gns3server.controller.compute import Compute @@ -160,3 +161,17 @@ def test_capture(async_run, project): assert link.capturing is False compute1.post.assert_any_call("/projects/{}/iou/vms/{}/adapters/3/ports/1/stop_capture".format(project.id, vm_iou.id)) + + +def test_read_pcap(project, async_run): + compute1 = MagicMock() + + link = UDPLink(project) + async_run(link.addVM(compute1, 0, 4)) + async_run(link.addVM(compute1, 3, 1)) + + capture = async_run(link.start_capture()) + assert link._capture_vm is not None + + async_run(link.read_pcap()) + link._capture_vm["vm"].compute.streamFile.assert_called_with(project, "tmp/captures/" + link.capture_file_name()) diff --git a/tests/handlers/api/compute/test_file.py b/tests/handlers/api/compute/test_file.py deleted file mode 100644 index 76b692a4..00000000 --- a/tests/handlers/api/compute/test_file.py +++ /dev/null @@ -1,62 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2015 GNS3 Technologies Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -This test suite check /files endpoint -""" - -import json -import asyncio -import aiohttp - -from gns3server.version import __version__ - - -def test_stream(http_compute, tmpdir, loop): - with open(str(tmpdir / "test"), 'w+') as f: - f.write("hello") - - def go(future): - query = json.dumps({"location": str(tmpdir / "test")}) - headers = {'content-type': 'application/json'} - response = yield from aiohttp.request("GET", http_compute.get_url("/files/stream"), data=query, headers=headers) - response.body = yield from response.content.read(5) - with open(str(tmpdir / "test"), 'a') as f: - f.write("world") - response.body += yield from response.content.read(5) - response.close() - future.set_result(response) - - future = asyncio.Future() - asyncio.async(go(future)) - response = loop.run_until_complete(future) - assert response.status == 200 - assert response.body == b'helloworld' - - -def test_stream_file_not_found(http_compute, tmpdir, loop): - def go(future): - query = json.dumps({"location": str(tmpdir / "test")}) - headers = {'content-type': 'application/json'} - response = yield from aiohttp.request("GET", http_compute.get_url("/files/stream"), data=query, headers=headers) - response.close() - future.set_result(response) - - future = asyncio.Future() - asyncio.async(go(future)) - response = loop.run_until_complete(future) - assert response.status == 404 diff --git a/tests/handlers/api/compute/test_project.py b/tests/handlers/api/compute/test_project.py index 497e6f03..6fcbf0ca 100644 --- a/tests/handlers/api/compute/test_project.py +++ b/tests/handlers/api/compute/test_project.py @@ -218,6 +218,25 @@ def test_get_file(http_compute, tmpdir): assert response.status == 403 +def test_stream_file(http_compute, tmpdir): + + with patch("gns3server.config.Config.get_section_config", return_value={"project_directory": str(tmpdir)}): + project = ProjectManager.instance().create_project(project_id="01010203-0405-0607-0809-0a0b0c0d0e0b") + + with open(os.path.join(project.path, "hello"), "w+") as f: + f.write("world") + + response = http_compute.get("/projects/{project_id}/files/hello".format(project_id=project.id), raw=True) + assert response.status == 200 + assert response.body == b"world" + + response = http_compute.get("/projects/{project_id}/files/false".format(project_id=project.id), raw=True) + assert response.status == 404 + + response = http_compute.get("/projects/{project_id}/files/../hello".format(project_id=project.id), raw=True) + assert response.status == 403 + + def test_export(http_compute, tmpdir, loop, project): os.makedirs(project.path, exist_ok=True) diff --git a/tests/handlers/api/controller/test_link.py b/tests/handlers/api/controller/test_link.py index fd37ba90..521b841e 100644 --- a/tests/handlers/api/controller/test_link.py +++ b/tests/handlers/api/controller/test_link.py @@ -95,6 +95,15 @@ def test_stop_capture(http_controller, tmpdir, project, compute, async_run): assert response.status == 204 +def test_pcap(http_controller, tmpdir, project, compute, async_run): + link = Link(project) + link + project._links = {link.id: link} + with asyncio_patch("gns3server.controller.link.Link.read_pcap", return_value=None) as mock: + response = http_controller.get("/projects/{}/links/{}/pcap".format(project.id, link.id), example=True) + assert mock.called + + def test_delete_link(http_controller, tmpdir, project, compute, async_run): link = Link(project)