ExternalVendorCode/gns3-server
1
0
Fork 0
mirror of https://github.com/GNS3/gns3-server.git synced 2024-12-19 21:07:57 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Compute a md5sum of images for futur purpose

Browse Source 
Fix #234
This commit is contained in:
Julien Duponchelle 2015-06-17 17:11:25 +02:00
parent 0f15e4b56a
commit f041697311
15 changed files with 248 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
gns3server/handlers/upload_handler.py
View File

@ -21,6 +21,7 @@ import stat
from ..config import Config
from ..web.route import Route
from ..utils.images import remove_checksum, md5sum
class UploadHandler:
@ -36,7 +37,7 @@ class UploadHandler:
try:
for root, _, files in os.walk(UploadHandler.image_directory()):
for filename in files:
if not filename.startswith("."):
if not filename.startswith(".") and not filename.endswith(".md5sum"):
image_file = os.path.join(root, filename)
uploaded_files.append(image_file)
except OSError:
@ -70,12 +71,14 @@ class UploadHandler:
destination_path = os.path.join(destination_dir, data["file"].filename)
try:
os.makedirs(destination_dir, exist_ok=True)
remove_checksum(destination_path)
with open(destination_path, "wb+") as f:
while True:
chunk = data["file"].file.read(512)
if not chunk:
break
f.write(chunk)
md5sum(destination_path)
st = os.stat(destination_path)
os.chmod(destination_path, st.st_mode | stat.S_IXUSR)
except OSError as e:

5
gns3server/modules/base_manager.py
View File

@ -37,6 +37,7 @@ from .nios.nio_udp import NIOUDP
from .nios.nio_tap import NIOTAP
from .nios.nio_nat import NIONAT
from .nios.nio_generic_ethernet import NIOGenericEthernet
from ..utils.images import md5sum, remove_checksum
class BaseManager:
@ -444,7 +445,7 @@ class BaseManager:
files.sort()
images = []
for filename in files:
if filename[0] != ".":
if filename[0] != "." and not filename.endswith(".md5sum"):
images.append({"filename": filename})
return images
@ -461,6 +462,7 @@ class BaseManager:
path = os.path.join(directory, os.path.basename(filename))
log.info("Writting image file %s", path)
try:
remove_checksum(path)
os.makedirs(directory, exist_ok=True)
with open(path, 'wb+') as f:
while True:
@ -469,5 +471,6 @@ class BaseManager:
break
f.write(packet)
os.chmod(path, stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC)
md5sum(path)
except OSError as e:
raise aiohttp.web.HTTPConflict(text="Could not write image: {} to {}".format(filename, e))

2
gns3server/modules/dynamips/nodes/router.py
View File

@ -37,6 +37,7 @@ from ..nios.nio_udp import NIOUDP
from gns3server.config import Config
from gns3server.utils.asyncio import wait_run_in_executor, monitor_process
from gns3server.utils.images import md5sum
class Router(BaseVM):
@ -134,6 +135,7 @@ class Router(BaseVM):
"dynamips_id": self._dynamips_id,
"platform": self._platform,
"image": self._image,
"image_md5sum": md5sum(self._image),
"startup_config": self._startup_config,
"private_config": self._private_config,
"ram": self._ram,

2
gns3server/modules/iou/iou_vm.py
View File

@ -46,6 +46,7 @@ from .utils.iou_import import nvram_import
from .utils.iou_export import nvram_export
from .ioucon import start_ioucon
import gns3server.utils.asyncio
import gns3server.utils.images
import logging
@ -208,6 +209,7 @@ class IOUVM(BaseVM):
"console": self._console,
"project_id": self.project.id,
"path": self.path,
"md5sum": gns3server.utils.images.md5sum(self.path),
"ethernet_adapters": len(self._ethernet_adapters),
"serial_adapters": len(self._serial_adapters),
"ram": self._ram,

11
gns3server/modules/qemu/qemu_vm.py
View File

@ -38,6 +38,7 @@ from ..nios.nio_nat import NIONAT
from ..base_vm import BaseVM
from ...schemas.qemu import QEMU_OBJECT_SCHEMA, QEMU_PLATFORMS
from ...utils.asyncio import monitor_process
from ...utils.images import md5sum
import logging
log = logging.getLogger(__name__)
@ -1217,13 +1218,23 @@ class QemuVM(BaseVM):
# Qemu has a long list of options. The JSON schema is the single source of information
for field in QEMU_OBJECT_SCHEMA["required"]:
if field not in answer:
try:
answer[field] = getattr(self, field)
except AttributeError:
pass
answer["hda_disk_image"] = self.manager.get_relative_image_path(self._hda_disk_image)
answer["hda_disk_image_md5sum"] = md5sum(self._hda_disk_image)
answer["hdb_disk_image"] = self.manager.get_relative_image_path(self._hdb_disk_image)
answer["hdb_disk_image_md5sum"] = md5sum(self._hdb_disk_image)
answer["hdc_disk_image"] = self.manager.get_relative_image_path(self._hdc_disk_image)
answer["hdc_disk_image_md5sum"] = md5sum(self._hdc_disk_image)
answer["hdd_disk_image"] = self.manager.get_relative_image_path(self._hdd_disk_image)
answer["hdd_disk_image_md5sum"] = md5sum(self._hdd_disk_image)
answer["initrd"] = self.manager.get_relative_image_path(self._initrd)
answer["initrd_md5sum"] = md5sum(self._initrd)
answer["kernel_image"] = self.manager.get_relative_image_path(self._kernel_image)
answer["kernel_image_md5sum"] = md5sum(self._kernel_image)
return answer

5
gns3server/schemas/dynamips_vm.py
View File

@ -546,6 +546,11 @@ VM_OBJECT_SCHEMA = {
"type": "string",
"minLength": 1,
},
"image_md5sum": {
"description": "checksum of the IOS image",
"type": "string",
"minLength": 1,
},
"startup_config": {
"description": "path to the IOS startup configuration file",
"type": "string",

6
gns3server/schemas/iou.py
View File

@ -189,6 +189,10 @@ IOU_OBJECT_SCHEMA = {
"description": "Path of iou binary",
"type": "string"
},
"md5sum": {
"description": "Checksum of iou binary",
"type": "string"
},
"serial_adapters": {
"description": "How many serial adapters are connected to the IOU",
"type": "integer"
@ -227,7 +231,7 @@ IOU_OBJECT_SCHEMA = {
}
},
"additionalProperties": False,
"required": ["name", "vm_id", "console", "project_id", "path", "serial_adapters", "ethernet_adapters",
"required": ["name", "vm_id", "console", "project_id", "path", "md5sum", "serial_adapters", "ethernet_adapters",
"ram", "nvram", "l1_keepalives", "startup_config", "private_config", "use_default_iou_values"]
}

31
gns3server/schemas/qemu.py
View File

@ -282,18 +282,34 @@ QEMU_OBJECT_SCHEMA = {
"description": "QEMU hda disk image path",
"type": "string",
},
"hda_disk_image_md5sum": {
"description": "QEMU hda disk image checksum",
"type": ["string", "null"]
},
"hdb_disk_image": {
"description": "QEMU hdb disk image path",
"type": "string",
},
"hdb_disk_image_md5sum": {
"description": "QEMU hdb disk image checksum",
"type": ["string", "null"],
},
"hdc_disk_image": {
"description": "QEMU hdc disk image path",
"type": "string",
},
"hdc_disk_image_md5sum": {
"description": "QEMU hdc disk image checksum",
"type": ["string", "null"],
},
"hdd_disk_image": {
"description": "QEMU hdd disk image path",
"type": "string",
},
"hdd_disk_image_md5sum": {
"description": "QEMU hdd disk image checksum",
"type": ["string", "null"],
},
"ram": {
"description": "amount of RAM in MB",
"type": "integer"
@ -325,10 +341,18 @@ QEMU_OBJECT_SCHEMA = {
"description": "QEMU initrd path",
"type": "string",
},
"initrd_md5sum": {
"description": "QEMU initrd path",
"type": ["string", "null"],
},
"kernel_image": {
"description": "QEMU kernel image path",
"type": "string",
},
"kernel_image_md5sum": {
"description": "QEMU kernel image checksum",
"type": ["string", "null"],
},
"kernel_command_line": {
"description": "QEMU kernel command line",
"type": "string",
@ -367,9 +391,10 @@ QEMU_OBJECT_SCHEMA = {
},
"additionalProperties": False,
"required": ["vm_id", "project_id", "name", "qemu_path", "platform", "hda_disk_image", "hdb_disk_image",
"hdc_disk_image", "hdd_disk_image", "ram", "adapters", "adapter_type", "mac_address", "console",
"initrd", "kernel_image", "kernel_command_line", "legacy_networking", "acpi_shutdown", "kvm",
"cpu_throttling", "process_priority", "options"]
"hdc_disk_image", "hdd_disk_image", "hda_disk_image_md5sum", "hdb_disk_image_md5sum",
"hdc_disk_image_md5sum", "hdd_disk_image_md5sum", "ram", "adapters", "adapter_type", "mac_address",
"console", "initrd", "kernel_image", "initrd_md5sum", "kernel_image_md5sum", "kernel_command_line",
"legacy_networking", "acpi_shutdown", "kvm", "cpu_throttling", "process_priority", "options"]
}
QEMU_BINARY_LIST_SCHEMA = {

64
gns3server/utils/images.py Normal file
View File

@ -0,0 +1,64 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2014 GNS3 Technologies Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import hashlib
def md5sum(path):
"""
Return the md5sum of an image and cache it on disk
:param path: Path to the image
:returns: Digest of the image
"""
if path is None or len(path) == 0:
return None
try:
with open(path + '.md5sum') as f:
return f.read()
except OSError:
pass
m = hashlib.md5()
with open(path, 'rb') as f:
while True:
buf = f.read(128)
if not buf:
break
m.update(buf)
digest = m.hexdigest()
try:
with open('{}.md5sum'.format(path), 'w+') as f:
f.write(digest)
except OSError as e:
log.error("Can't write digest of %s: %s", path, str(e))
return digest
def remove_checksum(path):
"""
Remove the checksum of an image from cache if exists
"""
path = '{}.md5sum'.format(path)
if os.path.exists(path):
os.remove(path)

4
tests/handlers/api/test_dynamips.py
View File

@ -156,6 +156,10 @@ def test_upload_vm(server, tmpdir):
with open(str(tmpdir / "test2")) as f:
assert f.read() == "TEST"
with open(str(tmpdir / "test2.md5sum")) as f:
checksum = f.read()
assert checksum == "033bd94b1168d7e4f0d644c3c95e35bf"
def test_upload_vm_permission_denied(server, tmpdir):
with open(str(tmpdir / "test2"), "w+") as f:

5
tests/handlers/api/test_iou.py
View File

@ -301,6 +301,7 @@ def test_get_configs_without_configs_file(server, vm):
assert "startup_config" not in response.json
assert "private_config" not in response.json
def test_get_configs_with_startup_config_file(server, project, vm):
path = startup_config_file(project, vm)
@ -328,6 +329,10 @@ def test_upload_vm(server, tmpdir):
with open(str(tmpdir / "test2")) as f:
assert f.read() == "TEST"
with open(str(tmpdir / "test2.md5sum")) as f:
checksum = f.read()
assert checksum == "033bd94b1168d7e4f0d644c3c95e35bf"
def test_upload_vm_permission_denied(server, tmpdir):
with open(str(tmpdir / "test2"), "w+") as f:

16
tests/handlers/api/test_qemu.py
View File

@ -88,10 +88,10 @@ def test_qemu_create_platform(server, project, base_params, fake_qemu_bin):
assert response.json["platform"] == "x86_64"
def test_qemu_create_with_params(server, project, base_params):
def test_qemu_create_with_params(server, project, base_params, fake_qemu_vm):
params = base_params
params["ram"] = 1024
params["hda_disk_image"] = "/tmp/hda"
params["hda_disk_image"] = fake_qemu_vm
response = server.post("/projects/{project_id}/qemu/vms".format(project_id=project.id), params, example=True)
assert response.status == 201
@ -99,7 +99,7 @@ def test_qemu_create_with_params(server, project, base_params):
assert response.json["name"] == "PC TEST 1"
assert response.json["project_id"] == project.id
assert response.json["ram"] == 1024
assert response.json["hda_disk_image"] == "/tmp/hda"
assert response.json["hda_disk_image"] == fake_qemu_vm
def test_qemu_get(server, project, vm):
@ -152,18 +152,18 @@ def test_qemu_delete(server, vm):
assert response.status == 204
def test_qemu_update(server, vm, tmpdir, free_console_port, project):
def test_qemu_update(server, vm, tmpdir, free_console_port, project, fake_qemu_vm):
params = {
"name": "test",
"console": free_console_port,
"ram": 1024,
"hdb_disk_image": "/tmp/hdb"
"hdb_disk_image": fake_qemu_vm
}
response = server.put("/projects/{project_id}/qemu/vms/{vm_id}".format(project_id=vm["project_id"], vm_id=vm["vm_id"]), params, example=True)
assert response.status == 200
assert response.json["name"] == "test"
assert response.json["console"] == free_console_port
assert response.json["hdb_disk_image"] == "/tmp/hdb"
assert response.json["hdb_disk_image"] == fake_qemu_vm
assert response.json["ram"] == 1024
@ -225,6 +225,10 @@ def test_upload_vm(server, tmpdir):
with open(str(tmpdir / "test2")) as f:
assert f.read() == "TEST"
with open(str(tmpdir / "test2.md5sum")) as f:
checksum = f.read()
assert checksum == "033bd94b1168d7e4f0d644c3c95e35bf"
def test_upload_vm_permission_denied(server, tmpdir):
with open(str(tmpdir / "test2"), "w+") as f:

47
tests/handlers/test_upload.py
View File

@ -21,12 +21,23 @@ import os
from unittest.mock import patch
from gns3server.config import Config
def test_index_upload(server):
def test_index_upload(server, tmpdir):
Config.instance().set("Server", "images_path", str(tmpdir))
open(str(tmpdir / "alpha"), "w+").close()
open(str(tmpdir / "alpha.md5sum"), "w+").close()
open(str(tmpdir / ".beta"), "w+").close()
response = server.get('/upload', api_version=None)
assert response.status == 200
html = response.html
assert "GNS3 Server" in html
assert "Select & Upload" in html
assert "alpha" in html
assert ".beta" not in html
assert "alpha.md5sum" not in html
def test_upload(server, tmpdir):
@ -40,9 +51,43 @@ def test_upload(server, tmpdir):
body.add_field("file", open(str(tmpdir / "test"), "rb"), content_type="application/iou", filename="test2")
Config.instance().set("Server", "images_path", str(tmpdir))
response = server.post('/upload', api_version=None, body=body, raw=True)
assert "test2" in response.body.decode("utf-8")
with open(str(tmpdir / "QEMU" / "test2")) as f:
assert f.read() == content
with open(str(tmpdir / "QEMU" / "test2.md5sum")) as f:
checksum = f.read()
assert checksum == "ae187e1febee2a150b64849c32d566ca"
def test_upload_previous_checksum(server, tmpdir):
content = ''.join(['a' for _ in range(0, 1025)])
with open(str(tmpdir / "test"), "w+") as f:
f.write(content)
body = aiohttp.FormData()
body.add_field("type", "QEMU")
body.add_field("file", open(str(tmpdir / "test"), "rb"), content_type="application/iou", filename="test2")
Config.instance().set("Server", "images_path", str(tmpdir))
os.makedirs(str(tmpdir / "QEMU"))
with open(str(tmpdir / "QEMU" / "test2.md5sum"), 'w+') as f:
f.write("FAKE checksum")
response = server.post('/upload', api_version=None, body=body, raw=True)
assert "test2" in response.body.decode("utf-8")
with open(str(tmpdir / "QEMU" / "test2")) as f:
assert f.read() == content
with open(str(tmpdir / "QEMU" / "test2.md5sum")) as f:
checksum = f.read()
assert checksum == "ae187e1febee2a150b64849c32d566ca"

2
tests/modules/test_manager.py
View File

@ -125,7 +125,7 @@ def test_get_relative_image_path(qemu, tmpdir):
def test_list_images(loop, qemu, tmpdir):
fake_images = ["a.bin", "b.bin", ".blu.bin"]
fake_images = ["a.bin", "b.bin", ".blu.bin", "a.bin.md5sum"]
for image in fake_images:
with open(str(tmpdir / image), "w+") as f:
f.write("1")

55
tests/utils/test_images.py Normal file
View File

@ -0,0 +1,55 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2014 GNS3 Technologies Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
from gns3server.utils.images import md5sum, remove_checksum
def test_md5sum(tmpdir):
fake_img = str(tmpdir / 'hello')
with open(fake_img, 'w+') as f:
f.write('hello')
assert md5sum(fake_img) == '5d41402abc4b2a76b9719d911017c592'
with open(str(tmpdir / 'hello.md5sum')) as f:
assert f.read() == '5d41402abc4b2a76b9719d911017c592'
def test_md5sum_existing_digest(tmpdir):
fake_img = str(tmpdir / 'hello')
with open(str(tmpdir / 'hello.md5sum'), 'w+') as f:
f.write('aaaaa02abc4b2a76b9719d911017c592')
assert md5sum(fake_img) == 'aaaaa02abc4b2a76b9719d911017c592'
def test_md5sum_none(tmpdir):
assert md5sum(None) is None
def test_remove_checksum(tmpdir):
with open(str(tmpdir / 'hello.md5sum'), 'w+') as f:
f.write('aaaaa02abc4b2a76b9719d911017c592')
remove_checksum(str(tmpdir / 'hello'))
assert not os.path.exists(str(tmpdir / 'hello.md5sum'))
remove_checksum(str(tmpdir / 'not_exists'))