diff --git a/CHANGELOG b/CHANGELOG index 9fd09c9b..86a37fa8 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,14 @@ # Change Log +## 2.2.52 02/12/2024 + +* Bundle web-ui v2.2.52 +* Sync appliances +* Remove restrictions based on file extension when listing images and fix ELF header checks +* Fix use project name instead of ID for fast duplication when running local server. Fixes #2446 +* Overwrite user resources when the originals have changed. +* Relax setuptools requirement to allow for easier Debian packaging on Ubuntu Focal & Jammy + ## 3.0.0rc2 20/11/2024 * Bundle web-ui v3.0.0rc2 diff --git a/gns3server/appliances/cisco-7200.gns3a b/gns3server/appliances/cisco-7200.gns3a index 6638702b..3e723e50 100644 --- a/gns3server/appliances/cisco-7200.gns3a +++ b/gns3server/appliances/cisco-7200.gns3a @@ -33,11 +33,23 @@ "md5sum": "cbbbea66a253f1dac0fcf81274dc778d", "filesize": 87756936 }, + { + "filename": "c7200-adventerprisek9-mz.152-4.M11.image", + "version": "152-4.M11", + "md5sum": "9a2005ad09ce1ec6fe7cf9af1e9b099e", + "filesize": 128487680 + }, { "filename": "c7200-adventerprisek9-mz.124-24.T5.image", "version": "124-24.T5", "md5sum": "6b89d0d804e1f2bb5b8bda66b5692047", "filesize": 102345240 + }, + { + "filename": "c7200-a3jk9s-mz.124-25g.image", + "version": "124-25G", + "md5sum": "9c7cc9b3f3b3571411a7f62faaa2c036", + "filesize": 71528984 } ], "versions": [ @@ -55,12 +67,26 @@ "image": "c7200-advipservicesk9-mz.152-4.S5.image" } }, + { + "name": "152-4.M11", + "idlepc": "0x6062e5c0", + "images": { + "image": "c7200-adventerprisek9-mz.152-4.M11.image" + } + }, { "name": "124-24.T5", "idlepc": "0x606df838", "images": { "image": "c7200-adventerprisek9-mz.124-24.T5.image" } + }, + { + "name": "124-25G", + "idlepc": "0x6066a998", + "images": { + "image": "c7200-a3jk9s-mz.124-25g.image" + } } ] } diff --git a/gns3server/appliances/cisco-asav.gns3a b/gns3server/appliances/cisco-asav.gns3a index 48ee3196..eacc48d8 100644 --- a/gns3server/appliances/cisco-asav.gns3a +++ b/gns3server/appliances/cisco-asav.gns3a @@ -12,7 +12,7 @@ "status": "stable", "maintainer": "GNS3 Team", "maintainer_email": "developers@gns3.net", - "usage": "There is no default password and enable password. A default configuration is present. ASAv goes through a double-boot before becoming active. This is normal and expected.", + "usage": "There is no default password and enable password. A default configuration is present. ASAv goes through a double-boot before becoming active. This is normal and expected. Switch to the Telnet console type after the first boot.", "symbol": ":/symbols/asa.svg", "first_port_name": "Management0/0", "port_name_format": "Gi0/{0}", @@ -26,6 +26,13 @@ "kvm": "require" }, "images": [ + { + "filename": "asav9-22-1-1.qcow2", + "version": "9.22.1.1 CML", + "md5sum": "250a924cdc2370208eaac9d1dc8dc9e3", + "filesize": 379518976, + "download_url": "https://learningnetworkstore.cisco.com/cisco-modeling-labs-personal/cisco-modeling-labs-personal/CML-PERSONAL.html" + }, { "filename": "asav9-18-2.qcow2", "version": "9.18.2 CML", @@ -126,6 +133,12 @@ } ], "versions": [ + { + "name": "9.22.1.1 CML", + "images": { + "hda_disk_image": "asav9-22-1-1.qcow2" + } + }, { "name": "9.18.2 CML", "images": { diff --git a/gns3server/appliances/cisco-iou-l2.gns3a b/gns3server/appliances/cisco-iou-l2.gns3a index 546cc36c..031a608a 100644 --- a/gns3server/appliances/cisco-iou-l2.gns3a +++ b/gns3server/appliances/cisco-iou-l2.gns3a @@ -13,11 +13,17 @@ "iou": { "ethernet_adapters": 4, "serial_adapters": 0, - "nvram": 128, - "ram": 256, + "nvram": 512, + "ram": 512, "startup_config": "iou_l2_base_startup-config.txt" }, "images": [ + { + "filename": "x86_64_crb_linux_l2-adventerprisek9-ms.iol", + "version": "17.15.1", + "md5sum": "6c587cdfd5056078e70b3f6c26800d66", + "filesize": 243251976 + }, { "filename": "x86_64_crb_linux_l2-adventerprisek9-ms.bin", "version": "17.12.1", @@ -44,6 +50,12 @@ } ], "versions": [ + { + "name": "17.15.1", + "images": { + "image": "x86_64_crb_linux_l2-adventerprisek9-ms.iol" + } + }, { "name": "17.12.1", "images": { diff --git a/gns3server/appliances/cisco-iou-l3.gns3a b/gns3server/appliances/cisco-iou-l3.gns3a index f7328449..296fd05d 100644 --- a/gns3server/appliances/cisco-iou-l3.gns3a +++ b/gns3server/appliances/cisco-iou-l3.gns3a @@ -13,11 +13,17 @@ "iou": { "ethernet_adapters": 2, "serial_adapters": 2, - "nvram": 128, - "ram": 256, + "nvram": 512, + "ram": 512, "startup_config": "iou_l3_base_startup-config.txt" }, "images": [ + { + "filename": "x86_64_crb_linux-adventerprisek9-ms.iol", + "version": "17.15.1", + "md5sum": "5d584f6cfbeaadc87d55f613da1049ed", + "filesize": 292001512 + }, { "filename": "x86_64_crb_linux-adventerprisek9-ms.bin", "version": "17.12.1", @@ -44,6 +50,12 @@ } ], "versions": [ + { + "name": "17.15.1", + "images": { + "image": "x86_64_crb_linux-adventerprisek9-ms.iol" + } + }, { "name": "17.12.1", "images": { diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index c1da4586..2d91069a 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -595,7 +595,7 @@ class Project: if node_type == "iou": async with self._iou_id_lock: - # wait for a IOU node to be completely created before adding a new one + # wait for an IOU node to be completely created before adding a new one # this is important otherwise we allocate the same application ID (used # to generate MAC addresses) when creating multiple IOU node at the same time if "properties" in kwargs.keys(): @@ -1352,7 +1352,10 @@ class Project: p_work = pathlib.Path(self.path).parent.absolute() t0 = time.time() new_project_id = str(uuid.uuid4()) - new_project_path = p_work.joinpath(new_project_id) + if location: + new_project_path = p_work.joinpath(location) + else: + new_project_path = p_work.joinpath(new_project_id) # copy dir await wait_run_in_executor(shutil.copytree, self.path, new_project_path.as_posix(), symlinks=True, ignore_dangling_symlinks=True) log.info("Project content copied from '{}' to '{}' in {}s".format(self.path, new_project_path, time.time() - t0)) diff --git a/gns3server/utils/images.py b/gns3server/utils/images.py index 2223ddcf..f91041ab 100644 --- a/gns3server/utils/images.py +++ b/gns3server/utils/images.py @@ -62,42 +62,53 @@ async def list_images(image_type): directory = os.path.normpath(directory) for root, _, filenames in _os_walk(directory, recurse=recurse): for filename in filenames: - if filename not in files: - if filename.endswith(".md5sum") or filename.startswith("."): + if filename in files: + log.debug("File {} has already been found, skipping...".format(filename)) + continue + if filename.endswith(".md5sum") or filename.startswith("."): + continue + + files.add(filename) + + filesize = os.stat(os.path.join(root, filename)).st_size + if filesize < 7: + log.debug("File {} is too small to be an image, skipping...".format(filename)) + continue + + try: + with open(os.path.join(root, filename), "rb") as f: + # read the first 7 bytes of the file. + elf_header_start = f.read(7) + if image_type == "dynamips" and elf_header_start != b'\x7fELF\x01\x02\x01': + # IOS images must start with the ELF magic number, be 32-bit, big endian and have an ELF version of 1 + log.warning("IOS image {} does not start with a valid ELF magic number, skipping...".format(filename)) + continue + elif image_type == "iou" and elf_header_start != b'\x7fELF\x02\x01\x01' and elf_header_start != b'\x7fELF\x01\x01\x01': + # IOU images must start with the ELF magic number, be 32-bit or 64-bit, little endian and have an ELF version of 1 + log.warning("IOU image {} does not start with a valid ELF magic number, skipping...".format(filename)) + continue + elif image_type == "qemu" and elf_header_start[:4] == b'\x7fELF': + # QEMU images should not start with an ELF magic number + log.warning("QEMU image {} starts with an ELF magic number, skipping...".format(filename)) continue - elif ( - ((filename.endswith(".image") or filename.endswith(".bin")) and image_type == "dynamips") - or ((filename.endswith(".bin") or filename.startswith("i86bi")) and image_type == "iou") - or (not filename.endswith(".bin") and not filename.endswith(".image") and image_type == "qemu") - ): - files.add(filename) - # It the image is located in the standard directory the path is relative - if os.path.commonprefix([root, default_directory]) != default_directory: - path = os.path.join(root, filename) - else: - path = os.path.relpath(os.path.join(root, filename), default_directory) + # It the image is located in the standard directory the path is relative + if os.path.commonprefix([root, default_directory]) != default_directory: + path = os.path.join(root, filename) + else: + path = os.path.relpath(os.path.join(root, filename), default_directory) - try: - if image_type in ["dynamips", "iou"]: - with open(os.path.join(root, filename), "rb") as f: - # read the first 7 bytes of the file. - elf_header_start = f.read(7) - # valid IOU or IOS images must start with the ELF magic number, be 32-bit or 64-bit, - # little endian and have an ELF version of 1 - if elf_header_start != b'\x7fELF\x02\x01\x01' and elf_header_start != b'\x7fELF\x01\x01\x01': - continue - images.append( - { - "filename": filename, - "path": force_unix_path(path), - "md5sum": await wait_run_in_executor(md5sum, os.path.join(root, filename)), - "filesize": os.stat(os.path.join(root, filename)).st_size, + images.append( + { + "filename": filename, + "path": force_unix_path(path), + "md5sum": await wait_run_in_executor(md5sum, os.path.join(root, filename)), + "filesize": filesize, } ) - except OSError as e: - log.warning(f"Can't add image {path}: {str(e)}") + except OSError as e: + log.warning(f"Can't add image {path}: {str(e)}") return images @@ -155,6 +166,23 @@ async def discover_images(image_type: str, skip_image_paths: list = None) -> Lis except InvalidImageError as e: log.debug(str(e)) continue + # It the image is located in the standard directory the path is relative + if os.path.commonprefix([root, default_directory]) != default_directory: + path = os.path.join(root, filename) + else: + path = os.path.relpath(os.path.join(root, filename), default_directory) + + images.append( + { + "filename": filename, + "path": force_unix_path(path), + "md5sum": md5sum(os.path.join(root, filename)), + "filesize": filesize + } + ) + + except OSError as e: + log.warning("Can't add image {}: {}".format(path, str(e))) return images diff --git a/tests/api/routes/compute/test_dynamips_nodes.py b/tests/api/routes/compute/test_dynamips_nodes.py index 7f430a53..19bc691c 100644 --- a/tests/api/routes/compute/test_dynamips_nodes.py +++ b/tests/api/routes/compute/test_dynamips_nodes.py @@ -146,7 +146,7 @@ def fake_image(tmpdir) -> str: path = str(tmpdir / "7200.bin") with open(path, "wb+") as f: - f.write(b'\x7fELF\x01\x01\x01') + f.write(b'\x7fELF\x01\x02\x01') os.chmod(path, stat.S_IREAD) return path @@ -170,7 +170,7 @@ async def test_images(app: FastAPI, compute_client: AsyncClient, tmpdir, fake_im assert response.json() == [{"filename": "7200.bin", "path": "7200.bin", "filesize": 7, - "md5sum": "e573e8f5c93c6c00783f20c7a170aa6c"}] + "md5sum": "b0d5aa897d937aced5a6b1046e8f7e2e"}] async def test_upload_image(app: FastAPI, compute_client: AsyncClient, images_dir: str) -> None: diff --git a/tests/compute/test_manager.py b/tests/compute/test_manager.py index e92a82ee..0a2a8123 100644 --- a/tests/compute/test_manager.py +++ b/tests/compute/test_manager.py @@ -217,12 +217,12 @@ async def test_list_images(qemu, tmpdir): os.makedirs(tmp_images_dir, exist_ok=True) for image in fake_images: with open(os.path.join(tmp_images_dir, image), "w+") as f: - f.write("1") + f.write("1234567") with patch("gns3server.utils.images.default_images_directory", return_value=str(tmp_images_dir)): assert sorted(await qemu.list_images(), key=lambda k: k['filename']) == [ - {"filename": "a.qcow2", "path": "a.qcow2", "md5sum": "c4ca4238a0b923820dcc509a6f75849b", "filesize": 1}, - {"filename": "b.qcow2", "path": "b.qcow2", "md5sum": "c4ca4238a0b923820dcc509a6f75849b", "filesize": 1} + {"filename": "a.qcow2", "path": "a.qcow2", "md5sum": "fcea920f7412b5da7be0cf42b8c93759", "filesize": 7}, + {"filename": "b.qcow2", "path": "b.qcow2", "md5sum": "fcea920f7412b5da7be0cf42b8c93759", "filesize": 7} ] @@ -234,19 +234,19 @@ async def test_list_images_recursives(qemu, tmpdir): fake_images = ["a.qcow2", "b.qcow2", ".blu.qcow2", "a.qcow2.md5sum"] for image in fake_images: with open(os.path.join(tmp_images_dir, image), "w+") as f: - f.write("1") + f.write("1234567") os.makedirs(os.path.join(tmp_images_dir, "c")) fake_images = ["c.qcow2", "c.qcow2.md5sum"] for image in fake_images: with open(os.path.join(tmp_images_dir, "c", image), "w+") as f: - f.write("1") + f.write("1234567") with patch("gns3server.utils.images.default_images_directory", return_value=str(tmp_images_dir)): assert sorted(await qemu.list_images(), key=lambda k: k['filename']) == [ - {"filename": "a.qcow2", "path": "a.qcow2", "md5sum": "c4ca4238a0b923820dcc509a6f75849b", "filesize": 1}, - {"filename": "b.qcow2", "path": "b.qcow2", "md5sum": "c4ca4238a0b923820dcc509a6f75849b", "filesize": 1}, - {"filename": "c.qcow2", "path": force_unix_path(os.path.sep.join(["c", "c.qcow2"])), "md5sum": "c4ca4238a0b923820dcc509a6f75849b", "filesize": 1} + {"filename": "a.qcow2", "path": "a.qcow2", "md5sum": "fcea920f7412b5da7be0cf42b8c93759", "filesize": 7}, + {"filename": "b.qcow2", "path": "b.qcow2", "md5sum": "fcea920f7412b5da7be0cf42b8c93759", "filesize": 7}, + {"filename": "c.qcow2", "path": force_unix_path(os.path.sep.join(["c", "c.qcow2"])), "md5sum": "fcea920f7412b5da7be0cf42b8c93759", "filesize": 7} ] diff --git a/tests/utils/test_images.py b/tests/utils/test_images.py index e934b91d..21bbbd9a 100644 --- a/tests/utils/test_images.py +++ b/tests/utils/test_images.py @@ -114,64 +114,89 @@ def test_remove_checksum(tmpdir): @pytest.mark.asyncio async def test_list_images(tmpdir, config): - path1 = tmpdir / "images1" / "IOS" / "test1.image" - path1.write(b'\x7fELF\x01\x01\x01', ensure=True) - path1 = force_unix_path(str(path1)) + # IOS image in the images directory + ios_image_1 = tmpdir / "images1" / "IOS" / "ios_image_1.image" + ios_image_1.write(b'\x7fELF\x01\x02\x01', ensure=True) + ios_image_1 = force_unix_path(str(ios_image_1)) - path2 = tmpdir / "images2" / "test2.image" - path2.write(b'\x7fELF\x01\x01\x01', ensure=True) - path2 = force_unix_path(str(path2)) + # IOS image in an additional images path + ios_image_2 = tmpdir / "images2" / "ios_image_2.image" + ios_image_2.write(b'\x7fELF\x01\x02\x01', ensure=True) + ios_image_2 = force_unix_path(str(ios_image_2)) - # Invalid image because not a valid elf file - path = tmpdir / "images2" / "test_invalid.image" - path.write(b'NOTANELF', ensure=True) + # Not a valid elf file + not_elf_file = tmpdir / "images1" / "IOS" / "not_elf.image" + not_elf_file.write(b'NOTANELF', ensure=True) + not_elf_file = force_unix_path(str(not_elf_file)) + + # Invalid image because it is very small + small_file = tmpdir / "images1" / "too_small.image" + small_file.write(b'1', ensure=True) if sys.platform.startswith("linux"): - path3 = tmpdir / "images1" / "IOU" / "test3.bin" - path3.write(b'\x7fELF\x02\x01\x01', ensure=True) - path3 = force_unix_path(str(path3)) + # 64-bit IOU image + iou_image_1 = tmpdir / "images1" / "IOU" / "iou64.bin" + iou_image_1.write(b'\x7fELF\x02\x01\x01', ensure=True) + iou_image_1 = force_unix_path(str(iou_image_1)) + # 32-bit IOU image + iou_image_2 = tmpdir / "images1" / "IOU" / "iou32.bin" + iou_image_2.write(b'\x7fELF\x01\x01\x01', ensure=True) # 32-bit IOU image + iou_image_2 = force_unix_path(str(iou_image_2)) - path4 = tmpdir / "images1" / "QEMU" / "test4.qcow2" - path4.write("1", ensure=True) - path4 = force_unix_path(str(path4)) - path5 = tmpdir / "images1" / "QEMU" / "test4.qcow2.md5sum" - path5.write("1", ensure=True) - path5 = force_unix_path(str(path5)) + # Qemu image + qemu_image_1 = tmpdir / "images1" / "QEMU" / "qemu_image.qcow2" + qemu_image_1.write("1234567", ensure=True) + qemu_image_1 = force_unix_path(str(qemu_image_1)) + + # ELF file inside the Qemu + elf_file = tmpdir / "images1" / "QEMU" / "elf_file.bin" + elf_file.write(b'\x7fELF\x02\x01\x01', ensure=True) # ELF file + elf_file = force_unix_path(str(elf_file)) + + md5sum_file = tmpdir / "images1" / "QEMU" / "image.qcow2.md5sum" + md5sum_file.write("1", ensure=True) + md5sum_file = force_unix_path(str(md5sum_file)) config.settings.Server.images_path = str(tmpdir / "images1") config.settings.Server.additional_images_paths = "/tmp/null24564;" + str(tmpdir / "images2") - assert await list_images("dynamips") == [ + assert list_images("dynamips") == [ { - 'filename': 'test1.image', + 'filename': 'ios_image_1.image', 'filesize': 7, - 'md5sum': 'e573e8f5c93c6c00783f20c7a170aa6c', - 'path': 'test1.image' + 'md5sum': 'b0d5aa897d937aced5a6b1046e8f7e2e', + 'path': 'ios_image_1.image' }, { - 'filename': 'test2.image', + 'filename': 'ios_image_2.image', 'filesize': 7, - 'md5sum': 'e573e8f5c93c6c00783f20c7a170aa6c', - 'path': str(path2) + 'md5sum': 'b0d5aa897d937aced5a6b1046e8f7e2e', + 'path': str(ios_image_2) } ] if sys.platform.startswith("linux"): - assert await list_images("iou") == [ + assert list_images("iou") == [ { - 'filename': 'test3.bin', + 'filename': 'iou64.bin', 'filesize': 7, 'md5sum': 'c73626d23469519894d58bc98bee9655', - 'path': 'test3.bin' + 'path': 'iou64.bin' + }, + { + 'filename': 'iou32.bin', + 'filesize': 7, + 'md5sum': 'e573e8f5c93c6c00783f20c7a170aa6c', + 'path': 'iou32.bin' } ] - assert await list_images("qemu") == [ + assert list_images("qemu") == [ { - 'filename': 'test4.qcow2', - 'filesize': 1, - 'md5sum': 'c4ca4238a0b923820dcc509a6f75849b', - 'path': 'test4.qcow2' + 'filename': 'qemu_image.qcow2', + 'filesize': 7, + 'md5sum': 'fcea920f7412b5da7be0cf42b8c93759', + 'path': 'qemu_image.qcow2' } ]