From ef8c6b603c4f9b248058c987dc25a3222b87b58f Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Mon, 24 Nov 2014 14:32:06 -0700 Subject: [PATCH 1/9] Add debugging to deadman process --- gns3server/modules/deadman/__init__.py | 2 +- gns3server/start_server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gns3server/modules/deadman/__init__.py b/gns3server/modules/deadman/__init__.py index c5619c96..3ea22783 100644 --- a/gns3server/modules/deadman/__init__.py +++ b/gns3server/modules/deadman/__init__.py @@ -83,6 +83,7 @@ class DeadMan(IModule): cmd.append("--file") cmd.append("%s" % (self._heartbeat_file)) cmd.append("--background") + cmd.append("--debug") log.info("Deadman: Running command: %s"%(cmd)) process = subprocess.Popen(cmd, stderr=subprocess.STDOUT, shell=False) @@ -94,7 +95,6 @@ class DeadMan(IModule): """ cmd = [] - cmd.append("gns3dms") cmd.append("-k") log.info("Deadman: Running command: %s"%(cmd)) diff --git a/gns3server/start_server.py b/gns3server/start_server.py index 79fb8924..c32ed3ca 100644 --- a/gns3server/start_server.py +++ b/gns3server/start_server.py @@ -44,7 +44,7 @@ import uuid SCRIPT_NAME = os.path.basename(__file__) -#Is the full path when used as an import +# This is the full path when used as an import SCRIPT_PATH = os.path.dirname(__file__) if not SCRIPT_PATH: From cfdff773c4447eeab44dbf564b10f8b70d4060d4 Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Mon, 24 Nov 2014 15:08:58 -0700 Subject: [PATCH 2/9] deadtime -> dead_time --- gns3dms/main.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/gns3dms/main.py b/gns3dms/main.py index 3fadb32e..40eb5f95 100644 --- a/gns3dms/main.py +++ b/gns3dms/main.py @@ -79,12 +79,12 @@ Options: --instance_id ID of the Rackspace instance to terminate --cloud_region Region of instance - --deadtime How long in seconds can the communication lose exist before we + --dead_time How long in seconds can the communication lose exist before we shutdown this instance. Default: - Example --deadtime=3600 (60 minutes) + Example --dead_time=3600 (60 minutes) - --check-interval Defaults to --deadtime, used for debugging + --check-interval Defaults to --dead_time, used for debugging --init-wait Inital wait time, how long before we start pulling the file. Default: 300 (5 min) @@ -113,7 +113,7 @@ def parse_cmd_line(argv): "cloud_api_key=", "instance_id=", "region=", - "deadtime=", + "dead_time=", "init-wait=", "check-interval=", "file=", @@ -133,7 +133,7 @@ def parse_cmd_line(argv): cmd_line_option_list["cloud_api_key"] = None cmd_line_option_list["instance_id"] = None cmd_line_option_list["region"] = None - cmd_line_option_list["deadtime"] = 60 * 60 #minutes + cmd_line_option_list["dead_time"] = 60 * 60 #minutes cmd_line_option_list["check-interval"] = None cmd_line_option_list["init-wait"] = 5 * 60 cmd_line_option_list["file"] = None @@ -167,8 +167,8 @@ def parse_cmd_line(argv): cmd_line_option_list["instance_id"] = val elif (opt in ("--region")): cmd_line_option_list["region"] = val - elif (opt in ("--deadtime")): - cmd_line_option_list["deadtime"] = int(val) + elif (opt in ("--dead_time")): + cmd_line_option_list["dead_time"] = int(val) elif (opt in ("--check-interval")): cmd_line_option_list["check-interval"] = int(val) elif (opt in ("--init-wait")): @@ -183,7 +183,7 @@ def parse_cmd_line(argv): if cmd_line_option_list["shutdown"] == False: if cmd_line_option_list["check-interval"] is None: - cmd_line_option_list["check-interval"] = cmd_line_option_list["deadtime"] + 120 + cmd_line_option_list["check-interval"] = cmd_line_option_list["dead_time"] + 120 if cmd_line_option_list["cloud_user_name"] is None: print("You need to specify a username!!!!") @@ -317,9 +317,9 @@ def monitor_loop(options): delta = now - file_last_modified log.debug("File last updated: %s seconds ago" % (delta.seconds)) - if delta.seconds > options["deadtime"]: - log.warning("Deadtime exceeded, terminating instance ...") - #Terminate involes many layers of HTTP / API calls, lots of + if delta.seconds > options["dead_time"]: + log.warning("Dead time exceeded, terminating instance ...") + #Terminate involves many layers of HTTP / API calls, lots of #different errors types could occur here. try: rksp = Rackspace(options) From b9a4531544ce40127584b1d2be23d984cdad32ea Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Mon, 24 Nov 2014 15:35:09 -0700 Subject: [PATCH 3/9] Ensuring dead_time is an integer --- gns3dms/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gns3dms/main.py b/gns3dms/main.py index 40eb5f95..6cdad64e 100644 --- a/gns3dms/main.py +++ b/gns3dms/main.py @@ -150,6 +150,7 @@ def parse_cmd_line(argv): get_gns3secrets(cmd_line_option_list) + cmd_line_option_list["dead_time"] = int(cmd_line_option_list["dead_time"]) for opt, val in opts: if (opt in ("-h", "--help")): From 7ef8dd794a91e17c09c45336ea07c5ac7e8a5a46 Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Mon, 24 Nov 2014 16:24:57 -0700 Subject: [PATCH 4/9] Copy updated cloud/ module from gns3-gui --- gns3dms/cloud/base_cloud_ctrl.py | 70 ++++++++++++++++++++++++++++---- gns3dms/cloud/rackspace_ctrl.py | 56 +------------------------ 2 files changed, 63 insertions(+), 63 deletions(-) diff --git a/gns3dms/cloud/base_cloud_ctrl.py b/gns3dms/cloud/base_cloud_ctrl.py index b9335aa8..236cdccc 100644 --- a/gns3dms/cloud/base_cloud_ctrl.py +++ b/gns3dms/cloud/base_cloud_ctrl.py @@ -29,7 +29,7 @@ import logging from io import StringIO, BytesIO from libcloud.compute.base import NodeAuthSSHKey -from libcloud.storage.types import ContainerAlreadyExistsError, ContainerDoesNotExistError +from libcloud.storage.types import ContainerAlreadyExistsError, ContainerDoesNotExistError, ObjectDoesNotExistError from .exceptions import ItemNotFound, KeyPairExists, MethodNotAllowed from .exceptions import OverLimit, BadRequest, ServiceUnavailable @@ -216,11 +216,11 @@ class BaseCloudCtrl(object): return self.driver.list_key_pairs() - def upload_file(self, file_path, folder): + def upload_file(self, file_path, cloud_object_name): """ Uploads file to cloud storage (if it is not identical to a file already in cloud storage). :param file_path: path to file to upload - :param folder: folder in cloud storage to save file in + :param cloud_object_name: name of file saved in cloud storage :return: True if file was uploaded, False if it was skipped because it already existed and was identical """ try: @@ -231,7 +231,6 @@ class BaseCloudCtrl(object): with open(file_path, 'rb') as file: local_file_hash = hashlib.md5(file.read()).hexdigest() - cloud_object_name = folder + '/' + os.path.basename(file_path) cloud_hash_name = cloud_object_name + '.md5' cloud_objects = [obj.name for obj in gns3_container.list_objects()] @@ -254,23 +253,24 @@ class BaseCloudCtrl(object): def list_projects(self): """ Lists projects in cloud storage - :return: List of (project name, object name in storage) + :return: Dictionary where project names are keys and values are names of objects in storage """ try: gns3_container = self.storage_driver.get_container(self.GNS3_CONTAINER_NAME) - projects = [ - (obj.name.replace('projects/', '').replace('.zip', ''), obj.name) + projects = { + obj.name.replace('projects/', '').replace('.zip', ''): obj.name for obj in gns3_container.list_objects() if obj.name.startswith('projects/') and obj.name[-4:] == '.zip' - ] + } return projects except ContainerDoesNotExistError: return [] def download_file(self, file_name, destination=None): """ - Downloads file from cloud storage + Downloads file from cloud storage. If a file exists at destination, and it is identical to the file in cloud + storage, it is not downloaded. :param file_name: name of file in cloud storage to download :param destination: local path to save file to (if None, returns file contents as a file-like object) :return: A file-like object if file contents are returned, or None if file is saved to filesystem @@ -278,7 +278,22 @@ class BaseCloudCtrl(object): gns3_container = self.storage_driver.get_container(self.GNS3_CONTAINER_NAME) storage_object = gns3_container.get_object(file_name) + if destination is not None: + if os.path.isfile(destination): + # if a file exists at destination and its hash matches that of the + # file in cloud storage, don't download it + with open(destination, 'rb') as f: + local_file_hash = hashlib.md5(f.read()).hexdigest() + + hash_object = gns3_container.get_object(file_name + '.md5') + cloud_object_hash = '' + for chunk in hash_object.as_stream(): + cloud_object_hash += chunk.decode('utf8') + + if local_file_hash == cloud_object_hash: + return + storage_object.download(destination) else: contents = b'' @@ -287,3 +302,40 @@ class BaseCloudCtrl(object): contents += chunk return BytesIO(contents) + + def find_storage_image_names(self, images_to_find): + """ + Maps names of image files to their full name in cloud storage + :param images_to_find: list of image names to find + :return: A dictionary where keys are image names, and values are the corresponding names of + the files in cloud storage + """ + gns3_container = self.storage_driver.get_container(self.GNS3_CONTAINER_NAME) + images_in_storage = [obj.name for obj in gns3_container.list_objects() if obj.name.startswith('images/')] + + images = {} + for image_name in images_to_find: + images_with_same_name =\ + list(filter(lambda storage_image_name: storage_image_name.endswith(image_name), images_in_storage)) + + if len(images_with_same_name) == 1: + images[image_name] = images_with_same_name[0] + else: + raise Exception('Image does not exist in cloud storage or is duplicated') + + return images + + def delete_file(self, file_name): + gns3_container = self.storage_driver.get_container(self.GNS3_CONTAINER_NAME) + + try: + object_to_delete = gns3_container.get_object(file_name) + object_to_delete.delete() + except ObjectDoesNotExistError: + pass + + try: + hash_object = gns3_container.get_object(file_name + '.md5') + hash_object.delete() + except ObjectDoesNotExistError: + pass diff --git a/gns3dms/cloud/rackspace_ctrl.py b/gns3dms/cloud/rackspace_ctrl.py index 455f87ba..aee7f46d 100644 --- a/gns3dms/cloud/rackspace_ctrl.py +++ b/gns3dms/cloud/rackspace_ctrl.py @@ -42,11 +42,9 @@ class RackspaceCtrl(BaseCloudCtrl): """ Controller class for interacting with Rackspace API. """ - def __init__(self, username, api_key, gns3_ias_url): + def __init__(self, username, api_key, *args, **kwargs): super(RackspaceCtrl, self).__init__(username, api_key) - self.gns3_ias_url = gns3_ias_url - # set this up so it can be swapped out with a mock for testing self.post_fn = requests.post self.driver_cls = get_driver(Provider.RACKSPACE) @@ -225,55 +223,6 @@ class RackspaceCtrl(BaseCloudCtrl): self.region = region return True - def _get_shared_images(self, username, region, gns3_version): - """ - Given a GNS3 version, ask gns3-ias to share compatible images - - Response: - [{"created_at": "", "schema": "", "status": "", "member_id": "", "image_id": "", "updated_at": ""},] - or, if access was already asked - [{"image_id": "", "member_id": "", "status": "ALREADYREQUESTED"},] - """ - endpoint = self.gns3_ias_url+"/images/grant_access" - params = { - "user_id": username, - "user_region": region.upper(), - "gns3_version": gns3_version, - } - try: - response = requests.get(endpoint, params=params) - except requests.ConnectionError: - raise ApiError("Unable to connect to IAS") - - status = response.status_code - - if status == 200: - return response.json() - elif status == 404: - raise ItemNotFound() - else: - raise ApiError("IAS status code: %d" % status) - - def list_images(self): - """ - Return a dictionary containing RackSpace server images - retrieved from gns3-ias server - """ - if not (self.tenant_id and self.region): - return {} - - try: - shared_images = self._get_shared_images(self.tenant_id, self.region, __version__) - images = {} - for i in shared_images: - images[i['image_id']] = i['image_name'] - return images - except ItemNotFound: - return {} - except ApiError as e: - log.error('Error while retrieving image list: %s' % e) - return {} - def get_image(self, image_id): return self.driver.get_image(image_id) @@ -290,12 +239,11 @@ def get_provider(cloud_settings): username = cloud_settings['cloud_user_name'] apikey = cloud_settings['cloud_api_key'] region = cloud_settings['cloud_region'] - ias_url = cloud_settings.get('gns3_ias_url', '') except KeyError as e: log.error("Unable to create cloud provider: {}".format(e)) return - provider = RackspaceCtrl(username, apikey, ias_url) + provider = RackspaceCtrl(username, apikey) if not provider.authenticate(): log.error("Authentication failed for cloud provider") From b220cffcf447092e2531585251ac879e2c10fbce Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Mon, 24 Nov 2014 16:40:55 -0700 Subject: [PATCH 5/9] Updating dms to look for cloud_region instead of region --- gns3dms/modules/rackspace_cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gns3dms/modules/rackspace_cloud.py b/gns3dms/modules/rackspace_cloud.py index 00059047..868f95ed 100644 --- a/gns3dms/modules/rackspace_cloud.py +++ b/gns3dms/modules/rackspace_cloud.py @@ -41,7 +41,7 @@ class Rackspace(object): self.authenticated = False self.hostname = socket.gethostname() self.instance_id = options["instance_id"] - self.region = options["region"] + self.region = options["cloud_region"] log.debug("Authenticating with Rackspace") log.debug("My hostname: %s" % (self.hostname)) From 9ec1373d36948283cceab8a340187bbbe13e75e4 Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Mon, 24 Nov 2014 16:49:01 -0700 Subject: [PATCH 6/9] dms terminates instance if id matches --- gns3dms/modules/rackspace_cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gns3dms/modules/rackspace_cloud.py b/gns3dms/modules/rackspace_cloud.py index 868f95ed..487e6f9f 100644 --- a/gns3dms/modules/rackspace_cloud.py +++ b/gns3dms/modules/rackspace_cloud.py @@ -59,7 +59,7 @@ class Rackspace(object): self.rksp.set_region(self.region) for server in self.rksp.list_instances(): log.debug("Checking server: %s" % (server.name)) - if server.name.lower() == self.hostname.lower() and server.id == self.instance_id: + if server.id == self.instance_id: log.info("Found matching instance: %s" % (server.id)) log.info("Startup id: %s" % (self.instance_id)) return server From f1503ae655b13fda99336c4c499ae3cff1f2c4d7 Mon Sep 17 00:00:00 2001 From: Jerry Seutter Date: Mon, 8 Dec 2014 20:14:55 -0700 Subject: [PATCH 7/9] Handle initrd and kernel paths in qemu --- gns3server/modules/qemu/qemu_vm.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/gns3server/modules/qemu/qemu_vm.py b/gns3server/modules/qemu/qemu_vm.py index 57a5b42d..45fa483c 100644 --- a/gns3server/modules/qemu/qemu_vm.py +++ b/gns3server/modules/qemu/qemu_vm.py @@ -761,6 +761,29 @@ class QemuVM(object): log.debug("Download of {} complete.".format(src)) self.hdb_disk_image = dst + if self.initrd != "": + _, filename = ntpath.split(self.initrd) + src = '{}/{}'.format(self.cloud_path, filename) + dst = os.path.join(self.working_dir, filename) + if not os.path.isfile(dst): + cloud_settings = Config.instance().cloud_settings() + provider = get_provider(cloud_settings) + log.debug("Downloading file from {} to {}...".format(src, dst)) + provider.download_file(src, dst) + log.debug("Download of {} complete.".format(src)) + self.initrd = dst + if self.kernel_image != "": + _, filename = ntpath.split(self.kernel_image) + src = '{}/{}'.format(self.cloud_path, filename) + dst = os.path.join(self.working_dir, filename) + if not os.path.isfile(dst): + cloud_settings = Config.instance().cloud_settings() + provider = get_provider(cloud_settings) + log.debug("Downloading file from {} to {}...".format(src, dst)) + provider.download_file(src, dst) + log.debug("Download of {} complete.".format(src)) + self.kernel_image = dst + self._command = self._build_command() try: log.info("starting QEMU: {}".format(self._command)) From 6a81390808ae645688b260248f62d4356f5f2e32 Mon Sep 17 00:00:00 2001 From: Daniel Lintott Date: Tue, 9 Dec 2014 13:49:45 +0000 Subject: [PATCH 8/9] Update test with correct hypervisor IP --- tests/dynamips/test_hypervisor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dynamips/test_hypervisor.py b/tests/dynamips/test_hypervisor.py index e6c096b8..81a8176e 100644 --- a/tests/dynamips/test_hypervisor.py +++ b/tests/dynamips/test_hypervisor.py @@ -14,7 +14,7 @@ def test_port(hypervisor): def test_host(hypervisor): - assert hypervisor.host == "127.0.0.1" + assert hypervisor.host == "0.0.0.0" def test_working_dir(hypervisor): From 317a37147ef004cdadb56313aafb39b4e5f272e5 Mon Sep 17 00:00:00 2001 From: Daniel Lintott Date: Tue, 9 Dec 2014 19:39:22 +0000 Subject: [PATCH 9/9] Remove extra line that was over-writing the auto Idle-PC value Fixes GNS3/gns3-gui#158 --- gns3server/modules/dynamips/backends/vm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gns3server/modules/dynamips/backends/vm.py b/gns3server/modules/dynamips/backends/vm.py index e43767c9..187e61d1 100644 --- a/gns3server/modules/dynamips/backends/vm.py +++ b/gns3server/modules/dynamips/backends/vm.py @@ -760,7 +760,6 @@ class VM(object): if was_auto_started: router.stop() - validated_idlepc = "0x0" response = {"id": router.id, "logs": logs, "idlepc": validated_idlepc}