From 5958a23557fe8abd120b62e8c7b877666fdf063a Mon Sep 17 00:00:00 2001
From: grossmj <grossmj@gns3.net>
Date: Sun, 19 Aug 2018 16:02:47 +0700
Subject: [PATCH] Update appliance templates from online registry. Ref #2490.

---
 gns3server/controller/__init__.py             | 59 +++++++++++++++++--
 .../api/controller/appliance_handler.py       |  3 +
 2 files changed, 58 insertions(+), 4 deletions(-)

diff --git a/gns3server/controller/__init__.py b/gns3server/controller/__init__.py
index 615d8c64..760aff7c 100644
--- a/gns3server/controller/__init__.py
+++ b/gns3server/controller/__init__.py
@@ -49,6 +49,7 @@ class Controller:
     def __init__(self):
         self._computes = {}
         self._projects = {}
+        self._downloaded_appliance_templates_cache = {}
 
         self._notification = Notification(self)
         self.gns3vm = GNS3VM(self)
@@ -62,12 +63,58 @@ class Controller:
         self._config_file = os.path.join(Config.instance().config_dir, "gns3_controller.conf")
         log.info("Load controller configuration file {}".format(self._config_file))
 
-    def load_appliances(self):
+    @asyncio.coroutine
+    def download_appliance_templates(self):
+
+        session = aiohttp.ClientSession()
+        response = yield from session.get('https://api.github.com/repos/GNS3/gns3-registry/contents/appliances')
+        json_data = yield from response.json()
+        if response.status != 200:
+            raise aiohttp.web.HTTPConflict(text="Could not retrieve appliance templates on GitHub")
+        response.close()
+        try:
+            appliances_dir = get_resource('appliances')
+            for appliance in json_data:
+                if appliance["type"] == "file":
+                    headers = {}
+                    appliance_name = appliance["name"]
+                    #if appliance_name in self._downloaded_appliance_templates_cache:
+                    #    headers["If-None-Match"] = self._downloaded_appliance_templates_cache[appliance_name]
+                    #log.debug("Download appliance template file from '{}'".format(appliance["download_url"]))
+                    response = yield from session.get(appliance["download_url"], headers=headers)
+                    if response.status != 200:
+                        log.debug("Could not download '{}'".format(appliance["download_url"]))
+                        continue
+                    #if resp.status == 304:
+                    #    log.debug("{} is already up-to-date".format(appliance_name))
+                    #    continue
+
+                    try:
+                        appliance_data = yield from response.read()
+                    except asyncio.TimeoutError:
+                        log.warning("Timeout while downloading '{}'".format(appliance["download_url"]))
+                        continue
+                    path = os.path.join(appliances_dir, appliance_name)
+
+                    try:
+                        log.debug("Saving {} file to {}".format(appliance_name, path))
+                        with open(path, 'wb') as f:
+                            f.write(appliance_data)
+                    except OSError as e:
+                        raise aiohttp.web.HTTPConflict(text="Could not write appliance template file '{}': {}".format(path, e))
+
+                    #etag = response.headers.get("ETag")
+                    #if etag:
+                    #    self._downloaded_appliance_templates_cache[appliance_name] = etag
+        except ValueError as e:
+            raise aiohttp.web.HTTPConflict(text="Could not read appliance templates information from GitHub: {}".format(e))
+        finally:
+            session.close()
+
+    def load_appliance_templates(self):
 
         self._appliance_templates = {}
-        for directory, builtin in (
-            (get_resource('appliances'), True,), (self.appliances_path(), False,)
-        ):
+        for directory, builtin in ((get_resource('appliances'), True,), (self.appliances_path(), False,)):
             if os.path.isdir(directory):
                 for file in os.listdir(directory):
                     if not file.endswith('.gns3a') and not file.endswith('.gns3appliance'):
@@ -84,6 +131,8 @@ class Controller:
                         log.warning("Cannot load appliance template file '%s': %s", path, str(e))
                         continue
 
+    def load_appliances(self):
+
         self._appliances = {}
         vms = []
         for vm in self._settings.get("Qemu", {}).get("vms", []):
@@ -291,6 +340,7 @@ class Controller:
         if "gns3vm" in data:
             self.gns3vm.settings = data["gns3vm"]
 
+        self.load_appliance_templates()
         self.load_appliances()
         return data.get("computes", [])
 
@@ -428,6 +478,7 @@ class Controller:
         self._settings = val
         self._settings["modification_uuid"] = str(uuid.uuid4())  # We add a modification id to the settings to help the gui to detect changes
         self.save()
+        self.load_appliance_templates()
         self.load_appliances()
         self.notification.controller_emit("settings.updated", val)
 
diff --git a/gns3server/handlers/api/controller/appliance_handler.py b/gns3server/handlers/api/controller/appliance_handler.py
index b9dbc4cf..f5345593 100644
--- a/gns3server/handlers/api/controller/appliance_handler.py
+++ b/gns3server/handlers/api/controller/appliance_handler.py
@@ -37,6 +37,9 @@ class ApplianceHandler:
     def list_templates(request, response):
 
         controller = Controller.instance()
+        if request.query.get("update", "no") == "yes":
+            yield from controller.download_appliance_templates()
+            controller.load_appliance_templates()
         response.json([c for c in controller.appliance_templates.values()])
 
     @Route.get(