Merge pull request #46 from GNS3/fix_check

Fix check.py with feedbacks from @ehlers

.travis.yml

@@ -9,3 +9,5 @@ addons:
     - imagemagick
 script:
 - python check.py
+- python check_urls.py
+

README.rst

@@ -31,7 +31,7 @@ Tools
 
 All tools require python3 and the installation of dependencies via:
 
-.. code:: bash 
+.. code:: bash
 
     pip3 install -r requirements.txt
 
@@ -40,9 +40,11 @@ Check appliance files
 -----------------------
 
 .. code:: bash
-    
-    python3 check.py
 
+    python3 check.py
+    python3 check_urls.py
+
+You need to install `imagemagick` before running check.py.
 
 Create a new appliance
 -----------------------

appliances/hp-vsr1001.gns3a

@@ -2,17 +2,17 @@
     "name": "HP VSR1001",
     "category": "router",
     "description": "The HP VSR1000 Virtual Services Router Series is a software application, running on a server, which provides functionality similar to that of a physical router: robust routing between networked devices using a number of popular routing protocols. It also delivers the critical network services associated with today's enterprise routers such as VPN gateway, firewall and other security and traffic management functions.\n\nThe virtual services router (VSR) application runs on a hypervisor on the server, and supports VMware vSphere and Linux KVM hypervisors. From one to eight virtual CPUs are supported, depending on license.\n\nBecause the VSR1000 Series application runs the same HP Comware version 7 operating system as HP switches and routers, it enables significant operational savings. And being virtual, additional agility and ease of deployment is realized, as resources on the VSR can be dynamically allocated and upgraded upon demand as performance requirements grow.\n\nA variety of deployment models are supported including enterprise branch CPE routing, and cloud offload for small to medium workloads.",
-    "vendor_name": "HP", 
-    "vendor_url": "http://www.hp.com", 
-    "documentation_url": "http://h20564.www2.hp.com/portal/site/hpsc/public/psi/home/?sp4ts.oid=5443163", 
+    "vendor_name": "HP",
+    "vendor_url": "http://www.hp.com",
+    "documentation_url": "http://h20195.www2.hp.com/v2/default.aspx?cc=us&lc=en&oid=5443878",
     "product_name": "VSR1001",
-    "product_url": "http://www8.hp.com/us/en/products/networking-routers/product-detail.html?oid=5443163", 
-    "registry_version": 1, 
+    "product_url": "http://www8.hp.com/us/en/products/networking-routers/product-detail.html?oid=5443878",
+    "registry_version": 1,
     "status": "stable",
     "maintainer": "GNS3 Team",
     "maintainer_email": "developers@gns3.net",
     "usage": "At first boot the router will be installed from the cdrom.",
- 
+
     "qemu": {
         "adapter_type": "e1000",
         "adapters": 16,

appliances/juniper-vsrx.gns3a

@@ -9,7 +9,7 @@
     "product_url": "https://www.juniper.net/us/en/products-services/security/srx-series/vsrx/",
     "category": "firewall",
     "status": "experimental",
-    "vendor_url": "https://www.juniper.net",
+    "vendor_url": "https://www.juniper.net/us/en/",
     "registry_version": 1,
     "usage": "Initial username is root, no password",
     "port_name_format": "ge-0/0/{0}",

check.py

@@ -20,28 +20,6 @@ import jsonschema
 import json
 import sys
 import subprocess
-import urllib.request
-from multiprocessing import Pool
-
-
-class MyHTTPRedirectHandler(urllib.request.HTTPRedirectHandler):
-    def redirect_request(self, req, fp, code, msg, hdrs, newurl):
-        return None
-
-urllib.request.install_opener(urllib.request.build_opener(MyHTTPRedirectHandler))
-
-def check_url(url, appliance):
-    try:
-        print("Check " + url)
-        req = urllib.request.Request(url, method='HEAD')
-        urllib.request.urlopen(req, 5)
-    except urllib.error.HTTPError as err:
-        if err.getcode() >= 400:
-            print('Error with url ' + url + ' - ' + str(err))
-            sys.exit(1)
-    except urllib.error.URLError:
-        print('Invalid URL ' + url)
-        sys.exit(1)
 
 
 def check_appliance(appliance):
@@ -79,24 +57,6 @@ def check_appliance(appliance):
                 sys.exit(1)
 
 
-def check_urls(pool, appliance):
-    with open(os.path.join('appliances', appliance)) as f:
-        appliance_json = json.load(f)
-
-    for image in appliance_json['images']:
-        if 'direct_download_url' in image:
-            pool.apply_async(check_url, [image['direct_download_url'], appliance])
-        if 'download_url' in image:
-            pool.apply_async(check_url, [image['download_url'], appliance])
-
-    if 'vendor_url' in appliance_json:
-        pool.apply_async(check_url, [appliance_json['vendor_url'], appliance])
-    if 'documentation_url' in appliance_json:
-        pool.apply_async(check_url, [appliance_json['documentation_url'], appliance])
-    if 'product_url' in appliance_json:
-        pool.apply_async(check_url, [appliance_json['product_url'], appliance])
-
-
 def check_packer(packer):
     path = os.path.join('packer', packer)
     if not os.path.isdir(path):
@@ -120,13 +80,10 @@ def check_symbol(symbol):
 
 
 def main():
-    pool = Pool(processes=8)
-
     print("=> Check appliances")
     for appliance in os.listdir('appliances'):
         print('Check {}'.format(appliance))
         check_appliance(appliance)
-        check_urls(pool, appliance)
     print("=> Check symbols")
     for symbol in os.listdir('symbols'):
         if symbol.endswith('.svg'):
@@ -135,9 +92,7 @@ def main():
     print("=> Check packer files")
     for packer in os.listdir('packer'):
         check_packer(packer)
-    print("=> Check URL in appliances")
-    pool.close()
-    pool.join()
+    print("Everything is ok!")
 
 if __name__ == '__main__':
     main()

check_urls.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2015 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import json
+import sys
+import socket
+import time
+import urllib.request
+import http.client
+from multiprocessing import Pool
+
+class CheckError(Exception):
+    def __init__(self, m):
+        self.message = m
+
+    def __str__(self):
+        return self.message
+
+
+class MyHTTPRedirectHandler(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, hdrs, newurl):
+        return None
+
+
+urllib.request.install_opener(urllib.request.build_opener(MyHTTPRedirectHandler))
+
+
+def check_url(args):
+    url, appliance = args
+    print("Check " + url)
+
+    remaining_failure = 5
+    error = None
+    while remaining_failure != 0:
+        try:
+            req = urllib.request.Request(url, method='HEAD')
+            req.add_header
+            urllib.request.urlopen(req, timeout=45) #Yeah a big big timeout for broken websites...
+        except urllib.error.HTTPError as err:
+            if err.getcode() >= 400:
+                error = CheckError('Error with url {} ({})'.format(url, str(err)))
+            else:
+                # We allow error code like 302
+                return
+        except http.client.BadStatusLine as err:
+            error = CheckError('Bad status line {} ({})'.format(url, str(err)))
+        except urllib.error.URLError as err:
+            error = CheckError('Invalid URL {} ({})'.format(url, str(err)))
+        except socket.timeout as err:
+            error = CheckError('Timeout URL {} ({})'.format(url, str(err)))
+        else:
+            return
+        remaining_failure -= 1
+        time.sleep(5)
+    raise error
+
+
+def check_urls(pool, appliance):
+    with open(os.path.join('appliances', appliance)) as f:
+        appliance_json = json.load(f)
+
+    calls = []
+
+    for image in appliance_json['images']:
+        if 'direct_download_url' in image:
+            calls.append((image['direct_download_url'], appliance))
+        if 'download_url' in image:
+            calls.append((image['download_url'], appliance))
+
+    if 'vendor_url' in appliance_json:
+        calls.append((appliance_json['vendor_url'], appliance))
+    if 'documentation_url' in appliance_json:
+        calls.append((appliance_json['documentation_url'], appliance))
+    if 'product_url' in appliance_json:
+        calls.append((appliance_json['product_url'], appliance))
+    return calls
+
+
+def main():
+    pool = Pool(processes=8)
+
+    calls_check_url = []
+    for appliance in os.listdir('appliances'):
+        calls_check_url += check_urls(pool, appliance)
+    print("=> Check URL in appliances")
+    try:
+        pool.map_async(check_url, calls_check_url).get()
+    except CheckError as e:
+        print(e)
+        sys.exit(1)
+    pool.close()
+    pool.join()
+    print("Everything is ok!")
+
+if __name__ == '__main__':
+    main()