From ccb6e955b4f113b0ae4c68ad1f0d1ba07f8187be Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Tue, 10 Nov 2015 14:56:58 +0100 Subject: [PATCH] Fix check.py with feedbacks from @ehlers Fix #43 --- README.rst | 5 +++-- appliances/hp-vsr1001.gns3a | 12 ++++++------ check.py | 39 ++++++++++++++++++++++++------------- 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/README.rst b/README.rst index 6291065..d5ffca0 100644 --- a/README.rst +++ b/README.rst @@ -31,7 +31,7 @@ Tools All tools require python3 and the installation of dependencies via: -.. code:: bash +.. code:: bash pip3 install -r requirements.txt @@ -40,9 +40,10 @@ Check appliance files ----------------------- .. code:: bash - + python3 check.py +You need to install `imagemagick` before running check.py. Create a new appliance ----------------------- diff --git a/appliances/hp-vsr1001.gns3a b/appliances/hp-vsr1001.gns3a index a3d4a6e..3609f56 100644 --- a/appliances/hp-vsr1001.gns3a +++ b/appliances/hp-vsr1001.gns3a @@ -2,17 +2,17 @@ "name": "HP VSR1001", "category": "router", "description": "The HP VSR1000 Virtual Services Router Series is a software application, running on a server, which provides functionality similar to that of a physical router: robust routing between networked devices using a number of popular routing protocols. It also delivers the critical network services associated with today's enterprise routers such as VPN gateway, firewall and other security and traffic management functions.\n\nThe virtual services router (VSR) application runs on a hypervisor on the server, and supports VMware vSphere and Linux KVM hypervisors. From one to eight virtual CPUs are supported, depending on license.\n\nBecause the VSR1000 Series application runs the same HP Comware version 7 operating system as HP switches and routers, it enables significant operational savings. And being virtual, additional agility and ease of deployment is realized, as resources on the VSR can be dynamically allocated and upgraded upon demand as performance requirements grow.\n\nA variety of deployment models are supported including enterprise branch CPE routing, and cloud offload for small to medium workloads.", - "vendor_name": "HP", - "vendor_url": "http://www.hp.com", - "documentation_url": "http://h20564.www2.hp.com/portal/site/hpsc/public/psi/home/?sp4ts.oid=5443163", + "vendor_name": "HP", + "vendor_url": "http://www.hp.com", + "documentation_url": "http://h20195.www2.hp.com/v2/default.aspx?cc=us&lc=en&oid=5443878", "product_name": "VSR1001", - "product_url": "http://www8.hp.com/us/en/products/networking-routers/product-detail.html?oid=5443163", - "registry_version": 1, + "product_url": "http://www8.hp.com/us/en/products/networking-routers/product-detail.html?oid=5443878", + "registry_version": 1, "status": "stable", "maintainer": "GNS3 Team", "maintainer_email": "developers@gns3.net", "usage": "At first boot the router will be installed from the cdrom.", - + "qemu": { "adapter_type": "e1000", "adapters": 16, diff --git a/check.py b/check.py index e10ceba..bc7d8fc 100644 --- a/check.py +++ b/check.py @@ -19,6 +19,7 @@ import os import jsonschema import json import sys +import socket import subprocess import urllib.request from multiprocessing import Pool @@ -30,18 +31,19 @@ class MyHTTPRedirectHandler(urllib.request.HTTPRedirectHandler): urllib.request.install_opener(urllib.request.build_opener(MyHTTPRedirectHandler)) -def check_url(url, appliance): +def check_url(args): + url, appliance = args try: print("Check " + url) req = urllib.request.Request(url, method='HEAD') - urllib.request.urlopen(req, 5) + urllib.request.urlopen(req, timeout=20) except urllib.error.HTTPError as err: if err.getcode() >= 400: - print('Error with url ' + url + ' - ' + str(err)) - sys.exit(1) - except urllib.error.URLError: - print('Invalid URL ' + url) - sys.exit(1) + raise Exception('Error with url ' + url + ' - ' + str(err)) + except urllib.error.URLError as err: + raise Exception('Invalid URL ' + url) + except socket.timeout as e: + raise Exception('Timeout URL ' + url) def check_appliance(appliance): @@ -83,19 +85,21 @@ def check_urls(pool, appliance): with open(os.path.join('appliances', appliance)) as f: appliance_json = json.load(f) + calls = [] + for image in appliance_json['images']: if 'direct_download_url' in image: - pool.apply_async(check_url, [image['direct_download_url'], appliance]) + calls.append((image['direct_download_url'], appliance)) if 'download_url' in image: - pool.apply_async(check_url, [image['download_url'], appliance]) + calls.append((image['download_url'], appliance)) if 'vendor_url' in appliance_json: - pool.apply_async(check_url, [appliance_json['vendor_url'], appliance]) + calls.append((appliance_json['vendor_url'], appliance)) if 'documentation_url' in appliance_json: - pool.apply_async(check_url, [appliance_json['documentation_url'], appliance]) + calls.append((appliance_json['documentation_url'], appliance)) if 'product_url' in appliance_json: - pool.apply_async(check_url, [appliance_json['product_url'], appliance]) - + calls.append((appliance_json['product_url'], appliance)) + return calls def check_packer(packer): path = os.path.join('packer', packer) @@ -122,11 +126,12 @@ def check_symbol(symbol): def main(): pool = Pool(processes=8) + calls_check_url = [] print("=> Check appliances") for appliance in os.listdir('appliances'): print('Check {}'.format(appliance)) check_appliance(appliance) - check_urls(pool, appliance) + calls_check_url += check_urls(pool, appliance) print("=> Check symbols") for symbol in os.listdir('symbols'): if symbol.endswith('.svg'): @@ -136,8 +141,14 @@ def main(): for packer in os.listdir('packer'): check_packer(packer) print("=> Check URL in appliances") + try: + pool.map_async(check_url, calls_check_url).get() + except Exception as e: + print(e) + sys.exit(1) pool.close() pool.join() + print("Everything is ok!") if __name__ == '__main__': main()