From ccb6e955b4f113b0ae4c68ad1f0d1ba07f8187be Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Tue, 10 Nov 2015 14:56:58 +0100 Subject: [PATCH 1/3] Fix check.py with feedbacks from @ehlers Fix #43 --- README.rst | 5 +++-- appliances/hp-vsr1001.gns3a | 12 ++++++------ check.py | 39 ++++++++++++++++++++++++------------- 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/README.rst b/README.rst index 6291065..d5ffca0 100644 --- a/README.rst +++ b/README.rst @@ -31,7 +31,7 @@ Tools All tools require python3 and the installation of dependencies via: -.. code:: bash +.. code:: bash pip3 install -r requirements.txt @@ -40,9 +40,10 @@ Check appliance files ----------------------- .. code:: bash - + python3 check.py +You need to install `imagemagick` before running check.py. Create a new appliance ----------------------- diff --git a/appliances/hp-vsr1001.gns3a b/appliances/hp-vsr1001.gns3a index a3d4a6e..3609f56 100644 --- a/appliances/hp-vsr1001.gns3a +++ b/appliances/hp-vsr1001.gns3a @@ -2,17 +2,17 @@ "name": "HP VSR1001", "category": "router", "description": "The HP VSR1000 Virtual Services Router Series is a software application, running on a server, which provides functionality similar to that of a physical router: robust routing between networked devices using a number of popular routing protocols. It also delivers the critical network services associated with today's enterprise routers such as VPN gateway, firewall and other security and traffic management functions.\n\nThe virtual services router (VSR) application runs on a hypervisor on the server, and supports VMware vSphere and Linux KVM hypervisors. From one to eight virtual CPUs are supported, depending on license.\n\nBecause the VSR1000 Series application runs the same HP Comware version 7 operating system as HP switches and routers, it enables significant operational savings. And being virtual, additional agility and ease of deployment is realized, as resources on the VSR can be dynamically allocated and upgraded upon demand as performance requirements grow.\n\nA variety of deployment models are supported including enterprise branch CPE routing, and cloud offload for small to medium workloads.", - "vendor_name": "HP", - "vendor_url": "http://www.hp.com", - "documentation_url": "http://h20564.www2.hp.com/portal/site/hpsc/public/psi/home/?sp4ts.oid=5443163", + "vendor_name": "HP", + "vendor_url": "http://www.hp.com", + "documentation_url": "http://h20195.www2.hp.com/v2/default.aspx?cc=us&lc=en&oid=5443878", "product_name": "VSR1001", - "product_url": "http://www8.hp.com/us/en/products/networking-routers/product-detail.html?oid=5443163", - "registry_version": 1, + "product_url": "http://www8.hp.com/us/en/products/networking-routers/product-detail.html?oid=5443878", + "registry_version": 1, "status": "stable", "maintainer": "GNS3 Team", "maintainer_email": "developers@gns3.net", "usage": "At first boot the router will be installed from the cdrom.", - + "qemu": { "adapter_type": "e1000", "adapters": 16, diff --git a/check.py b/check.py index e10ceba..bc7d8fc 100644 --- a/check.py +++ b/check.py @@ -19,6 +19,7 @@ import os import jsonschema import json import sys +import socket import subprocess import urllib.request from multiprocessing import Pool @@ -30,18 +31,19 @@ class MyHTTPRedirectHandler(urllib.request.HTTPRedirectHandler): urllib.request.install_opener(urllib.request.build_opener(MyHTTPRedirectHandler)) -def check_url(url, appliance): +def check_url(args): + url, appliance = args try: print("Check " + url) req = urllib.request.Request(url, method='HEAD') - urllib.request.urlopen(req, 5) + urllib.request.urlopen(req, timeout=20) except urllib.error.HTTPError as err: if err.getcode() >= 400: - print('Error with url ' + url + ' - ' + str(err)) - sys.exit(1) - except urllib.error.URLError: - print('Invalid URL ' + url) - sys.exit(1) + raise Exception('Error with url ' + url + ' - ' + str(err)) + except urllib.error.URLError as err: + raise Exception('Invalid URL ' + url) + except socket.timeout as e: + raise Exception('Timeout URL ' + url) def check_appliance(appliance): @@ -83,19 +85,21 @@ def check_urls(pool, appliance): with open(os.path.join('appliances', appliance)) as f: appliance_json = json.load(f) + calls = [] + for image in appliance_json['images']: if 'direct_download_url' in image: - pool.apply_async(check_url, [image['direct_download_url'], appliance]) + calls.append((image['direct_download_url'], appliance)) if 'download_url' in image: - pool.apply_async(check_url, [image['download_url'], appliance]) + calls.append((image['download_url'], appliance)) if 'vendor_url' in appliance_json: - pool.apply_async(check_url, [appliance_json['vendor_url'], appliance]) + calls.append((appliance_json['vendor_url'], appliance)) if 'documentation_url' in appliance_json: - pool.apply_async(check_url, [appliance_json['documentation_url'], appliance]) + calls.append((appliance_json['documentation_url'], appliance)) if 'product_url' in appliance_json: - pool.apply_async(check_url, [appliance_json['product_url'], appliance]) - + calls.append((appliance_json['product_url'], appliance)) + return calls def check_packer(packer): path = os.path.join('packer', packer) @@ -122,11 +126,12 @@ def check_symbol(symbol): def main(): pool = Pool(processes=8) + calls_check_url = [] print("=> Check appliances") for appliance in os.listdir('appliances'): print('Check {}'.format(appliance)) check_appliance(appliance) - check_urls(pool, appliance) + calls_check_url += check_urls(pool, appliance) print("=> Check symbols") for symbol in os.listdir('symbols'): if symbol.endswith('.svg'): @@ -136,8 +141,14 @@ def main(): for packer in os.listdir('packer'): check_packer(packer) print("=> Check URL in appliances") + try: + pool.map_async(check_url, calls_check_url).get() + except Exception as e: + print(e) + sys.exit(1) pool.close() pool.join() + print("Everything is ok!") if __name__ == '__main__': main() From c40effa607caca8053e95acd67747e45df98acc2 Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Wed, 11 Nov 2015 10:43:47 +0100 Subject: [PATCH 2/3] Split check urls & check --- .travis.yml | 2 + README.rst | 1 + appliances/juniper-vsrx.gns3a | 2 +- check.py | 56 ------------------ check_urls.py | 108 ++++++++++++++++++++++++++++++++++ 5 files changed, 112 insertions(+), 57 deletions(-) create mode 100644 check_urls.py diff --git a/.travis.yml b/.travis.yml index 1be7a11..e0beb62 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,3 +9,5 @@ addons: - imagemagick script: - python check.py +- python check_urls.py + diff --git a/README.rst b/README.rst index d5ffca0..0123ef8 100644 --- a/README.rst +++ b/README.rst @@ -42,6 +42,7 @@ Check appliance files .. code:: bash python3 check.py + python3 check_urls.py You need to install `imagemagick` before running check.py. diff --git a/appliances/juniper-vsrx.gns3a b/appliances/juniper-vsrx.gns3a index 52810ca..20a2079 100644 --- a/appliances/juniper-vsrx.gns3a +++ b/appliances/juniper-vsrx.gns3a @@ -9,7 +9,7 @@ "product_url": "https://www.juniper.net/us/en/products-services/security/srx-series/vsrx/", "category": "firewall", "status": "experimental", - "vendor_url": "https://www.juniper.net", + "vendor_url": "https://www.juniper.net/us/en/", "registry_version": 1, "usage": "Initial username is root, no password", "port_name_format": "ge-0/0/{0}", diff --git a/check.py b/check.py index bc7d8fc..a48dac8 100644 --- a/check.py +++ b/check.py @@ -19,31 +19,7 @@ import os import jsonschema import json import sys -import socket import subprocess -import urllib.request -from multiprocessing import Pool - - -class MyHTTPRedirectHandler(urllib.request.HTTPRedirectHandler): - def redirect_request(self, req, fp, code, msg, hdrs, newurl): - return None - -urllib.request.install_opener(urllib.request.build_opener(MyHTTPRedirectHandler)) - -def check_url(args): - url, appliance = args - try: - print("Check " + url) - req = urllib.request.Request(url, method='HEAD') - urllib.request.urlopen(req, timeout=20) - except urllib.error.HTTPError as err: - if err.getcode() >= 400: - raise Exception('Error with url ' + url + ' - ' + str(err)) - except urllib.error.URLError as err: - raise Exception('Invalid URL ' + url) - except socket.timeout as e: - raise Exception('Timeout URL ' + url) def check_appliance(appliance): @@ -81,26 +57,6 @@ def check_appliance(appliance): sys.exit(1) -def check_urls(pool, appliance): - with open(os.path.join('appliances', appliance)) as f: - appliance_json = json.load(f) - - calls = [] - - for image in appliance_json['images']: - if 'direct_download_url' in image: - calls.append((image['direct_download_url'], appliance)) - if 'download_url' in image: - calls.append((image['download_url'], appliance)) - - if 'vendor_url' in appliance_json: - calls.append((appliance_json['vendor_url'], appliance)) - if 'documentation_url' in appliance_json: - calls.append((appliance_json['documentation_url'], appliance)) - if 'product_url' in appliance_json: - calls.append((appliance_json['product_url'], appliance)) - return calls - def check_packer(packer): path = os.path.join('packer', packer) if not os.path.isdir(path): @@ -124,14 +80,10 @@ def check_symbol(symbol): def main(): - pool = Pool(processes=8) - - calls_check_url = [] print("=> Check appliances") for appliance in os.listdir('appliances'): print('Check {}'.format(appliance)) check_appliance(appliance) - calls_check_url += check_urls(pool, appliance) print("=> Check symbols") for symbol in os.listdir('symbols'): if symbol.endswith('.svg'): @@ -140,14 +92,6 @@ def main(): print("=> Check packer files") for packer in os.listdir('packer'): check_packer(packer) - print("=> Check URL in appliances") - try: - pool.map_async(check_url, calls_check_url).get() - except Exception as e: - print(e) - sys.exit(1) - pool.close() - pool.join() print("Everything is ok!") if __name__ == '__main__': diff --git a/check_urls.py b/check_urls.py new file mode 100644 index 0000000..f701c01 --- /dev/null +++ b/check_urls.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +# +# Copyright (C) 2015 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import json +import sys +import socket +import time +import urllib.request +from multiprocessing import Pool + +class CheckError(Exception): + def __init__(self, m): + self.message = m + + def __str__(self): + return self.message + + +class MyHTTPRedirectHandler(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, hdrs, newurl): + return None + + +urllib.request.install_opener(urllib.request.build_opener(MyHTTPRedirectHandler)) + + +def check_url(args): + url, appliance = args + print("Check " + url) + + remaining_failure = 5 + error = None + while remaining_failure != 0: + try: + req = urllib.request.Request(url, method='HEAD') + req.add_header + urllib.request.urlopen(req, timeout=45) #Yeah a big big timeout for broken websites... + except urllib.error.HTTPError as err: + if err.getcode() >= 400: + error = CheckError('Error with url {} ({})'.format(url, str(err))) + else: + # We allow error code like 302 + return + except urllib.error.URLError as err: + error = CheckError('Invalid URL {} ({})'.format(url, str(err))) + except socket.timeout as err: + error = CheckError('Timeout URL {} ({})'.format(url, str(err))) + else: + return + remaining_failure -= 1 + time.sleep(5) + raise error + + +def check_urls(pool, appliance): + with open(os.path.join('appliances', appliance)) as f: + appliance_json = json.load(f) + + calls = [] + + for image in appliance_json['images']: + if 'direct_download_url' in image: + calls.append((image['direct_download_url'], appliance)) + if 'download_url' in image: + calls.append((image['download_url'], appliance)) + + if 'vendor_url' in appliance_json: + calls.append((appliance_json['vendor_url'], appliance)) + if 'documentation_url' in appliance_json: + calls.append((appliance_json['documentation_url'], appliance)) + if 'product_url' in appliance_json: + calls.append((appliance_json['product_url'], appliance)) + return calls + + +def main(): + pool = Pool(processes=8) + + calls_check_url = [] + for appliance in os.listdir('appliances'): + calls_check_url += check_urls(pool, appliance) + print("=> Check URL in appliances") + try: + pool.map_async(check_url, calls_check_url).get() + except CheckError as e: + print(e) + sys.exit(1) + pool.close() + pool.join() + print("Everything is ok!") + +if __name__ == '__main__': + main() From 1b78cf013496905712e7db8dc70dbccb04fd7c26 Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Wed, 11 Nov 2015 10:48:38 +0100 Subject: [PATCH 3/3] Catch bad status line --- check_urls.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/check_urls.py b/check_urls.py index f701c01..eb1ff1c 100644 --- a/check_urls.py +++ b/check_urls.py @@ -21,6 +21,7 @@ import sys import socket import time import urllib.request +import http.client from multiprocessing import Pool class CheckError(Exception): @@ -56,6 +57,8 @@ def check_url(args): else: # We allow error code like 302 return + except http.client.BadStatusLine as err: + error = CheckError('Bad status line {} ({})'.format(url, str(err))) except urllib.error.URLError as err: error = CheckError('Invalid URL {} ({})'.format(url, str(err))) except socket.timeout as err: