From c40effa607caca8053e95acd67747e45df98acc2 Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Wed, 11 Nov 2015 10:43:47 +0100 Subject: [PATCH] Split check urls & check --- .travis.yml | 2 + README.rst | 1 + appliances/juniper-vsrx.gns3a | 2 +- check.py | 56 ------------------ check_urls.py | 108 ++++++++++++++++++++++++++++++++++ 5 files changed, 112 insertions(+), 57 deletions(-) create mode 100644 check_urls.py diff --git a/.travis.yml b/.travis.yml index 1be7a11..e0beb62 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,3 +9,5 @@ addons: - imagemagick script: - python check.py +- python check_urls.py + diff --git a/README.rst b/README.rst index d5ffca0..0123ef8 100644 --- a/README.rst +++ b/README.rst @@ -42,6 +42,7 @@ Check appliance files .. code:: bash python3 check.py + python3 check_urls.py You need to install `imagemagick` before running check.py. diff --git a/appliances/juniper-vsrx.gns3a b/appliances/juniper-vsrx.gns3a index 52810ca..20a2079 100644 --- a/appliances/juniper-vsrx.gns3a +++ b/appliances/juniper-vsrx.gns3a @@ -9,7 +9,7 @@ "product_url": "https://www.juniper.net/us/en/products-services/security/srx-series/vsrx/", "category": "firewall", "status": "experimental", - "vendor_url": "https://www.juniper.net", + "vendor_url": "https://www.juniper.net/us/en/", "registry_version": 1, "usage": "Initial username is root, no password", "port_name_format": "ge-0/0/{0}", diff --git a/check.py b/check.py index bc7d8fc..a48dac8 100644 --- a/check.py +++ b/check.py @@ -19,31 +19,7 @@ import os import jsonschema import json import sys -import socket import subprocess -import urllib.request -from multiprocessing import Pool - - -class MyHTTPRedirectHandler(urllib.request.HTTPRedirectHandler): - def redirect_request(self, req, fp, code, msg, hdrs, newurl): - return None - -urllib.request.install_opener(urllib.request.build_opener(MyHTTPRedirectHandler)) - -def check_url(args): - url, appliance = args - try: - print("Check " + url) - req = urllib.request.Request(url, method='HEAD') - urllib.request.urlopen(req, timeout=20) - except urllib.error.HTTPError as err: - if err.getcode() >= 400: - raise Exception('Error with url ' + url + ' - ' + str(err)) - except urllib.error.URLError as err: - raise Exception('Invalid URL ' + url) - except socket.timeout as e: - raise Exception('Timeout URL ' + url) def check_appliance(appliance): @@ -81,26 +57,6 @@ def check_appliance(appliance): sys.exit(1) -def check_urls(pool, appliance): - with open(os.path.join('appliances', appliance)) as f: - appliance_json = json.load(f) - - calls = [] - - for image in appliance_json['images']: - if 'direct_download_url' in image: - calls.append((image['direct_download_url'], appliance)) - if 'download_url' in image: - calls.append((image['download_url'], appliance)) - - if 'vendor_url' in appliance_json: - calls.append((appliance_json['vendor_url'], appliance)) - if 'documentation_url' in appliance_json: - calls.append((appliance_json['documentation_url'], appliance)) - if 'product_url' in appliance_json: - calls.append((appliance_json['product_url'], appliance)) - return calls - def check_packer(packer): path = os.path.join('packer', packer) if not os.path.isdir(path): @@ -124,14 +80,10 @@ def check_symbol(symbol): def main(): - pool = Pool(processes=8) - - calls_check_url = [] print("=> Check appliances") for appliance in os.listdir('appliances'): print('Check {}'.format(appliance)) check_appliance(appliance) - calls_check_url += check_urls(pool, appliance) print("=> Check symbols") for symbol in os.listdir('symbols'): if symbol.endswith('.svg'): @@ -140,14 +92,6 @@ def main(): print("=> Check packer files") for packer in os.listdir('packer'): check_packer(packer) - print("=> Check URL in appliances") - try: - pool.map_async(check_url, calls_check_url).get() - except Exception as e: - print(e) - sys.exit(1) - pool.close() - pool.join() print("Everything is ok!") if __name__ == '__main__': diff --git a/check_urls.py b/check_urls.py new file mode 100644 index 0000000..f701c01 --- /dev/null +++ b/check_urls.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +# +# Copyright (C) 2015 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import json +import sys +import socket +import time +import urllib.request +from multiprocessing import Pool + +class CheckError(Exception): + def __init__(self, m): + self.message = m + + def __str__(self): + return self.message + + +class MyHTTPRedirectHandler(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, hdrs, newurl): + return None + + +urllib.request.install_opener(urllib.request.build_opener(MyHTTPRedirectHandler)) + + +def check_url(args): + url, appliance = args + print("Check " + url) + + remaining_failure = 5 + error = None + while remaining_failure != 0: + try: + req = urllib.request.Request(url, method='HEAD') + req.add_header + urllib.request.urlopen(req, timeout=45) #Yeah a big big timeout for broken websites... + except urllib.error.HTTPError as err: + if err.getcode() >= 400: + error = CheckError('Error with url {} ({})'.format(url, str(err))) + else: + # We allow error code like 302 + return + except urllib.error.URLError as err: + error = CheckError('Invalid URL {} ({})'.format(url, str(err))) + except socket.timeout as err: + error = CheckError('Timeout URL {} ({})'.format(url, str(err))) + else: + return + remaining_failure -= 1 + time.sleep(5) + raise error + + +def check_urls(pool, appliance): + with open(os.path.join('appliances', appliance)) as f: + appliance_json = json.load(f) + + calls = [] + + for image in appliance_json['images']: + if 'direct_download_url' in image: + calls.append((image['direct_download_url'], appliance)) + if 'download_url' in image: + calls.append((image['download_url'], appliance)) + + if 'vendor_url' in appliance_json: + calls.append((appliance_json['vendor_url'], appliance)) + if 'documentation_url' in appliance_json: + calls.append((appliance_json['documentation_url'], appliance)) + if 'product_url' in appliance_json: + calls.append((appliance_json['product_url'], appliance)) + return calls + + +def main(): + pool = Pool(processes=8) + + calls_check_url = [] + for appliance in os.listdir('appliances'): + calls_check_url += check_urls(pool, appliance) + print("=> Check URL in appliances") + try: + pool.map_async(check_url, calls_check_url).get() + except CheckError as e: + print(e) + sys.exit(1) + pool.close() + pool.join() + print("Everything is ok!") + +if __name__ == '__main__': + main()