From 4eeadf87e2e338544d1311c702d850bfadfb6632 Mon Sep 17 00:00:00 2001 From: "Yann E. MORIN\"" Date: Fri, 1 Aug 2008 09:23:58 +0000 Subject: [PATCH] Rework a little bit of the LAN mirror and proxy settings. /trunk/scripts/crosstool.sh | 105 4 101 0 +------------------------ /trunk/scripts/functions | 151 129 22 0 +++++++++++++++++++++++++++++++----- /trunk/config/global/download.in | 193 99 94 0 ++++++++++++++++++++++++---------------------- 3 files changed, 232 insertions(+), 217 deletions(-) --- config/global/download.in | 193 +++++++++++++++++++------------------- scripts/crosstool.sh | 105 +-------------------- scripts/functions | 151 ++++++++++++++++++++++++----- 3 files changed, 232 insertions(+), 217 deletions(-) diff --git a/config/global/download.in b/config/global/download.in index fca6de42..32b8d4bd 100644 --- a/config/global/download.in +++ b/config/global/download.in @@ -11,93 +11,14 @@ config FORCE_DOWNLOAD Usefull if you suspect a tarball to be damaged. -menuconfig USE_LAN_MIRROR - bool - prompt "Use LAN mirror" - default n - help - If you have a machine on your LAN that mirrors some of the needed - tarballs, you can say 'Y' here, and configure adequate values in - the following options. - - Tarballs will be be preferably fetched from the LAN mirror, and if - not found there, standard places will be searched for. - -if USE_LAN_MIRROR - -choice - bool - prompt "Server type:" - -config LAN_MIRROR_HTTP - bool - prompt "http" - -config LAN_MIRROR_FTP - bool - prompt "ftp" - -endchoice # Server type - -config LAN_MIRROR_SCHEME - string - default "http" if LAN_MIRROR_HTTP - default "ftp" if LAN_MIRROR_FTP - -config LAN_MIRROR_HOSTNAME - string - prompt "hostname" - depends on EXPERIMENTAL - default "localhost" - help - Enter here the hostname on your LAN mirror. - -config LAN_MIRROR_PATTERNS - string - prompt "locations patterns (READ HELP!)" - default "/downloads /downloads/%pkg /downloads/%pkg/%ver /downloads/%pkg-%ver" - help - A space-separated list of patterns to find the tarballs on the - LAN mirror. - - You can use the following 'variables', and they'll get replaced by - adequate values: - %pkg : name of the package - %ver : version of the package - - Thus for gcc-4.2.4, %pkg will be replaced with 'gcc', and %ver with - '4.2.4' (both without quotes). Thus if you entered the default - pattern: "/downloads/%pkg/ /downloads/%pkg/%ver /downloads/%pkg-%ver" - the tarball for gcc-4.2.4 would be searched for in: - /downloads/gcc/ - /downloads/gcc/4.2.4/ - /downloads/gcc-4.2.4/ - - Of course, if the files on the LAN mirror are really arranged in weird - ways, entering a list of patterns will be tedious... But see below. - -config LAN_MIRROR_LS_R - bool - prompt "Use ls-lr.gz et al." - default n - help - If the tarball was not found at any of the above locations, see if the - server has a ls-lr.gz (or similar) file, and use that file to see if - the tarball is listed somewhere in that file. - - Common file names looked for are: - ls-lrRt.txt (used at ftp.gnu.org) - find.txt (ditto) - ls-lR - -endif # USE_LAN_MIRROR - menuconfig USE_PROXY bool - prompt "Proxy settings" + prompt "Use a proxy" default n help Say 'Y' here if you need to use a proxy to connect to the internet. + + You can then choose between an HTTP or a SOCKS 4/5 proxy. if USE_PROXY @@ -112,7 +33,7 @@ config PROXY_TYPE_HTTP Use an HTTP proxy to connect to to the internet. Only the http and ftp protocols will be tunneled through this proxy. - + Alternatively to setting this option, you can set and export the following variables in your environment: ftp_proxy=http://user:passwd@proxy.server:port/ @@ -150,18 +71,18 @@ config PROXY_TYPE_SOCKS prompt "SOCKS 4/5 proxy" help Use a Socks 4/5 proxy to connect to the internet. - All protocols can get tunneled through this kind of proxy (depending - on your proxy configuration, some do not allow all protocols, but - chances are that protocols needed by crosstool-NG are allowed). - + All protocols can get tunneled through this kind of proxy (your + proxy configuration may not allow all protocols, but chances are + that protocols needed by crosstool-NG are allowed). + Alternatively to setting this option, you can configure tsocks system-wide, and set and export the following variable in your environment: LD_PRELOAD=/path/to/your/tsocks-library.so - + This option makes use of the tsocks library. You will have to have tsocks installed on your system, of course. - + If you think you do not know what tsocks is, or how to configure it, chances are that you do not need to set this option. @@ -220,7 +141,7 @@ endif # USE_SOCKS_PROXY endchoice -endif # Proxy settings +endif # USE_PROXY config PROXY_TYPE string @@ -231,6 +152,90 @@ config PROXY_TYPE default "socks4" if PROXY_TYPE_SOCKS_4 default "socks5" if PROXY_TYPE_SOCKS_5 +menuconfig USE_LAN_MIRROR + bool + prompt "Use LAN mirror" + default n + help + If you have a machine on your LAN that mirrors some of the needed + tarballs, you can say 'Y' here, and configure adequate values in + the following options. + + Tarballs will be be preferably fetched from the LAN mirror, and if + not found there, standard places will be searched for. + + Obviously, nothing prevents you from using a mirror that is in fact + *not* on your LAN, for example on another subnet of your company's + network, or a mirror on the Internet. + +if USE_LAN_MIRROR + +config LAN_MIRROR_USE_PROXY + bool + prompt "Use the proxy" + default n + depends on USE_PROXY + help + Say 'Y' here if you need to use the proxy to connect to the LAN mirror. + + You'll need to say 'Y' if the LAN mirror is not really on your LAN. + +choice + bool + prompt "Server type:" + +config LAN_MIRROR_HTTP + bool + prompt "http" + +config LAN_MIRROR_FTP + bool + prompt "ftp" + +endchoice # Server type + +config LAN_MIRROR_SCHEME + string + default "http" if LAN_MIRROR_HTTP + default "ftp" if LAN_MIRROR_FTP + +config LAN_MIRROR_HOSTNAME + string + prompt "hostname" + default "" + help + Enter here the hostname on your LAN mirror. + +config LAN_MIRROR_BASE + string + prompt "Base directory" + default "" + help + This is the base directory searched for for tarballs. If you enter + /mirror, then the search is performed in the following directories + on the server: + /mirror// + /mirror/ + + where is replaced with the actual package name. + +config LAN_MIRROR_LS_R + bool +# prompt "Use ls-lR et al." + depends on EXPERIMENTAL + default n + help + If the tarball was not found at the above location, see if the server + has a ls-lr.gz (or similar) file, and use that file to see if the + tarball is listed somewhere in that file. + + Common file names looked for are: + ls-lrRt.txt (used at ftp.gnu.org) + find.txt (ditto) + ls-lR + +endif # USE_LAN_MIRROR + config CONNECT_TIMEOUT int prompt "connection timeout" @@ -238,7 +243,7 @@ config CONNECT_TIMEOUT help From the curl manual: Maximum time in seconds that you allow the connection to the server to take. - + The scenario is as follows; - some enterprise networks have firewalls that prohibit FTP traffic, while still allowing HTTP @@ -246,13 +251,13 @@ config CONNECT_TIMEOUT - after this number of seconds, it is considered that the connection could not be established, and the next URL in the list is tried, until we reach an URL that will go through the firewall, most probably an http:// URL. - + If you have a slow network, you'd better set this value higher than the default 10s. If you know a firewall is blocking connections, but your network is globally fast, you can try to lower this value to jump more quickly to allowed URLs. YMMV. - + Note that this value applies equally to wget if you have that installed. - + Of course, you'd be better off to use a proxy, as offered by the previous option. diff --git a/scripts/crosstool.sh b/scripts/crosstool.sh index a393d87b..2f6660a0 100755 --- a/scripts/crosstool.sh +++ b/scripts/crosstool.sh @@ -101,8 +101,10 @@ if [ -n "${CT_RESTART}" -a ! -d "${CT_STATE_DIR}" ]; then CT_Abort "I will stop here to avoid any carnage" fi -# Make all path absolute, it so much easier! -CT_LOCAL_TARBALLS_DIR=$(CT_MakeAbsolutePath "${CT_LOCAL_TARBALLS_DIR}") +if [ -n "${CT_LOCAL_TARBALLS_DIR}" ]; then + # Make absolute path, it so much easier! + CT_LOCAL_TARBALLS_DIR=$(CT_MakeAbsolutePath "${CT_LOCAL_TARBALLS_DIR}") +fi # If the local tarball directory does not exist, say so, and don't try to save there! if [ ! -d "${CT_LOCAL_TARBALLS_DIR}" ]; then @@ -211,105 +213,6 @@ case "${CT_LOG_TO_FILE}" in ;; esac -# Set environment for proxy access -# This has to be done even if we are restarting, as they don't get -# saved in the step snapshot. -case "${CT_PROXY_TYPE}" in - none) ;; - http) - http_proxy="http://" - case "${CT_PROXY_USER}:${CT_PROXY_PASS}" in - :) ;; - :*) http_proxy="${http_proxy}:${CT_PROXY_PASS}@";; - *:) http_proxy="${http_proxy}${CT_PROXY_USER}@";; - *:*) http_proxy="${http_proxy}${CT_PROXY_USER}:${CT_PROXY_PASS}@";; - esac - export http_proxy="${http_proxy}${CT_PROXY_HOST}:${CT_PROXY_PORT}/" - export https_proxy="${http_proxy}" - export ftp_proxy="${http_proxy}" - CT_DoLog DEBUG "http_proxy='${http_proxy}'" - ;; - sockssys) - # Force not using HTTP proxy - unset http_proxy ftp_proxy https_proxy - CT_HasOrAbort tsocks - . tsocks -on - ;; - socks*) - # Force not using HTTP proxy - unset http_proxy ftp_proxy https_proxy - # Remove any lingering config file from any previous run - rm -f "${CT_BUILD_DIR}/tsocks.conf" - # Find all interfaces and build locally accessible networks - server_ip=$(ping -c 1 -W 2 "${CT_PROXY_HOST}" |head -n 1 |sed -r -e 's/^[^\(]+\(([^\)]+)\).*$/\1/;' || true) - CT_TestOrAbort "SOCKS proxy '${CT_PROXY_HOST}' has no IP." -n "${server_ip}" - /sbin/ifconfig |awk -v server_ip="${server_ip}" ' - BEGIN { - split( server_ip, tmp, "\\." ); - server_ip_num = tmp[1] * 2^24 + tmp[2] * 2^16 + tmp[3] * 2^8 + tmp[4] * 2^0; - pairs = 0; - } - - $0 ~ /^[[:space:]]*inet addr:/ { - split( $2, tmp, ":|\\." ); - if( ( tmp[2] == 127 ) && ( tmp[3] == 0 ) && ( tmp[4] == 0 ) && ( tmp[5] == 1 ) ) { - /* Skip 127.0.0.1, it'\''s taken care of by tsocks itself */ - next; - } - ip_num = tmp[2] * 2^24 + tmp[3] * 2^16 + tmp[4] * 2 ^8 + tmp[5] * 2^0; - i = 32; - do { - i--; - mask = 2^32 - 2^i; - } while( (i!=0) && ( and( server_ip_num, mask ) == and( ip_num, mask ) ) ); - mask = and( 0xFFFFFFFF, lshift( mask, 1 ) ); - if( (i!=0) && (mask!=0) ) { - masked_ip = and( ip_num, mask ); - for( i=0; i"${CT_BUILD_DIR}/tsocks.conf" - ( echo "server = ${server_ip}"; - echo "server_port = ${CT_PROXY_PORT}"; - [ -n "${CT_PROXY_USER}" ] && echo "default_user=${CT_PROXY_USER}"; - [ -n "${CT_PROXY_PASS}" ] && echo "default_pass=${CT_PROXY_PASS}"; - ) >>"${CT_BUILD_DIR}/tsocks.conf" - case "${CT_PROXY_TYPE/socks}" in - 4|5) proxy_type="${CT_PROXY_TYPE/socks}";; - auto) - reply=$(inspectsocks "${server_ip}" "${CT_PROXY_PORT}" 2>&1 || true) - case "${reply}" in - *"server is a version 4 socks server") proxy_type=4;; - *"server is a version 5 socks server") proxy_type=5;; - *) CT_Abort "Unable to determine SOCKS proxy type for '${CT_PROXY_HOST}:${CT_PROXY_PORT}'" - esac - ;; - esac - echo "server_type = ${proxy_type}" >> "${CT_BUILD_DIR}/tsocks.conf" - CT_HasOrAbort tsocks - # If tsocks was found, then validateconf is present (distributed with tsocks). - CT_DoExecLog DEBUG validateconf -f "${CT_BUILD_DIR}/tsocks.conf" - export TSOCKS_CONF_FILE="${CT_BUILD_DIR}/tsocks.conf" - . tsocks -on - ;; -esac - # Setting up the rest of the environment only if not restarting if [ -z "${CT_RESTART}" ]; then # Determine build system if not set by the user diff --git a/scripts/functions b/scripts/functions index d71a3401..a494f62b 100644 --- a/scripts/functions +++ b/scripts/functions @@ -288,6 +288,106 @@ CT_GetFileExtension() { return 0 } +# Set environment for proxy access +# Usage: CT_DoSetProxy +# where proxy_type is one of 'none', 'http', 'sockssys', 'socks4' or 'socks5' +CT_DoSetProxy() { + case "${CT_PROXY_TYPE}" in + http) + http_proxy="http://" + case "${CT_PROXY_USER}:${CT_PROXY_PASS}" in + :) ;; + :*) http_proxy="${http_proxy}:${CT_PROXY_PASS}@";; + *:) http_proxy="${http_proxy}${CT_PROXY_USER}@";; + *:*) http_proxy="${http_proxy}${CT_PROXY_USER}:${CT_PROXY_PASS}@";; + esac + export http_proxy="${http_proxy}${CT_PROXY_HOST}:${CT_PROXY_PORT}/" + export https_proxy="${http_proxy}" + export ftp_proxy="${http_proxy}" + CT_DoLog DEBUG "http_proxy='${http_proxy}'" + ;; + sockssys) + # Force not using HTTP proxy + unset http_proxy ftp_proxy https_proxy + CT_HasOrAbort tsocks + . tsocks -on + ;; + socks*) + # Force not using HTTP proxy + unset http_proxy ftp_proxy https_proxy + # Remove any lingering config file from any previous run + rm -f "${CT_BUILD_DIR}/tsocks.conf" + # Find all interfaces and build locally accessible networks + server_ip=$(ping -c 1 -W 2 "${CT_PROXY_HOST}" |head -n 1 |sed -r -e 's/^[^\(]+\(([^\)]+)\).*$/\1/;' || true) + CT_TestOrAbort "SOCKS proxy '${CT_PROXY_HOST}' has no IP." -n "${server_ip}" + /sbin/ifconfig |awk -v server_ip="${server_ip}" ' + BEGIN { + split( server_ip, tmp, "\\." ); + server_ip_num = tmp[1] * 2^24 + tmp[2] * 2^16 + tmp[3] * 2^8 + tmp[4] * 2^0; + pairs = 0; + } + + $0 ~ /^[[:space:]]*inet addr:/ { + split( $2, tmp, ":|\\." ); + if( ( tmp[2] == 127 ) && ( tmp[3] == 0 ) && ( tmp[4] == 0 ) && ( tmp[5] == 1 ) ) { + /* Skip 127.0.0.1, it'\''s taken care of by tsocks itself */ + next; + } + ip_num = tmp[2] * 2^24 + tmp[3] * 2^16 + tmp[4] * 2 ^8 + tmp[5] * 2^0; + i = 32; + do { + i--; + mask = 2^32 - 2^i; + } while( (i!=0) && ( and( server_ip_num, mask ) == and( ip_num, mask ) ) ); + mask = and( 0xFFFFFFFF, lshift( mask, 1 ) ); + if( (i!=0) && (mask!=0) ) { + masked_ip = and( ip_num, mask ); + for( i=0; i"${CT_BUILD_DIR}/tsocks.conf" + ( echo "server = ${server_ip}"; + echo "server_port = ${CT_PROXY_PORT}"; + [ -n "${CT_PROXY_USER}" ] && echo "default_user=${CT_PROXY_USER}"; + [ -n "${CT_PROXY_PASS}" ] && echo "default_pass=${CT_PROXY_PASS}"; + ) >>"${CT_BUILD_DIR}/tsocks.conf" + case "${CT_PROXY_TYPE/socks}" in + 4|5) proxy_type="${CT_PROXY_TYPE/socks}";; + auto) + reply=$(inspectsocks "${server_ip}" "${CT_PROXY_PORT}" 2>&1 || true) + case "${reply}" in + *"server is a version 4 socks server") proxy_type=4;; + *"server is a version 5 socks server") proxy_type=5;; + *) CT_Abort "Unable to determine SOCKS proxy type for '${CT_PROXY_HOST}:${CT_PROXY_PORT}'" + esac + ;; + esac + echo "server_type = ${proxy_type}" >> "${CT_BUILD_DIR}/tsocks.conf" + CT_HasOrAbort tsocks + # If tsocks was found, then validateconf is present (distributed with tsocks). + CT_DoExecLog DEBUG validateconf -f "${CT_BUILD_DIR}/tsocks.conf" + export TSOCKS_CONF_FILE="${CT_BUILD_DIR}/tsocks.conf" + . tsocks -on + ;; + esac +} + # Download an URL using wget # Usage: CT_DoGetFileWget CT_DoGetFileWget() { @@ -353,35 +453,39 @@ CT_GetFile() { return 0 fi + # Try to retrieve the file CT_Pushd "${CT_TARBALLS_DIR}" - # We'd rather have a bzip2'ed tarball, then gzipped tarball, plain tarball, - # or, as a failover, a file without extension. - # Try local copy first, if it exists - for ext in ${first_ext} .tar.bz2 .tar.gz .tgz .tar ''; do - CT_DoLog DEBUG "Trying '${CT_LOCAL_TARBALLS_DIR}/${file}${ext}'" - if [ -r "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" -a \ - "${CT_FORCE_DOWNLOAD}" != "y" ]; then - CT_DoLog EXTRA "Using '${file}' from local storage" - ln -sv "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" "${file}${ext}" |CT_DoLog ALL - return 0 - fi - done + + if [ -n "${CT_LOCAL_TARBALLS_DIR}" ]; then + CT_DoLog DEBUG "Trying to retrieve an already downloaded copy of '${file}'" + # We'd rather have a bzip2'ed tarball, then gzipped tarball, plain tarball, + # or, as a failover, a file without extension. + # Try local copy first, if it exists + for ext in ${first_ext} .tar.bz2 .tar.gz .tgz .tar ''; do + CT_DoLog DEBUG "Trying '${CT_LOCAL_TARBALLS_DIR}/${file}${ext}'" + if [ -r "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" -a \ + "${CT_FORCE_DOWNLOAD}" != "y" ]; then + CT_DoLog EXTRA "Got '${file}' from local storage" + ln -sv "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" "${file}${ext}" |CT_DoLog ALL + return 0 + fi + done + fi # Not found locally, try from the network - CT_DoLog EXTRA "Retrieving '${file}' from network" # Start with LAN mirror if [ "${CT_USE_LAN_MIRROR}" = "y" ]; then - LAN_URLs= - for pat in ${CT_LAN_MIRROR_PATTERNS}; do + CT_DoSetProxy ${CT_LAN_MIRROR_USE_PROXY:+${CT_PROXY_TYPE}} + CT_DoLog DEBUG "Trying to retrieve a copy of '${file}' from LAN mirror '${CT_LAN_MIRROR_HOSTNAME}'" + CT_TestOrAbort "Please set the LAN mirror hostname" -n "${CT_LAN_MIRROR_HOSTNAME}" + CT_TestOrAbort "Please tell me where to find tarballs on the LAN mirror '${CT_LAN_MIRROR_HOSTNAME}'" -n "${CT_LAN_MIRROR_BASE}" + for ext in ${first_ext} .tar.bz2 .tar.gz .tgz .tar ''; do # Please note: we just have the file's basename in a single piece. # So we have to just try and split it back into name and version... :-( - pat="${pat//\%pkg/${file%-*}}" - pat="${pat//\%ver/${file##*-}}" - LAN_URLs="${LAN_URLs} ${CT_LAN_MIRROR_SCHEME}://${CT_LAN_MIRROR_HOSTNAME}/${pat}" - done - for ext in ${first_ext} .tar.bz2 .tar.gz .tgz .tar ''; do - for url in ${LAN_URLs}; do + for url in "${CT_LAN_MIRROR_SCHEME}://${CT_LAN_MIRROR_HOSTNAME}/${CT_LAN_MIRROR_BASE}/${file%-*}" \ + "${CT_LAN_MIRROR_SCHEME}://${CT_LAN_MIRROR_HOSTNAME}/${CT_LAN_MIRROR_BASE}"; \ + do CT_DoLog DEBUG "Trying '${url}/${file}${ext}'" CT_DoGetFile "${url}/${file}${ext}" if [ -f "${file}${ext}" ]; then @@ -393,6 +497,7 @@ CT_GetFile() { mv "${file}${ext}" "${CT_LOCAL_TARBALLS_DIR}" |CT_DoLog ALL ln -sv "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" "${file}${ext}" |CT_DoLog ALL fi + CT_DoLog EXTRA "Got '${file}' from the LAN mirror" return 0 fi done @@ -400,6 +505,7 @@ CT_GetFile() { fi # OK, available neither localy, nor from the LAN mirror (if any). + CT_DoSetProxy ${CT_PROXY_TYPE} for ext in ${first_ext} .tar.bz2 .tar.gz .tgz .tar ''; do # Try all urls in turn for url in "$@"; do @@ -414,13 +520,14 @@ CT_GetFile() { mv "${file}${ext}" "${CT_LOCAL_TARBALLS_DIR}" |CT_DoLog ALL ln -sv "${CT_LOCAL_TARBALLS_DIR}/${file}${ext}" "${file}${ext}" |CT_DoLog ALL fi + CT_DoLog EXTRA "Got '${file}' from the Internet" return 0 fi done done CT_Popd - CT_Abort "Could not download '${file}', and not present in '${CT_LOCAL_TARBALLS_DIR}'" + CT_Abort "Could not retrieve '${file}'." } # Extract a tarball and patch the resulting sources if necessary.