From 465cf358881cfad1293669f235a62e9f31539521 Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Tue, 18 Jun 2024 16:09:30 +0200 Subject: [PATCH] scripts/download.pl: detach mirror URLs from script file Detach URL from download script to make it easier to change and update the URLs. The mirror list is moved to a JSON file in the scripts directory called projectsmirrors.json. Link: https://github.com/openwrt/openwrt/pull/15745 Signed-off-by: Christian Marangi --- scripts/download.pl | 74 ++++++++++++------------------------ scripts/projectsmirrors.json | 70 ++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 50 deletions(-) create mode 100644 scripts/projectsmirrors.json diff --git a/scripts/download.pl b/scripts/download.pl index 3c57bcff228..eba48e84a0a 100755 --- a/scripts/download.pl +++ b/scripts/download.pl @@ -12,6 +12,7 @@ use warnings; use File::Basename; use File::Copy; use Text::ParseWords; +use JSON::PP; @ARGV > 2 or die "Syntax: $0 [ ...]\n"; @@ -56,6 +57,20 @@ sub localmirrors { return @mlist; } +sub projectsmirrors { + my $project = shift; + + open (PM, "$scriptdir/projectsmirrors.json") || + die "Can´t open $scriptdir/projectsmirrors.json: $!\n"; + local $/; + my $mirror_json = ; + my $mirror = decode_json $mirror_json; + + foreach (@{$mirror->{$project}}) { + push @mirrors, $_ . "/" . ($1 or ""); + } +} + sub which($) { my $prog = shift; my $res = `command -v $prog`; @@ -244,51 +259,23 @@ foreach my $mirror (@ARGV) { if ($mirror =~ /^\@SF\/(.+)$/) { # give sourceforge a few more tries, because it redirects to different mirrors for (1 .. 5) { - push @mirrors, "https://downloads.sourceforge.net/$1"; + projectsmirrors '@SF', $1; } } elsif ($mirror =~ /^\@OPENWRT$/) { # use OpenWrt source server directly } elsif ($mirror =~ /^\@DEBIAN\/(.+)$/) { - push @mirrors, "https://ftp.debian.org/debian/$1"; - push @mirrors, "https://mirror.leaseweb.com/debian/$1"; - push @mirrors, "https://mirror.netcologne.de/debian/$1"; - push @mirrors, "https://mirrors.tuna.tsinghua.edu.cn/debian/$1"; - push @mirrors, "https://mirrors.ustc.edu.cn/debian/$1" + projectsmirrors '@DEBIAN', $1; } elsif ($mirror =~ /^\@APACHE\/(.+)$/) { - push @mirrors, "https://dlcdn.apache.org/$1"; - push @mirrors, "https://mirror.aarnet.edu.au/pub/apache/$1"; - push @mirrors, "https://mirror.csclub.uwaterloo.ca/apache/$1"; - push @mirrors, "https://archive.apache.org/dist/$1"; - push @mirrors, "https://mirror.cogentco.com/pub/apache/$1"; - push @mirrors, "https://mirror.navercorp.com/apache/$1"; - push @mirrors, "https://ftp.jaist.ac.jp/pub/apache/$1"; - push @mirrors, "https://apache.cs.utah.edu/apache.org/$1"; - push @mirrors, "http://apache.mirrors.ovh.net/ftp.apache.org/dist/$1"; - push @mirrors, "https://mirrors.tuna.tsinghua.edu.cn/apache/$1"; - push @mirrors, "https://mirrors.ustc.edu.cn/apache/$1"; + projectsmirrors '@APACHE', $1; } elsif ($mirror =~ /^\@GITHUB\/(.+)$/) { # give github a few more tries (different mirrors) for (1 .. 5) { - push @mirrors, "https://raw.githubusercontent.com/$1"; + projectsmirrors '@GITHUB', $1; } } elsif ($mirror =~ /^\@GNU\/(.+)$/) { - push @mirrors, "https://ftpmirror.gnu.org/$1"; - push @mirrors, "https://mirror.csclub.uwaterloo.ca/gnu/$1"; - push @mirrors, "https://mirror.netcologne.de/gnu/$1"; - push @mirrors, "https://ftp.kddilabs.jp/GNU/gnu/$1"; - push @mirrors, "https://www.nic.funet.fi/pub/gnu/gnu/$1"; - push @mirrors, "https://mirror.navercorp.com/gnu/$1"; - push @mirrors, "https://mirrors.rit.edu/gnu/$1"; - push @mirrors, "https://ftp.gnu.org/gnu/$1"; - push @mirrors, "https://mirrors.tuna.tsinghua.edu.cn/gnu/$1"; - push @mirrors, "https://mirrors.ustc.edu.cn/gnu/$1"; + projectsmirrors '@GNU', $1; } elsif ($mirror =~ /^\@SAVANNAH\/(.+)$/) { - push @mirrors, "https://download.savannah.nongnu.org/releases/$1"; - push @mirrors, "https://mirror.netcologne.de/savannah/$1"; - push @mirrors, "https://mirror.csclub.uwaterloo.ca/nongnu/$1"; - push @mirrors, "https://ftp.acc.umu.se/mirror/gnu.org/savannah/$1"; - push @mirrors, "https://nongnu.uib.no/$1"; - push @mirrors, "https://cdimage.debian.org/mirror/gnu.org/savannah/$1"; + projectsmirrors '@SAVANNAH', $1; } elsif ($mirror =~ /^\@KERNEL\/(.+)$/) { my @extra = ( $1 ); if ($filename =~ /linux-\d+\.\d+(?:\.\d+)?-rc/) { @@ -297,29 +284,16 @@ foreach my $mirror (@ARGV) { push @extra, "$extra[0]/longterm/v$1"; } foreach my $dir (@extra) { - push @mirrors, "https://cdn.kernel.org/pub/$dir"; - push @mirrors, "https://mirrors.mit.edu/kernel/$dir"; - push @mirrors, "http://ftp.nara.wide.ad.jp/pub/kernel.org/$dir"; - push @mirrors, "http://www.ring.gr.jp/archives/linux/kernel.org/$dir"; - push @mirrors, "https://ftp.riken.jp/Linux/kernel.org/$dir"; - push @mirrors, "https://www.mirrorservice.org/sites/ftp.kernel.org/pub/$dir"; - push @mirrors, "https://mirrors.ustc.edu.cn/kernel.org/$dir"; + projectsmirrors '@KERNEL', $dir; } } elsif ($mirror =~ /^\@GNOME\/(.+)$/) { - push @mirrors, "https://download.gnome.org/sources/$1"; - push @mirrors, "https://mirror.csclub.uwaterloo.ca/gnome/sources/$1"; - push @mirrors, "https://ftp.acc.umu.se/pub/GNOME/sources/$1"; - push @mirrors, "http://ftp.cse.buffalo.edu/pub/Gnome/sources/$1"; - push @mirrors, "http://ftp.nara.wide.ad.jp/pub/X11/GNOME/sources/$1"; - push @mirrors, "https://mirrors.ustc.edu.cn/gnome/sources/$1"; + projectsmirrors '@GNOME', $1; } else { push @mirrors, $mirror; } } -push @mirrors, 'https://sources.cdn.openwrt.org'; -push @mirrors, 'https://sources.openwrt.org'; -push @mirrors, 'https://mirror2.openwrt.org/sources'; +projectsmirrors '@OPENWRT'; if (-f "$target/$filename") { $hash_cmd and do { diff --git a/scripts/projectsmirrors.json b/scripts/projectsmirrors.json new file mode 100644 index 00000000000..9bc2d7ac10e --- /dev/null +++ b/scripts/projectsmirrors.json @@ -0,0 +1,70 @@ +{ + "@SF": [ + "https://downloads.sourceforge.net" + ], + "@DEBIAN": [ + "https://ftp.debian.org/debian", + "https://mirror.leaseweb.com/debian", + "https://mirror.netcologne.de/debian", + "https://mirrors.tuna.tsinghua.edu.cn/debian", + "https://mirrors.ustc.edu.cn/debian" + ], + "@APACHE": [ + "https://dlcdn.apache.org", + "https://mirror.aarnet.edu.au/pub/apache", + "https://mirror.csclub.uwaterloo.ca/apache", + "https://archive.apache.org/dist", + "https://mirror.cogentco.com/pub/apache", + "https://mirror.navercorp.com/apache", + "https://ftp.jaist.ac.jp/pub/apache", + "https://apache.cs.utah.edu/apache.org", + "http://apache.mirrors.ovh.net/ftp.apache.org/dist", + "https://mirrors.tuna.tsinghua.edu.cn/apache", + "https://mirrors.ustc.edu.cn/apache" + ], + "@GITHUB": [ + "https://raw.githubusercontent.com" + ], + "@GNU": [ + "https://ftpmirror.gnu.org", + "https://mirror.csclub.uwaterloo.ca/gnu", + "https://mirror.netcologne.de/gnu", + "https://ftp.kddilabs.jp/GNU/gnu", + "https://www.nic.funet.fi/pub/gnu/gnu", + "https://mirror.navercorp.com/gnu", + "https://mirrors.rit.edu/gnu", + "https://ftp.gnu.org/gnu", + "https://mirrors.tuna.tsinghua.edu.cn/gnu", + "https://mirrors.ustc.edu.cn/gnu" + ], + "@SAVANNAH": [ + "https://download.savannah.nongnu.org/releases", + "https://mirror.netcologne.de/savannah", + "https://mirror.csclub.uwaterloo.ca/nongnu", + "https://ftp.acc.umu.se/mirror/gnu.org/savannah", + "https://nongnu.uib.no", + "https://cdimage.debian.org/mirror/gnu.org/savannah" + ], + "@KERNEL": [ + "https://cdn.kernel.org/pub", + "https://mirrors.mit.edu/kernel", + "http://ftp.nara.wide.ad.jp/pub/kernel.org", + "http://www.ring.gr.jp/archives/linux/kernel.org", + "https://ftp.riken.jp/Linux/kernel.org", + "https://www.mirrorservice.org/sites/ftp.kernel.org/pub", + "https://mirrors.ustc.edu.cn/kernel.org" + ], + "@GNOME": [ + "https://download.gnome.org/sources", + "https://mirror.csclub.uwaterloo.ca/gnome/sources", + "https://ftp.acc.umu.se/pub/GNOME/sources", + "http://ftp.cse.buffalo.edu/pub/Gnome/sources", + "http://ftp.nara.wide.ad.jp/pub/X11/GNOME/sources", + "https://mirrors.ustc.edu.cn/gnome/sources" + ], + "@OPENWRT": [ + "https://sources.cdn.openwrt.org", + "https://sources.openwrt.org", + "https://mirror2.openwrt.org/sources" + ] +}