From 5f167578abf6bd3a87dd5e196beded40a7d74fbc Mon Sep 17 00:00:00 2001 From: Bernhard Ehlers Date: Sat, 27 May 2023 16:08:34 +0200 Subject: [PATCH 01/13] Update docker container - chromium: add a dummy layer to get an updated container timestamp - ipterm: add a dummy layer to get an updated container timestamp - kalilinux: replaced outdated python-scapy by python3-scapy --- docker/chromium/Dockerfile | 3 +++ docker/ipterm/cli/Dockerfile | 3 +++ docker/kalilinux/Dockerfile | 2 +- docker/kalilinux/README.md | 2 +- docker/kalilinux/index-1.html | 2 +- 5 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docker/chromium/Dockerfile b/docker/chromium/Dockerfile index bab8476..5afae61 100644 --- a/docker/chromium/Dockerfile +++ b/docker/chromium/Dockerfile @@ -1,4 +1,7 @@ FROM jess/chromium MAINTAINER developers@gns3.net +# add a dummy layer to get an updated container timestamp +RUN /bin/true + VOLUME /data diff --git a/docker/ipterm/cli/Dockerfile b/docker/ipterm/cli/Dockerfile index dfa9009..d34eb8a 100644 --- a/docker/ipterm/cli/Dockerfile +++ b/docker/ipterm/cli/Dockerfile @@ -2,5 +2,8 @@ FROM gns3/ipterm-base +# add a dummy layer to get an updated container timestamp +RUN /bin/true + VOLUME [ "/root" ] CMD [ "sh", "-c", "cd; exec bash -i" ] diff --git a/docker/kalilinux/Dockerfile b/docker/kalilinux/Dockerfile index e013587..5282cf2 100644 --- a/docker/kalilinux/Dockerfile +++ b/docker/kalilinux/Dockerfile @@ -1,6 +1,6 @@ FROM kalilinux/kali-rolling -RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y tshark && apt-get install -y --no-install-recommends metasploit-framework nmap hydra sqlmap telnet openssh-client dnsutils yersinia ettercap-text-only cisco-global-exploiter cisco-auditing-tool snmp dsniff dnschef fping hping3 python-scapy\ +RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y tshark && apt-get install -y --no-install-recommends metasploit-framework nmap hydra sqlmap telnet openssh-client dnsutils yersinia ettercap-text-only cisco-global-exploiter cisco-auditing-tool snmp dsniff dnschef fping hping3 python3-scapy\ && rm -rf /var/lib/apt/lists/* CMD /bin/bash diff --git a/docker/kalilinux/README.md b/docker/kalilinux/README.md index 18ed695..2a503de 100644 --- a/docker/kalilinux/README.md +++ b/docker/kalilinux/README.md @@ -20,7 +20,7 @@ metasploit-framework * fping * hping3 * tshark - * python-scapy + * python3-scapy * yersinia ## Build and publish the Images diff --git a/docker/kalilinux/index-1.html b/docker/kalilinux/index-1.html index 3a1b27a..3f916b3 100644 --- a/docker/kalilinux/index-1.html +++ b/docker/kalilinux/index-1.html @@ -19,7 +19,7 @@ metasploit-framework * fping * hping3 * tshark - * python-scapy + * python3-scapy * yersinia

Build and publish the Images

First the base image has to be created:

From c80dd65dfc7a8d601ed66c4fa24b6eb377382dc9 Mon Sep 17 00:00:00 2001 From: Bernhard Ehlers Date: Sat, 27 May 2023 16:15:39 +0200 Subject: [PATCH 02/13] Docker Build System Docker images are rebuilt, when the base image or the build context has changed. --- .github/bin/docker_build | 453 +++++++++++++++++++++++++++++++++++ .github/bin/requirements.txt | 2 + .github/docker_build.md | 118 +++++++++ .github/workflows/build.yml | 59 +++++ docker/docker_images | 43 ++++ 5 files changed, 675 insertions(+) create mode 100755 .github/bin/docker_build create mode 100644 .github/bin/requirements.txt create mode 100644 .github/docker_build.md create mode 100644 .github/workflows/build.yml create mode 100644 docker/docker_images diff --git a/.github/bin/docker_build b/.github/bin/docker_build new file mode 100755 index 0000000..0c2bacd --- /dev/null +++ b/.github/bin/docker_build @@ -0,0 +1,453 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2022 Bernhard Ehlers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +docker_build - (re)build outdated docker images + +usage: docker_build [--help] [--all] [--dir DIR] [--dry-run] [--file FILE] + [image ...] + +positional arguments: + image images to build additionally + +optional arguments: + --help, -h, -? prints this screen + --all, -a build all images + --dir DIR, -C DIR change directory before building + --dry-run, -n do not build images, just print them + --file FILE, -f FILE use FILE as image config (default: 'docker_images') + +The docker images and their properties are configured in a +file, by default 'docker_images' in the current directory. + +Format of the lines in the configuration file: + Name Directory [ Base Image] [ Build Options] + or + Global Build Options + +When running without an image arg, it checks all images, +if the directory containing its Dockerfile has changed or +its base image has been updated. + +In some special cases a docker image needs a forced rebuild. +For that add the list of images or base images, to be rebuild, +to the arguments. When using the option -a/--all, all images are +forcibly rebuild, except those specified on the command line. + +The environment variable DOCKER_ACCOUNT must be set to the +registry/user of the Docker account to use. +""" + +import os +import sys +import argparse +import json +import re +import shlex +import subprocess +import dxf +import requests.exceptions +import dateutil.parser + +base_images = {} +images = [] + + +parser = argparse.ArgumentParser(add_help=False, \ + description='%(prog)s - (re)build outdated docker images') +parser.add_argument('--help', '-h', '-?', action='help', + help='prints this screen') +parser.add_argument('--all', '-a', action='store_true', + help='build all images') +parser.add_argument('--dir', '-C', action='append', + help='change directory before building') +parser.add_argument('--dry-run', '-n', action='store_true', + help='do not build images, just print them') +parser.add_argument('--file', '-f', default='docker_images', + help="use FILE as image config (default: '%(default)s')") +parser.add_argument('image', nargs="*", + help='images to build additionally') + + +# regex for repository +RE_REPOSITORY = re.compile(r''' +(?:(?P[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])? # host name + (?: # followed by ... + (?:\.[a-zA-Z0-9] # domain + (?:[a-zA-Z0-9-]*[a-zA-Z0-9])?)+ + (?::[0-9]+)? # and optional port + | # or ... + :[0-9]+) # port +)/)? # finally a / + +(?P[a-z0-9]+(?:(?:\.|__?|-+)[a-z0-9]+)* # repo component + (?:/[a-z0-9]+(?:(?:\.|__?|-+)[a-z0-9]+)*)* # more components +) +(?::(?P[a-zA-Z0-9_][a-zA-Z0-9_.-]{,127}))? # optional tag +(?:@(?P[a-z0-9]+(?:[.+_-][a-z0-9]+)* # optional digest + :[0-9a-f]{32,}))* +''', re.VERBOSE) + +def parse_repository(repository): + """ extract registry, user, repo and tag from repository """ + # verify repository format and extract components + match = RE_REPOSITORY.fullmatch(repository) + if not match: + raise ValueError("invalid reference format") + registry = match.group('host') or "docker.io" + repo = match.group('repo') + tag = match.group('digest') or match.group('tag') or "latest" + len_registry = len(registry) + # user is first component of repo, if repo has more than one part + user = repo.split("/", 1)[0] if "/" in repo else None + # special handling for docker.io + if registry == "docker.io": + registry = "registry-1.docker.io" + if not user: + repo = "library/" + repo + # check length of repository string (without tag) + if len_registry + len(repo) > 254: + raise ValueError("repository name must not be more than 255 characters") + return registry, user, repo, tag + + +def docker_auth_user(docker, response): + """ authenticate with user/password """ + docker.authenticate(docker_login["account"], docker_login["password"], + response=response) + + +def docker_auth_none(docker, response): + """ public access """ + docker.authenticate(None, None, response=response) + + +def get_time_layers(repository): + """ + get created time and layer info from the docker registry + + To retrieve this information the Docker registry client dxf is used. + https://github.com/davedoesdev/dxf + """ + + try: + registry, _, repo, tag = parse_repository(repository) + if registry == docker_login["registry"] and \ + docker_login["user"] and docker_login["password"]: + docker_auth = docker_auth_user + else: + docker_auth = docker_auth_none + + # open docker connection + with dxf.DXF(registry, repo, docker_auth, timeout=30) as docker: + # get config digest + try: + digest = docker.get_digest(tag, platform="linux/amd64") + except dxf.exceptions.DXFUnauthorizedError: + return (None, []) + except requests.exceptions.HTTPError as err: + if err.response.status_code not in (401, 403, 404): + raise + return (None, []) + + # get config: pull_blob(digest) + data = json.loads(b''.join(docker.pull_blob(digest))) + + return (dateutil.parser.parse(data["created"]), + data["rootfs"]["diff_ids"]) + + except json.JSONDecodeError: + sys.exit(f"{repository}: Invalid JSON") + except (dxf.exceptions.DXFError, ValueError) as err: + sys.exit(f"{repository}: {err}") + except requests.exceptions.RequestException as err: + msg = str(err) + match = re.search(r"\(Caused by ([a-zA-Z0-9_]+)\('?[^:']*[:'] *(.*)'\)", + msg) + if match: + msg = match.group(2) + sys.exit(f"{repository}: {msg}") + except KeyError: + sys.exit(f"{repository}: missing information from registry") + + +def expand_base_image(base_name): + """ expand base image """ + match = re.match(r"\$\{?DOCKER_ACCOUNT\}?/(.+)", base_name) + if not match: + return (base_name, []) + if not docker_login["account"]: + raise ValueError("Environment variable DOCKER_ACCOUNT " + "is not defined or is empty") + base_name = docker_login["account"] + "/" + match.group(1) + options = ["--build-arg", "DOCKER_ACCOUNT=" + docker_login["account"]] + return (base_name, options) + + +def full_image_name(image_name): + """ get full image name """ + if "/" in image_name: + return image_name + if not docker_login["account"]: + raise ValueError("Environment variable DOCKER_ACCOUNT " + "is not defined or is empty") + return docker_login["account"] + "/" + image_name + + +def dockerfile_base(directory): + """ get base repository from Dockerfile """ + base = None + re_from = re.compile(r'\s*FROM\s+(\S+)', re.IGNORECASE) + try: + with open(os.path.join(directory, "Dockerfile"), "r", + encoding="utf-8") as dockerfile: + for dockerline in dockerfile: + match = re_from.match(dockerline) + if match: + base = match.group(1) + break + except OSError as err: + raise ValueError(f"Dockerfile: {err}") from err + if not base: + raise ValueError("Dockerfile: Missing FROM instruction") + return base + + +RE_CONF_LINE = re.compile(r''' +(?:| # empty line or... + (?P[^\t\#][^\t]*)| # global option or... + (?P[^\t\#][^\t]*) # name + + \t+(?P[^\t\#][^\t]*) # directory + + (?:\t+(?P[^\t\#'"-][^\t]*))? # optional base + + (?:\t+(?P['"-][^\t]*))? # optional option +) +(?:[\t ]*\#.*)? # followed by optional comment +''', re.VERBOSE) + +def get_images(image_file): + """ read images configuration file + + Format of the lines in the configuration file: + Name Directory [ Base Image] [ Build Options] + or + Global Build Options + + If the base image is not given, it is extracted from /Dockerfile. + """ + gbl_options = [] + name_set = set() + try: + lineno = 0 + with open(image_file, "r", encoding="utf-8") as img_file: + for line in img_file: + lineno += 1 + match = RE_CONF_LINE.fullmatch(line.strip()) + if not match: + sys.exit(f"{image_file} line {lineno}: " + "invalid number of fields") + if match.group('gbl_opt'): + gbl_options = shlex.split(match.group('gbl_opt')) + if match.group('name') and match.group('dir'): + name = match.group('name') + full_name = full_image_name(name) + try: + parse_repository(full_name) + except ValueError: + sys.exit(f"{image_file} line {lineno}: " + f"invalid image name '{full_name}'") + if full_name in name_set: + sys.exit(f"{image_file}: " + f"multiple entries for {full_name}") + name_set.add(full_name) + directory = match.group('dir') + if not os.path.isdir(directory): + sys.exit(f"{image_file} line {lineno}: " + f"unknown directory '{directory}'") + base = match.group('base') + if not base: # extract base repo from Dockerfile + base = dockerfile_base(directory) + (base, options) = expand_base_image(base) + options += gbl_options + if match.group('opt'): + options += shlex.split(match.group('opt')) + images.append({"name": name, "dir": directory, + "base": base, "options": options}) + except OSError as err: + sys.exit(f"Can't read images file: {err}") + except ValueError as err: + sys.exit(f"{image_file} line {lineno}: {err}") + if not images: + sys.exit("Empty image configuration") + + +def init_base_images(): + """ initialize base image data structure """ + for image in images: + base_name = image["base"] + if base_name not in base_images: + base_images[base_name] = {"layer": False} + base_images["scratch"] = {"layer": None} + base_images["NONE"] = {"layer": None} + + +def mtime_tree(directory): + """ get modification time of a directory tree """ + mtime = 0 + for root, _, filenames in os.walk(directory): + mtime = max(mtime, os.stat(root).st_mtime) + for fname in filenames: + mtime = max(mtime, os.stat(os.path.join(root, fname)).st_mtime) + return mtime + + +def needs_rebuild(image): + """ check if an image needs rebuilding """ + # update base_image layer, if empty + base_img = base_images[image["base"]] + if base_img["layer"] is False: + _, layers = get_time_layers(image["base"]) + # store last layer + if layers: + base_img["layer"] = layers[-1] + else: + sys.exit(f"Missing base image: {image['base']}") + + # get image data + full_name = full_image_name(image["name"]) + itime, layers = get_time_layers(full_name) + if layers and full_name in base_images: # image is a base image + base_images[full_name]["layer"] = layers[-1] + + # check if base image has changed + if not layers: + return "Image missing in repository" + if base_img["layer"] and base_img["layer"] not in layers: + return "Base image has changed" + + # check if build directory has changed, needs full git history + env = os.environ.copy() + env["LC_ALL"] = "C" + try: + # check if git repository is up-to-date + proc = subprocess.run(["git", "-C", image["dir"], "status", + "--porcelain", "--", "."], + capture_output=True, + check=False, + env=env, + universal_newlines=True) + if proc.returncode != 0 and "not a git repository" not in proc.stderr: + # Fatal error + sys.exit(f"{image['name']}: Can't get git status: " + \ + proc.stderr.rstrip('\r\n')) + if proc.returncode != 0 or proc.stdout.rstrip('\r\n'): + # Non-fatal error or changes: use modification date of the files + mtime = mtime_tree(image["dir"]) + rebuild_reason = "Files in docker context more recent than image" + else: + # clean git repository: use "git log" to get commit time + proc = subprocess.run(["git", "-C", image["dir"], "log", "-n", "1", + "--pretty=tformat:%ct", "--", "."], + capture_output=True, + check=True, + env=env, + universal_newlines=True) + mtime = int(proc.stdout.strip()) + rebuild_reason = "Git change more recent than image" + except OSError as err: + sys.exit(f"Can't run git: {err}") + except subprocess.CalledProcessError as err: + sys.exit(f"{image['name']}: Can't get commit date: " + \ + err.stderr.rstrip('\r\n')) + except ValueError as err: + sys.exit(f"{image['name']}: Can't get commit date: {err}") + + return rebuild_reason if mtime > itime.timestamp() else None + + +def build(image): + """ build image """ + full_name = full_image_name(image["name"]) + try: + subprocess.run(["docker", "buildx", "build"] + image["options"] + \ + ["--push", "--tag", full_name, image["dir"]], + check=True) + except OSError as err: + sys.exit(f"Can't run docker: {err}") + except subprocess.CalledProcessError as err: + sys.exit(err.returncode) + print() + + if full_name in base_images: # just modified a base image + _, layers = get_time_layers(full_name) + # store last layer + if layers: + base_images[full_name]["layer"] = layers[-1] + else: + sys.exit(f"{image['name']}: Can't get image layers") + + +def xor(*params): + """ logical xor """ + result = False + for arg in params: + result = result != bool(arg) + return result + + +# main +args = parser.parse_args() +sys.stdout.reconfigure(line_buffering=True) + +docker_login = {"account": os.environ.get("DOCKER_ACCOUNT"), + "password": os.environ.pop("DOCKER_PASSWORD", None)} +if docker_login["account"]: + docker_login["account"] = docker_login["account"].rstrip("/") + try: + docker_login["registry"], docker_login["user"], *_ = \ + parse_repository(docker_login["account"] + "/dummy") + except ValueError as err_info: + sys.exit(f"DOCKER_ACCOUNT={docker_login['account']}: {err_info}") + if not docker_login["user"]: + sys.exit(f"DOCKER_ACCOUNT={docker_login['account']}: missing user") +else: + docker_login["registry"] = docker_login["user"] = None + +if args.dir: + try: + os.chdir(os.path.join(*args.dir)) + except OSError as err_info: + sys.exit(f"Can't change directory: {err_info}") +get_images(args.file) +init_base_images() + +# check arguments +all_inames = {img["name"] for img in images}.union(base_images.keys()) +for iname in args.image: + if iname not in all_inames: + sys.exit(f"Image {iname} not found in '{args.file}' configuration file") + +# rebuild images +for img in images: + if xor(args.all, img["name"] in args.image or img["base"] in args.image): + # pragma pylint: disable=invalid-name + reason = "Rebuild triggered by command line" + else: + reason = needs_rebuild(img) + if reason: + print(f"*** {img['name']}\nReason: {reason}\n") + if not args.dry_run: + build(img) diff --git a/.github/bin/requirements.txt b/.github/bin/requirements.txt new file mode 100644 index 0000000..01c8180 --- /dev/null +++ b/.github/bin/requirements.txt @@ -0,0 +1,2 @@ +python-dateutil +python-dxf>=11 diff --git a/.github/docker_build.md b/.github/docker_build.md new file mode 100644 index 0000000..3c9ad0d --- /dev/null +++ b/.github/docker_build.md @@ -0,0 +1,118 @@ +# Docker Build System + +The regular Docker build system has the disadvantage, +that only the cache system prevents a repetitive rebuild. +But when running the build in a VM this cache is initially +empty, resulting in a rebuild of all images on every run. +This not only increases the CPU load of the VM, but also +creates a lot of updated Docker images, that differ only +in the timestamps of the files. + +This Docker build system wants to improve this situation. +It rebuilds only, when the base image or the build context +has changed. There are some other situations, where an +image needs to be recreated, that are not detected. +This mainly affects installation packages and external +files, that got an update. Then a manual trigger is needed. + + +## Build Tool + +The `docker_build` tool reads a configuration file and +then starts building images with `docker buildx build`. + +If `docker_build` is launched without arguments, it checks +all configured images for an update of the base image. +Additionally it checks, if `git` shows an update of +the directory containing the docker build context. +When at least one of these conditions is met, the tool +starts a rebuild of that image. + +When `docker_build` is run with some image names as +arguments, then these images are additionally built. +When using a base image name as an argument, then all +images with that base image are rebuilt. + +When using the option -a/--all, all images are forcibly +rebuild, except those specified on the command line. + + +## Configuration File + +The build tool reads a configuration file, by default +'docker_images', located in the current directory. For each +target image, it contains its name, its context directory +and optionally the base image and some build options. + +The fields are separated by one or more \ characters. +Comments start with a `#` as its first field character. +An empty line or an all-comments line is ignored. + +A line with only one field is used for build options, +which are effective from that point until they are redefined. +This global build option and an optional image specific option +are combined and sent to the `docker buildx` command. + +Here an example: + +``` +# Name Directory [Base Image] [Build Options] + +--platform=linux/arm64,linux/amd64 # global options + +alpine-1 alpine-1 alpine --image-specific-option +alpine-1:test alpine-1a --another-image-specific-option +``` + +If the base image is `NONE` (all caps), the target +image is not checked against its real base image. +Changes to the base image will then not cause a rebuild. + +The target image may contain the full name, in which +case it will contain one or more '/' characters. + +Another option is to specify only the last part of the +image name. Then `docker_build` uses the `DOCKER_ACCOUNT` +environment variable as its initial part. For example, an +DOCKER_ACCOUNT value of "ghcr.io/b-ehlers" plus the image +name of "alpine-1" results in "ghcr.io/b-ehlers/alpine-1". + +This method is not applied to the base images, they always +have to contain the complete name. + +But there is a workaround. + +If the base image name starts with `$DOCKER_ACCOUNT` +or `${DOCKER_ACCOUNT}` the variable DOCKER_ACCOUNT +gets replaced by its value from the environment. +In the Dockerfile the variable must be declared by a +`ARG DOCKER_ACCOUNT` instruction. A Dockerfile would +then start with: + +``` +ARG DOCKER_ACCOUNT +FROM $DOCKER_ACCOUNT/base-image +``` + + +## Workflow Definition + +[GitHub Actions](https://docs.github.com/en/actions) +uses YAML files in the .github/workflows directory +to define, which tasks should be run. + +Before `docker_build` can be run the following steps +need to be done: + +* Check out the repository code +* Set up QEMU (for multi-arch building) +* Set up Docker Buildx +* Login to the Container Registry +* Install python requirements + +Then `docker_build` can be executed, +normally without any arguments. + +But what, when an image build needs to be forced? +For that, run the action manually and enter the list +of images, separated by spaces, into the input field. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..d5f1192 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,59 @@ +name: Build Docker images and upload to ghcr.io +on: + push: + branches: + - main + - master + schedule: + - cron: '37 7 * * 3' + workflow_dispatch: + inputs: + images: + description: 'List of images to be built' + required: false + type: string + +jobs: + docker-images: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + # https://github.com/marketplace/actions/checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up QEMU + # https://github.com/marketplace/actions/docker-setup-qemu + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + # https://github.com/marketplace/actions/docker-setup-buildx + uses: docker/setup-buildx-action@v2 + - name: Login to GitHub Container Registry + # https://github.com/marketplace/actions/docker-login + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + # DockerHub: + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Install python requirements + run: python3 -m pip install --requirement .github/bin/requirements.txt + - name: Build and push images + env: + DOCKER_ACCOUNT: ghcr.io/${{ github.repository_owner }} + # DOCKER_PASSWORD is optional, only needed for private repositories + # DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + # DockerHub: + # DOCKER_ACCOUNT: ${{ secrets.DOCKERHUB_USERNAME }} + # DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + # + IMAGES: ${{ inputs.images }} + run: | + set -f + set -- $IMAGES + set +f + # use option --dir to use a subdirectory as a docker base dir + python3 "$GITHUB_WORKSPACE/.github/bin/docker_build" --dir "docker" "$@" diff --git a/docker/docker_images b/docker/docker_images new file mode 100644 index 0000000..f3d59ea --- /dev/null +++ b/docker/docker_images @@ -0,0 +1,43 @@ +# Name Directory [Base Image] [Build Options] + +--platform=linux/amd64 # global options + +# chromium: base image is updated too often, use manual build on base updates +chromium chromium NONE +# ipterm: base image debian:jessie too old +#ipterm-base ipterm/base +#ipterm ipterm/cli +#webterm ipterm/web +jupyter:v2 jupyter +jupyter27:v2 jupyter-2.7 +# kalilinux: base image is updated too often, use manual build on base updates +kalilinux kalilinux NONE +mikrotik-winbox mikrotik-winbox +# ostinato-wireshark: failed to build: mesa-dri-swrast - no such package +#ostinato-wireshark ostinato-wireshark +openvswitch openvswitch +ovs-snmp ovs-snmp --platform=linux/arm64 +pyats pyats +ubuntu:focal ubuntu + +# +# The following images are not used by any appliance +# + +# Replaced by https://github.com/adosztal/gns3-containers +#network_automation network_automation +#python-go-perl-php python-go-perl-php + +# AJ Nouri +#endhost endhost +#haproxy haproxy +#network_automation_pycharm network_automation_pycharm +#openvswitch28 openvswitch28 + +# GNS3 Developers +#dhcp dhcp +#snmpsimulator snmpsimulator + +# Tests +#iou iou +#xeyes xeyes From 12289975dea6c23fb71a092d95226c75cbddcb86 Mon Sep 17 00:00:00 2001 From: Bernhard Ehlers Date: Sat, 27 May 2023 17:41:05 +0200 Subject: [PATCH 03/13] Docker build system: upload to DockerHub --- .github/workflows/build.yml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d5f1192..cf678b1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: Build Docker images and upload to ghcr.io +name: Build Docker images and upload to DockerHub on: push: branches: @@ -32,19 +32,23 @@ jobs: # https://github.com/marketplace/actions/docker-login uses: docker/login-action@v2 with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - # DockerHub: - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + # GitHub Container Registry: + # registry: ghcr.io + # username: ${{ github.repository_owner }} + # password: ${{ secrets.GITHUB_TOKEN }} + # DockerHub: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Install python requirements run: python3 -m pip install --requirement .github/bin/requirements.txt - name: Build and push images env: - DOCKER_ACCOUNT: ghcr.io/${{ github.repository_owner }} + DOCKER_ACCOUNT: ${{ secrets.DOCKERHUB_USERNAME }} # DOCKER_PASSWORD is optional, only needed for private repositories + # GitHub Container Registry: + # DOCKER_ACCOUNT: ghcr.io/${{ github.repository_owner }} # DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} # DockerHub: # DOCKER_ACCOUNT: ${{ secrets.DOCKERHUB_USERNAME }} From 502400b95246f8efdb5f44eb0c5caf5ea1e9a1f6 Mon Sep 17 00:00:00 2001 From: Bernhard Ehlers Date: Sat, 27 May 2023 17:56:56 +0200 Subject: [PATCH 04/13] mikrotik-winbox: add a dummy layer to get an updated container timestamp --- docker/mikrotik-winbox/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/mikrotik-winbox/Dockerfile b/docker/mikrotik-winbox/Dockerfile index 061f171..efc1a78 100644 --- a/docker/mikrotik-winbox/Dockerfile +++ b/docker/mikrotik-winbox/Dockerfile @@ -1,2 +1,6 @@ FROM alexhorner/winbox-dockerised + +# add a dummy layer to get an updated container timestamp +RUN /bin/true + ENV VNC_BUILTIN_DISABLED=true From 4748e168bd4efbe4d1161397b30167b65312cffa Mon Sep 17 00:00:00 2001 From: grossmj Date: Sun, 28 May 2023 13:58:47 +0800 Subject: [PATCH 05/13] Rename build.yml to build-docker-images.yml --- .github/workflows/{build.yml => build-docker-images.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{build.yml => build-docker-images.yml} (100%) diff --git a/.github/workflows/build.yml b/.github/workflows/build-docker-images.yml similarity index 100% rename from .github/workflows/build.yml rename to .github/workflows/build-docker-images.yml From 257b5f154faede9585d2f7aa26783ef10a885043 Mon Sep 17 00:00:00 2001 From: Bernhard Ehlers Date: Sun, 28 May 2023 17:49:57 +0200 Subject: [PATCH 06/13] Docker build: Update workflow --- .github/workflows/build-docker-images.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index cf678b1..fd20e7e 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -32,26 +32,26 @@ jobs: # https://github.com/marketplace/actions/docker-login uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} # GitHub Container Registry: # registry: ghcr.io # username: ${{ github.repository_owner }} # password: ${{ secrets.GITHUB_TOKEN }} + # # DockerHub: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Install python requirements run: python3 -m pip install --requirement .github/bin/requirements.txt - name: Build and push images env: - DOCKER_ACCOUNT: ${{ secrets.DOCKERHUB_USERNAME }} # DOCKER_PASSWORD is optional, only needed for private repositories + # # GitHub Container Registry: # DOCKER_ACCOUNT: ghcr.io/${{ github.repository_owner }} # DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + # # DockerHub: - # DOCKER_ACCOUNT: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKER_ACCOUNT: ${{ secrets.DOCKERHUB_USERNAME }} # DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} # IMAGES: ${{ inputs.images }} From 3386ba0e08c2c3d6e76f5b40eb9f3573c9835ef8 Mon Sep 17 00:00:00 2001 From: grossmj Date: Mon, 29 May 2023 17:31:00 +0800 Subject: [PATCH 07/13] Temporally Run build-docker-images.yml in PR --- .github/workflows/build-docker-images.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index fd20e7e..1901069 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -4,6 +4,9 @@ on: branches: - main - master + pull_request: + branches: + - "**" schedule: - cron: '37 7 * * 3' workflow_dispatch: From b954ae53f04f8880d003417cd7de25957a1800b1 Mon Sep 17 00:00:00 2001 From: grossmj Date: Mon, 29 May 2023 17:38:05 +0800 Subject: [PATCH 08/13] Push on ghcr.io registry --- .github/workflows/build-docker-images.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index 1901069..cb297c6 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -36,13 +36,13 @@ jobs: uses: docker/login-action@v2 with: # GitHub Container Registry: - # registry: ghcr.io - # username: ${{ github.repository_owner }} - # password: ${{ secrets.GITHUB_TOKEN }} + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} # # DockerHub: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + #username: ${{ secrets.DOCKERHUB_USERNAME }} + #password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Install python requirements run: python3 -m pip install --requirement .github/bin/requirements.txt - name: Build and push images @@ -50,11 +50,11 @@ jobs: # DOCKER_PASSWORD is optional, only needed for private repositories # # GitHub Container Registry: - # DOCKER_ACCOUNT: ghcr.io/${{ github.repository_owner }} + DOCKER_ACCOUNT: ghcr.io/${{ github.repository_owner }} # DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} # # DockerHub: - DOCKER_ACCOUNT: ${{ secrets.DOCKERHUB_USERNAME }} + # DOCKER_ACCOUNT: ${{ secrets.DOCKERHUB_USERNAME }} # DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} # IMAGES: ${{ inputs.images }} From 72c91376f352af40199596b26a322b9c87f31d3d Mon Sep 17 00:00:00 2001 From: grossmj Date: Mon, 29 May 2023 17:55:13 +0800 Subject: [PATCH 09/13] Force DOCKER_ACCOUNT to be lower case --- .github/bin/docker_build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/bin/docker_build b/.github/bin/docker_build index 0c2bacd..a962006 100755 --- a/.github/bin/docker_build +++ b/.github/bin/docker_build @@ -412,7 +412,7 @@ def xor(*params): args = parser.parse_args() sys.stdout.reconfigure(line_buffering=True) -docker_login = {"account": os.environ.get("DOCKER_ACCOUNT"), +docker_login = {"account": os.environ.get("DOCKER_ACCOUNT").lower(), "password": os.environ.pop("DOCKER_PASSWORD", None)} if docker_login["account"]: docker_login["account"] = docker_login["account"].rstrip("/") From da2d2d49d523bb5956d0a09c773155aef6ba6fd1 Mon Sep 17 00:00:00 2001 From: grossmj Date: Mon, 29 May 2023 19:32:57 +0800 Subject: [PATCH 10/13] Handle if environment variable DOCKER_ACCOUNT is undefined --- .github/bin/docker_build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/bin/docker_build b/.github/bin/docker_build index a962006..be46b9a 100755 --- a/.github/bin/docker_build +++ b/.github/bin/docker_build @@ -412,7 +412,7 @@ def xor(*params): args = parser.parse_args() sys.stdout.reconfigure(line_buffering=True) -docker_login = {"account": os.environ.get("DOCKER_ACCOUNT").lower(), +docker_login = {"account": os.environ.get("DOCKER_ACCOUNT", "").lower(), "password": os.environ.pop("DOCKER_PASSWORD", None)} if docker_login["account"]: docker_login["account"] = docker_login["account"].rstrip("/") From 7abc5f9c5a36a48222d1f039617c2eb4ff9f62f0 Mon Sep 17 00:00:00 2001 From: grossmj Date: Mon, 29 May 2023 19:53:39 +0800 Subject: [PATCH 11/13] Set 'packages' scope permission to write for GITHUB_TOKEN --- .github/workflows/build-docker-images.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index cb297c6..cc90e11 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -19,6 +19,8 @@ on: jobs: docker-images: runs-on: ubuntu-latest + permissions: + packages: write steps: - name: Check out repository code # https://github.com/marketplace/actions/checkout From 70fbe6a1b9d87442ddb3f39de35e3b3b0e736f2b Mon Sep 17 00:00:00 2001 From: grossmj Date: Mon, 29 May 2023 20:21:45 +0800 Subject: [PATCH 12/13] Revert "Temporally Run build-docker-images.yml in PR" This reverts commit 3386ba0e08c2c3d6e76f5b40eb9f3573c9835ef8. --- .github/workflows/build-docker-images.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index cc90e11..5d8d204 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -4,9 +4,6 @@ on: branches: - main - master - pull_request: - branches: - - "**" schedule: - cron: '37 7 * * 3' workflow_dispatch: From 141832faf4b266ec3f88efe5c3d33d02ebd5740a Mon Sep 17 00:00:00 2001 From: grossmj Date: Mon, 29 May 2023 20:23:27 +0800 Subject: [PATCH 13/13] Push on Docker Hub --- .github/workflows/build-docker-images.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index 5d8d204..cf7a9d2 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -35,13 +35,13 @@ jobs: uses: docker/login-action@v2 with: # GitHub Container Registry: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} + # registry: ghcr.io + # username: ${{ github.repository_owner }} + # password: ${{ secrets.GITHUB_TOKEN }} # # DockerHub: - #username: ${{ secrets.DOCKERHUB_USERNAME }} - #password: ${{ secrets.DOCKERHUB_TOKEN }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Install python requirements run: python3 -m pip install --requirement .github/bin/requirements.txt - name: Build and push images @@ -49,11 +49,11 @@ jobs: # DOCKER_PASSWORD is optional, only needed for private repositories # # GitHub Container Registry: - DOCKER_ACCOUNT: ghcr.io/${{ github.repository_owner }} + # DOCKER_ACCOUNT: ghcr.io/${{ github.repository_owner }} # DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} # # DockerHub: - # DOCKER_ACCOUNT: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKER_ACCOUNT: ${{ secrets.DOCKERHUB_USERNAME }} # DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} # IMAGES: ${{ inputs.images }}