diff --git a/experimental/kubernetes/.gitignore b/experimental/kubernetes/.gitignore new file mode 100644 index 0000000000..cce463ad5e --- /dev/null +++ b/experimental/kubernetes/.gitignore @@ -0,0 +1,2 @@ +built/ +config/ diff --git a/experimental/kubernetes/0001-Read-corda-rev-from-environment-var.patch b/experimental/kubernetes/0001-Read-corda-rev-from-environment-var.patch new file mode 100644 index 0000000000..cc078ace48 --- /dev/null +++ b/experimental/kubernetes/0001-Read-corda-rev-from-environment-var.patch @@ -0,0 +1,25 @@ +From b24ad7306e9e3c7faff032aa20fc7b42848ec815 Mon Sep 17 00:00:00 2001 +From: Thomas Schroeter +Date: Fri, 16 Feb 2018 11:07:58 +0000 +Subject: [PATCH] Read corda rev from environment var + +--- + build.gradle | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/build.gradle b/build.gradle +index 206335645f..0ffab4eac1 100644 +--- a/build.gradle ++++ b/build.gradle +@@ -109,7 +109,7 @@ plugins { + } + + ext { +- corda_revision = org.ajoberstar.grgit.Grgit.open(file('.')).head().id ++ corda_revision = System.getenv("rev") + } + + apply plugin: 'project-report' +-- +2.14.3 (Apple Git-98) + diff --git a/experimental/kubernetes/Readme.md b/experimental/kubernetes/Readme.md new file mode 100644 index 0000000000..a3e1daf8d6 --- /dev/null +++ b/experimental/kubernetes/Readme.md @@ -0,0 +1,91 @@ +# Compatibility Testing + +We can build any release we want to test, be it for performance or for compatibility, using the approach outlined below. The test environment includes a driver that lets the nodes issue and pay to a random peer. This way, the notary is put under stress and all combinations of deployed versions are tested eventually. + +We need to peak into the vaults to make sure the data is in good shape (TODO). + +#### The tools + +This package contains scripts and configuration files to +* build and publish the components of a Corda network, +* generate YAML files declaring a deployment for Kubernetes, +* deploy to a cluster, and +* remove the deployment. + +#### Preparation +To run locally you need to have kubectl and docker-cli setup and pointing to a cluster and you need to know your namespace. However, we can move the tasks to team-city, so installing kubectl locally is not required any more. + + +#### Parameters + +| Name | Description | +|------------------------|-------------------------------------------------------------------------------| +| revison | What to build, Git commit-ish, e.g. branch, tag, SHA1 | +| namespace | The kubernetes namespace to deploy to (TODO can be autogenerated) | +| storage-class | The storage class used to provision volumes via persistent volume claims | +| docker-registry | The docker registry to push and pull container images | +| kubernetes cluster | The kubernetes context to use | + +#### Assumptions + +- You (or the build environment) have access to a kubernetes cluster +- build environment has push access to docker registry +- kubernetes cluster has pull access to docker registry via `regsecret` secret (TODO: setup howto) +- storage can be provisioned (TODO: howto setup) + +#### Build the container images from a git commit(-ish), e.g. branch-name, tag-name or SHA1. + +``` +./bin/build-and-publish/node.sh [] +./bin/build-and-publish/cordapps.sh [] +./bin/build-and-publish/doorman.sh [] +./bin/build-and-publish/notary.sh [] + ./bin/build-and-publish/healthcheck.sh origin/thomas-compatibility-testing +``` + +The build is carried out in a new temporary git worktree that is removed after +successful builds. + +This appends to a file `built/node-images.txt`. To build a couple of releases +you can specify the releases in a file (one per line) and use +``` +for r in $(cat releases.txt); do ./bin/build-and-publish/node.sh $r; done +``` + +#### Generate the deployment config files using +``` +python generate_config.py -n -s +``` + +#### Deploy +You can either use `bin/start.sh` to boot and `bin/delete-all.sh` to tear down the network, +``` +bin/start.sh +bin/delete-all.sh +``` + +or apply the config manually +``` +kubectl apply -f config/ +kubectl scale statefulset --replicas=1 +``` + +### Notes + +The notary jar is the same as the one of the regular node, but we currently include the +notary init script for the interaction with the doorman and we include the node.conf +in the container images. In the future we could use config maps for both the init +script and the node.conf. + + +### TODO +- extend the workload to test exercise notary changes and contract upgrades +- write the generate-config part in Kotlin +- design an write better tooling in Kotlin +- multi notary network +- generate distributed notary singular identity +- perhaps config maps support for config +- publish metrics from the load generators, e.g. error count +- investigate Helm +- add more databases +- use deployments diff --git a/experimental/kubernetes/bin/bootstrap.sh b/experimental/kubernetes/bin/bootstrap.sh new file mode 100755 index 0000000000..1bc0d8fc4b --- /dev/null +++ b/experimental/kubernetes/bin/bootstrap.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +set -eux + +doorman_name="doorman" +notary_name="notary" + +kubectl create -f config/doorman/service.yml +kubectl create -f config/notary/service.yml + +kubectl create -f config/doorman/pod-init.yml + +wait_for_doorman() { + set +e + # TODO: use wait-for + while :; do + sleep 5 + kubectl logs $doorman_name | grep 'services started on doorman' + if [[ $? -eq 0 ]]; then + break + fi + done + set -e +} + +wait_for_doorman + +kubectl cp ${doorman_name}:/data/doorman/certificates/distribute-nodes/network-root-truststore.jks . +kubectl delete secret truststore-3.0.0 || true +kubectl create secret generic truststore-3.0.0 --from-file=network-root-truststore.jks +rm network-root-truststore.jks + +set +e + +kubectl create -f config/notary/pod-init.yml + +while :; do + sleep 5 + kubectl logs ${notary_name} | grep 'DONE_BOOTSTRAPPING' + if [[ $? -eq 0 ]]; then + break + fi +done + +set -e + +kubectl cp ${notary_name}:/data/notary-node-info . +kubectl cp notary-node-info ${doorman_name}:/data/ +rm notary-node-info + +kubectl delete po ${notary_name} ${doorman_name} + +# TODO: wait for containers to be terminated, e.g. with grep on kubectl get po + +while :; do + sleep 5 + n=$(kubectl get po | wc -l) + if [[ n -eq 0 ]]; then + break + fi +done + +kubectl create -f config/doorman/pod.yml + +wait_for_doorman + +kubectl create -f config/notary/pod.yml diff --git a/experimental/kubernetes/bin/build-and-publish/bft-notary.sh b/experimental/kubernetes/bin/build-and-publish/bft-notary.sh new file mode 100755 index 0000000000..c255a41d41 --- /dev/null +++ b/experimental/kubernetes/bin/build-and-publish/bft-notary.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +set -eux + +target_rev=${1:-"HEAD"} +container_name=${2:-"bft-notary"} +registry=${CONTAINER_REGISTRY:-"ctesting.azurecr.io"} + +export rev=$(git rev-parse $target_rev) +rev_short=$(git rev-parse --short $target_rev) + +workspace=$(mktemp -d -t kubeform-XXX) + + +git worktree add $workspace $rev +cp -r build-contexts $workspace + +docker_cmd="docker" +if [ "$(uname)" = "Linux" ]; then + docker_cmd="sudo docker" +fi + +patch="0001-Read-corda-rev-from-environment-var.patch" +cp $patch $workspace + +container_image="${registry}/r3/${container_name}:$rev" + +( + cd $workspace + git apply $patch + ./gradlew --debug jar + JAR=$(ls -S node/capsule/build/libs | head -n1) + cp node/capsule/build/libs/$JAR build-contexts/bft-notary/corda.jar + + cd build-contexts/bft-notary + ${docker_cmd} build -t $container_image . + ${docker_cmd} push $container_image +) + +mkdir -p built +echo "r3-$rev_short $container_image" > built/bft-notary.txt + +rm -rf $workspace + +git worktree prune diff --git a/experimental/kubernetes/bin/build-and-publish/cordapps.sh b/experimental/kubernetes/bin/build-and-publish/cordapps.sh new file mode 100755 index 0000000000..c3222685e7 --- /dev/null +++ b/experimental/kubernetes/bin/build-and-publish/cordapps.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +set -eux + +target_rev=${1:-"HEAD"} +registry=${CONTAINER_REGISTRY:-"ctesting.azurecr.io"} + +workspace=$(mktemp -d -t kubeform-XXX) + +export rev=$(git rev-parse $target_rev) +rev_short=$(git rev-parse --short $target_rev) + +git worktree add $workspace $rev +cp -r build-contexts $workspace + +docker_cmd="docker" +if [ "$(uname)" = "Linux" ]; then + docker_cmd="sudo docker" +fi + +patch="0001-Read-corda-rev-from-environment-var.patch" +cp $patch $workspace + +load_gen_image="${registry}/r3/load-gen-cordapps:$rev" + +( + cd $workspace + git apply $patch + + # Build Healthcheck Cordapps + ./gradlew finance:jar + JAR=$(ls -S finance/build/libs | head -n1) + cp finance/build/libs/$JAR build-contexts/cordapps/ + + cd build-contexts/cordapps + ${docker_cmd} build -t $load_gen_image . + ${docker_cmd} push $load_gen_image +) + +mkdir -p built +echo "$load_gen_image" > built/cordapps.txt + +rm -rf $workspace + +git worktree prune diff --git a/experimental/kubernetes/bin/build-and-publish/doorman.sh b/experimental/kubernetes/bin/build-and-publish/doorman.sh new file mode 100755 index 0000000000..bffd162db9 --- /dev/null +++ b/experimental/kubernetes/bin/build-and-publish/doorman.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +set -eux + +target_rev=${1:-"HEAD"} +registry=${CONTAINER_REGISTRY:-"ctesting.azurecr.io"} + +workspace=$(mktemp -d -t kubeform-XXX) + +export rev=$(git rev-parse $target_rev) +rev_short=$(git rev-parse --short $target_rev) + +git worktree add $workspace $rev +cp -r build-contexts $workspace + +docker_cmd="docker" +if [ "$(uname)" = "Linux" ]; then + docker_cmd="sudo docker" +fi + +patch="0001-Read-corda-rev-from-environment-var.patch" +cp $patch $workspace + +container_image="${registry}/r3/doorman:$rev" + +( + cd $workspace + git apply $patch + ./gradlew network-management:capsule:buildDoormanJAR + JAR=$(ls -S network-management/capsule/build/libs/doorman-*.jar | head -n1) + cp $JAR build-contexts/doorman/doorman.jar + + cd build-contexts/doorman + ${docker_cmd} build -t $container_image . + ${docker_cmd} push $container_image +) + +mkdir -p built +echo "doorman-r3-$rev_short $container_image" > built/doorman.txt + +rm -rf $workspace + +git worktree prune diff --git a/experimental/kubernetes/bin/build-and-publish/ha-node.sh b/experimental/kubernetes/bin/build-and-publish/ha-node.sh new file mode 100755 index 0000000000..bf1c8cd46b --- /dev/null +++ b/experimental/kubernetes/bin/build-and-publish/ha-node.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +set -eux + +target_rev=${1:-"HEAD"} +container_name=${2:-"ha-node"} +registry=${CONTAINER_REGISTRY:-"ctesting.azurecr.io"} + +export rev=$(git rev-parse $target_rev) +rev_short=$(git rev-parse --short $target_rev) + +workspace=$(mktemp -d -t kubeform-XXX) + + +git worktree add $workspace $rev +cp -r build-contexts $workspace + +docker_cmd="docker" +if [ "$(uname)" = "Linux" ]; then + docker_cmd="sudo docker" +fi + +patch="0001-Read-corda-rev-from-environment-var.patch" +cp $patch $workspace + +container_image="${registry}/r3/${container_name}:$rev" + +( + cd $workspace + git apply $patch + ./gradlew jar + JAR=$(ls -S node/capsule/build/libs | head -n1) + cp node/capsule/build/libs/$JAR build-contexts/ha-node/corda.jar + + cd build-contexts/ha-node + ${docker_cmd} build -t $container_image . + ${docker_cmd} push $container_image +) + +mkdir -p built +echo "$container_name-$rev_short $container_image" > built/ha-node-image.txt + +rm -rf $workspace + +git worktree prune diff --git a/experimental/kubernetes/bin/build-and-publish/healthcheck.sh b/experimental/kubernetes/bin/build-and-publish/healthcheck.sh new file mode 100755 index 0000000000..f5f90b1e05 --- /dev/null +++ b/experimental/kubernetes/bin/build-and-publish/healthcheck.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +set -eux + +target_rev=${1:-"HEAD"} +registry=${CONTAINER_REGISTRY:-"ctesting.azurecr.io"} + +workspace=$(mktemp -d -t kubeform-XXX) + +export rev=$(git rev-parse $target_rev) +rev_short=$(git rev-parse --short $target_rev) + +git worktree add $workspace $rev +cp -r build-contexts $workspace + +docker_cmd="docker" +if [ "$(uname)" = "Linux" ]; then + docker_cmd="sudo docker" +fi + +patch="0001-Read-corda-rev-from-environment-var.patch" +cp $patch $workspace + +load_gen_image="${registry}/r3/load-generator:$rev" + +( + cd $workspace + git apply $patch + + # Build Healthcheck + ./gradlew tools:notaryhealthcheck:shadowJar + cp tools/notaryhealthcheck/build/libs/shadow.jar build-contexts/load-generator/app.jar + + cd build-contexts/load-generator + ${docker_cmd} build -t $load_gen_image . + ${docker_cmd} push $load_gen_image +) + +mkdir -p built +echo "$load_gen_image" > built/load_generator.txt + +rm -rf $workspace + +git worktree prune diff --git a/experimental/kubernetes/bin/build-and-publish/hot-warm.sh b/experimental/kubernetes/bin/build-and-publish/hot-warm.sh new file mode 100755 index 0000000000..a4e096ada9 --- /dev/null +++ b/experimental/kubernetes/bin/build-and-publish/hot-warm.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +set -eux + +target_rev=${1:-"HEAD"} +container_name=${2:-"hot-warm-node"} +registry=${CONTAINER_REGISTRY:-"ctesting.azurecr.io"} + +export rev=$(git rev-parse $target_rev) +rev_short=$(git rev-parse --short $target_rev) + +workspace=$(mktemp -d -t kubeform-XXX) + + +git worktree add $workspace $rev +cp -r build-contexts $workspace + +docker_cmd="docker" +if [ "$(uname)" = "Linux" ]; then + docker_cmd="sudo docker" +fi + +patch="0001-Read-corda-rev-from-environment-var.patch" +cp $patch $workspace + +container_image="${registry}/r3/${container_name}:$rev" + +( + cd $workspace + git apply $patch + ./gradlew jar + JAR=$(ls -S node/capsule/build/libs | head -n1) + cp node/capsule/build/libs/$JAR build-contexts/hot-warm/corda.jar + + cd build-contexts/hot-warm + ${docker_cmd} build -t $container_image . + ${docker_cmd} push $container_image +) + +mkdir -p built +echo "$container_name-$rev_short $container_image" > built/hot-warm-image.txt + +rm -rf $workspace + +git worktree prune diff --git a/experimental/kubernetes/bin/build-and-publish/node.sh b/experimental/kubernetes/bin/build-and-publish/node.sh new file mode 100755 index 0000000000..3e25961c77 --- /dev/null +++ b/experimental/kubernetes/bin/build-and-publish/node.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +set -eux + +target_rev=${1:-"HEAD"} +container_name=${2:-"r3-corda"} +registry=${CONTAINER_REGISTRY:-"ctesting.azurecr.io"} + +export rev=$(git rev-parse $target_rev) +rev_short=$(git rev-parse --short $target_rev) + +workspace=$(mktemp -d -t kubeform-XXX) + + +git worktree add $workspace $rev +cp -r build-contexts $workspace + +docker_cmd="docker" +if [ "$(uname)" = "Linux" ]; then + docker_cmd="sudo docker" +fi + +patch="0001-Read-corda-rev-from-environment-var.patch" +cp $patch $workspace + +container_image="${registry}/r3/${container_name}:$rev" + +( + cd $workspace + git apply $patch + ./gradlew jar + JAR=$(ls -S node/capsule/build/libs | head -n1) + cp node/capsule/build/libs/$JAR build-contexts/node/corda.jar + + cd build-contexts/node + ${docker_cmd} build -t $container_image . + ${docker_cmd} push $container_image +) + +mkdir -p built +echo "r3-$rev_short $container_image" >> built/node-images.txt + +rm -rf $workspace + +git worktree prune diff --git a/experimental/kubernetes/bin/build-and-publish/notary.sh b/experimental/kubernetes/bin/build-and-publish/notary.sh new file mode 100755 index 0000000000..ea42782b38 --- /dev/null +++ b/experimental/kubernetes/bin/build-and-publish/notary.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +set -eux + +target_rev=${1:-"HEAD"} +registry=${CONTAINER_REGISTRY:-"ctesting.azurecr.io"} + +workspace=$(mktemp -d -t kubeform-XXX) + +export rev=$(git rev-parse $target_rev) +rev_short=$(git rev-parse --short $target_rev) + +git worktree add $workspace $rev +cp -r build-contexts $workspace + +docker_cmd="docker" +if [ "$(uname)" = "Linux" ]; then + docker_cmd="sudo docker" +fi + +patch="0001-Read-corda-rev-from-environment-var.patch" +cp $patch $workspace + +container_image="${registry}/r3/r3-notary:$rev" + +( + cd $workspace + git apply $patch + ./gradlew jar + JAR=$(ls -S node/capsule/build/libs | head -n1) + cp node/capsule/build/libs/$JAR build-contexts/notary/corda.jar + + cd build-contexts/notary + ${docker_cmd} build -t $container_image . + ${docker_cmd} push $container_image +) + +mkdir -p built +echo "notary-r3-$rev_short $container_image" > built/notary.txt + +rm -rf $workspace + +git worktree prune diff --git a/experimental/kubernetes/bin/delete-all.sh b/experimental/kubernetes/bin/delete-all.sh new file mode 100755 index 0000000000..0bfc1f8db3 --- /dev/null +++ b/experimental/kubernetes/bin/delete-all.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +set -eux + +# TODO: perhaps delte the namespace and recreate the PVCs? + +kubectl delete configmap corda +kubectl delete configmap doorman +kubectl delete --all statefulsets +kubectl delete --all deployments +kubectl delete --all services +kubectl delete --all pods +kubectl delete --all jobs + +while :; do + sleep 5 + n=$(kubectl get pods | wc -l) + if [[ n -eq 0 ]]; then + break + fi +done + +kubectl delete --all persistentvolumeclaims diff --git a/experimental/kubernetes/bin/start-hot-cold.sh b/experimental/kubernetes/bin/start-hot-cold.sh new file mode 100755 index 0000000000..cb8e933fc5 --- /dev/null +++ b/experimental/kubernetes/bin/start-hot-cold.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +kubectl apply -f config/ha/db-service.yml +kubectl apply -f config/ha/db-pod.yml +kubectl apply -f config/ha/service.yml +kubectl apply -f config/ha/node-a.yml diff --git a/experimental/kubernetes/bin/start-hot-warm.sh b/experimental/kubernetes/bin/start-hot-warm.sh new file mode 100755 index 0000000000..cf336eac2a --- /dev/null +++ b/experimental/kubernetes/bin/start-hot-warm.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +kubectl apply -f config/ha/db-service.yml +kubectl apply -f config/ha/db-pod.yml +kubectl apply -f config/ha/zk-service.yml +kubectl apply -f config/ha/zk-pod.yml +kubectl apply -f templates/services/hotwarm.yml +kubectl apply -f config/ha/hot-warm.yml diff --git a/experimental/kubernetes/bin/start.sh b/experimental/kubernetes/bin/start.sh new file mode 100755 index 0000000000..8d33a62c32 --- /dev/null +++ b/experimental/kubernetes/bin/start.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +set -eux + +# Configuration files +kubectl create configmap corda --from-file=config-files +kubectl create configmap doorman --from-file=config-files/doorman + +# Claim volumes. +kubectl apply -f config/persistent-volume-claims/ +# TODO: do we need to wait for the claims to be bound? + +# Distribute cordapps. +kubectl apply -f config/load-generator/distribute-cordapp-job.yml +set +e +while :; do + sleep 5 + kubectl describe job distribute-cordapps | grep '1 Succeeded' + if [[ $? -eq 0 ]]; then + break + fi +done +set -e +kubectl delete job distribute-cordapps + +# Apply config. +for d in $(find config -name 'r3-*' -type d); do + kubectl apply -f $d +done +for d in $(find config -name 'corda-*' -type d); do + kubectl apply -f $d +done + +# Bootup doorman and notary. +./bin/bootstrap.sh + +# Startup corda nodes and load generator. +for d in $(find config -name 'r3-*' -type d); do + kubectl scale sts $(basename $d) --replicas=1 +done +for d in $(find config -name 'corda-*' -type d); do + kubectl scale sts $(basename $d) --replicas=1 +done +kubectl apply -f config/load-generator/load-generator.yml +kubectl scale deployment load-generator --replicas=1 diff --git a/experimental/kubernetes/build-contexts/bft-notary/Dockerfile b/experimental/kubernetes/build-contexts/bft-notary/Dockerfile new file mode 100644 index 0000000000..f3dc157dd8 --- /dev/null +++ b/experimental/kubernetes/build-contexts/bft-notary/Dockerfile @@ -0,0 +1,13 @@ +FROM openjdk:8u151-jre-alpine + +COPY corda.jar /app/corda.jar + +COPY entrypoint.sh /app/entrypoint.sh + +COPY node.conf /app/node.conf + +WORKDIR /app + +EXPOSE 10001 10002 + +CMD ["/bin/sh", "entrypoint.sh"] diff --git a/experimental/kubernetes/build-contexts/bft-notary/entrypoint.sh b/experimental/kubernetes/build-contexts/bft-notary/entrypoint.sh new file mode 100755 index 0000000000..7751842467 --- /dev/null +++ b/experimental/kubernetes/build-contexts/bft-notary/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +set -eux + +export REPLICA_ID=$(echo $HOSTNAME | grep -o "\d\+") + +java -Xmx700m -jar corda.jar --log-to-console --no-local-shell diff --git a/experimental/kubernetes/build-contexts/bft-notary/node.conf b/experimental/kubernetes/build-contexts/bft-notary/node.conf new file mode 100644 index 0000000000..ca36a226c0 --- /dev/null +++ b/experimental/kubernetes/build-contexts/bft-notary/node.conf @@ -0,0 +1,25 @@ +database { + runMigration=true +} +myLegalName="O=NotaryService"${REPLICA_ID}", L=Zurich, C=CH" +notary { + bftSMaRt { + clusterAddresses=[ + "bft-0.bft.thomas.svc.cluster.local:12000", + "bft-1.bft.thomas.svc.cluster.local:12000", + "bft-2.bft.thomas.svc.cluster.local:12000", + "bft-3.bft.thomas.svc.cluster.local:12000" + ] + debug=false + exposeRaces=false + replicaId=${REPLICA_ID} + } + custom=false + validating=false +} +p2pAddress="localhost:10002" +rpcSettings { + address="localhost:10022" + adminAddress="localhost:10122" +} +rpcUsers=[] diff --git a/experimental/kubernetes/build-contexts/cordapps/Dockerfile b/experimental/kubernetes/build-contexts/cordapps/Dockerfile new file mode 100644 index 0000000000..f3761a208f --- /dev/null +++ b/experimental/kubernetes/build-contexts/cordapps/Dockerfile @@ -0,0 +1,5 @@ +FROM busybox + +RUN mkdir /cordapps + +COPY *.jar /cordapps diff --git a/experimental/kubernetes/build-contexts/doorman/Dockerfile b/experimental/kubernetes/build-contexts/doorman/Dockerfile new file mode 100644 index 0000000000..2f56b4a9a9 --- /dev/null +++ b/experimental/kubernetes/build-contexts/doorman/Dockerfile @@ -0,0 +1,10 @@ +FROM openjdk:8u151-jre-alpine + +RUN mkdir /app +COPY doorman.jar /app +COPY init.sh /app +COPY entrypoint.sh /app + +WORKDIR /app + +ENTRYPOINT ["/bin/sh", "entrypoint.sh"] diff --git a/experimental/kubernetes/build-contexts/doorman/entrypoint.sh b/experimental/kubernetes/build-contexts/doorman/entrypoint.sh new file mode 100644 index 0000000000..a19634436e --- /dev/null +++ b/experimental/kubernetes/build-contexts/doorman/entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -eux + +cd /data + +java -Xmx700m -jar /app/doorman.jar --config-file /config/doorman.conf \ + --set-network-parameters /config/network-parameters.conf || true +exec java -Xmx700m -jar /app/doorman.jar --config-file /config/doorman.conf diff --git a/experimental/kubernetes/build-contexts/doorman/init.sh b/experimental/kubernetes/build-contexts/doorman/init.sh new file mode 100755 index 0000000000..674a07896b --- /dev/null +++ b/experimental/kubernetes/build-contexts/doorman/init.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +set -eux + +cd /data + +rm -rf * + +mkdir -p /data/doorman + +yes '' | java -jar /app/doorman.jar --config-file ${NODE_INIT_CONFIG:-"/config/doorman-init.conf"} --mode ROOT_KEYGEN +yes '' | java -jar /app/doorman.jar --config-file ${NODE_INIT_CONFIG:-"/config/doorman-init.conf"} --mode CA_KEYGEN + +java -Xmx700m -jar /app/doorman.jar --config-file ${NODE_INIT_CONFIG:-"/config/doorman-init.conf"} diff --git a/experimental/kubernetes/build-contexts/ha-node/Dockerfile b/experimental/kubernetes/build-contexts/ha-node/Dockerfile new file mode 100644 index 0000000000..6b52f4309c --- /dev/null +++ b/experimental/kubernetes/build-contexts/ha-node/Dockerfile @@ -0,0 +1,12 @@ +FROM openjdk:8u151-jre-alpine + +COPY corda.jar /app/corda.jar +COPY plugins /app/plugins/ +COPY entrypoint.sh /app/entrypoint.sh +COPY node.conf /app/node.conf + +WORKDIR /app + +EXPOSE 10001 10002 + +CMD ["/bin/sh", "entrypoint.sh"] diff --git a/experimental/kubernetes/build-contexts/ha-node/build-and-publish.sh b/experimental/kubernetes/build-contexts/ha-node/build-and-publish.sh new file mode 100755 index 0000000000..84c95b89c7 --- /dev/null +++ b/experimental/kubernetes/build-contexts/ha-node/build-and-publish.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -eux + +img="ctesting.azurecr.io/r3/hanode:$(git rev-parse --short HEAD)" + +docker build -t $img . +docker push $img diff --git a/experimental/kubernetes/build-contexts/ha-node/entrypoint.sh b/experimental/kubernetes/build-contexts/ha-node/entrypoint.sh new file mode 100755 index 0000000000..5bb7ab649b --- /dev/null +++ b/experimental/kubernetes/build-contexts/ha-node/entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +set -eux + +java -Xmx700m -jar corda.jar --initial-registration \ + --network-root-truststore /truststore/network-root-truststore.jks \ + --network-root-truststore-password '' || true + +exec java -Xmx700m -jar corda.jar \ + --no-local-shell \ + --log-to-console diff --git a/experimental/kubernetes/build-contexts/ha-node/node.conf b/experimental/kubernetes/build-contexts/ha-node/node.conf new file mode 100644 index 0000000000..18186fac0e --- /dev/null +++ b/experimental/kubernetes/build-contexts/ha-node/node.conf @@ -0,0 +1,41 @@ +p2pAddress : "hanode:10002" +rpcSettings { + address : ${HOSTNAME}":10003" + adminAddress: ${HOSTNAME}":12000" +} +myLegalName : "O=Corda HA, L=London, C=GB" +keyStorePassword : "cordacadevpass" +trustStorePassword : "trustpass" +devMode : true + +jarDirs = ["plugins", "cordapps"] + +dataSourceProperties = { + dataSourceClassName = "org.postgresql.ds.PGSimpleDataSource" + dataSource.url = "jdbc:postgresql://db:5432/postgres" + dataSource.user = postgres + dataSource.password = "" +} +database = { + transactionIsolationLevel = READ_COMMITTED + runMigration = true +} +enterpriseConfiguration = { + mutualExclusionConfiguration = { + on = true + machineName = ${HOSTNAME} + updateInterval = 20000 + waitInterval = 40000 + } +} +rpcUsers=[ + { + user=demou + password=demop + permissions=[ + ALL + ] + } +] + +compatibilityZoneURL=${COMPATIBILITY_ZONE_URL} diff --git a/experimental/kubernetes/build-contexts/ha-node/plugins/postgresql-42.1.4.jar b/experimental/kubernetes/build-contexts/ha-node/plugins/postgresql-42.1.4.jar new file mode 100644 index 0000000000..08a54b105f Binary files /dev/null and b/experimental/kubernetes/build-contexts/ha-node/plugins/postgresql-42.1.4.jar differ diff --git a/experimental/kubernetes/build-contexts/hot-warm/Dockerfile b/experimental/kubernetes/build-contexts/hot-warm/Dockerfile new file mode 100644 index 0000000000..6b52f4309c --- /dev/null +++ b/experimental/kubernetes/build-contexts/hot-warm/Dockerfile @@ -0,0 +1,12 @@ +FROM openjdk:8u151-jre-alpine + +COPY corda.jar /app/corda.jar +COPY plugins /app/plugins/ +COPY entrypoint.sh /app/entrypoint.sh +COPY node.conf /app/node.conf + +WORKDIR /app + +EXPOSE 10001 10002 + +CMD ["/bin/sh", "entrypoint.sh"] diff --git a/experimental/kubernetes/build-contexts/hot-warm/build-and-publish.sh b/experimental/kubernetes/build-contexts/hot-warm/build-and-publish.sh new file mode 100755 index 0000000000..84c95b89c7 --- /dev/null +++ b/experimental/kubernetes/build-contexts/hot-warm/build-and-publish.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -eux + +img="ctesting.azurecr.io/r3/hanode:$(git rev-parse --short HEAD)" + +docker build -t $img . +docker push $img diff --git a/experimental/kubernetes/build-contexts/hot-warm/entrypoint.sh b/experimental/kubernetes/build-contexts/hot-warm/entrypoint.sh new file mode 100755 index 0000000000..ed7293f8cf --- /dev/null +++ b/experimental/kubernetes/build-contexts/hot-warm/entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +set -eux + +java -Xmx700m -jar corda.jar --initial-registration \ + --config-file=${CONFIG_FILE} \ + --network-root-truststore /truststore/network-root-truststore.jks \ + --network-root-truststore-password '' || true + +exec java -Xmx700m -jar corda.jar \ + --config-file=${CONFIG_FILE} \ + --no-local-shell \ + --log-to-console \ + "$@" diff --git a/experimental/kubernetes/build-contexts/hot-warm/node.conf b/experimental/kubernetes/build-contexts/hot-warm/node.conf new file mode 100644 index 0000000000..adbe0c604e --- /dev/null +++ b/experimental/kubernetes/build-contexts/hot-warm/node.conf @@ -0,0 +1,39 @@ +p2pAddress : "hanode:10002" +rpcSettings { + address : ${HOSTNAME}":10003" + adminAddress: ${HOSTNAME}":12000" +} +myLegalName : "O=Corda HA, L=London, C=GB" +keyStorePassword : "cordacadevpass" +trustStorePassword : "trustpass" +devMode : true + +dataSourceProperties = { + dataSourceClassName = "org.postgresql.ds.PGSimpleDataSource" + dataSource.url = "jdbc:postgresql://db:5432/postgres" + dataSource.user = postgres + dataSource.password = "" +} +database = { + transactionIsolationLevel = READ_COMMITTED + runMigration = true +} +enterpriseConfiguration = { + mutualExclusionConfiguration = { + on = true + machineName = ${HOSTNAME} + updateInterval = 20000 + waitInterval = 40000 + } +} +rpcUsers=[ + { + user=demou + password=demop + permissions=[ + ALL + ] + } +] + +compatibilityZoneURL=${COMPATIBILITY_ZONE_URL} diff --git a/experimental/kubernetes/build-contexts/hot-warm/plugins/postgresql-42.1.4.jar b/experimental/kubernetes/build-contexts/hot-warm/plugins/postgresql-42.1.4.jar new file mode 100644 index 0000000000..08a54b105f Binary files /dev/null and b/experimental/kubernetes/build-contexts/hot-warm/plugins/postgresql-42.1.4.jar differ diff --git a/experimental/kubernetes/build-contexts/load-generator/Dockerfile b/experimental/kubernetes/build-contexts/load-generator/Dockerfile new file mode 100644 index 0000000000..3019b992c6 --- /dev/null +++ b/experimental/kubernetes/build-contexts/load-generator/Dockerfile @@ -0,0 +1,9 @@ +FROM openjdk:8u151-jre-alpine + +COPY app.jar /app/app.jar + +COPY entrypoint.sh /app/entrypoint.sh + +WORKDIR /app + +CMD ["/bin/sh", "entrypoint.sh"] diff --git a/experimental/kubernetes/build-contexts/load-generator/entrypoint.sh b/experimental/kubernetes/build-contexts/load-generator/entrypoint.sh new file mode 100755 index 0000000000..d9627b4adf --- /dev/null +++ b/experimental/kubernetes/build-contexts/load-generator/entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +set -eux + +exec java -Xmx700m -jar app.jar ${TARGET_HOST}:${RPC_PORT} diff --git a/experimental/kubernetes/build-contexts/node/Dockerfile b/experimental/kubernetes/build-contexts/node/Dockerfile new file mode 100644 index 0000000000..268b4ff4e9 --- /dev/null +++ b/experimental/kubernetes/build-contexts/node/Dockerfile @@ -0,0 +1,11 @@ +FROM openjdk:8u151-jre-alpine + +COPY corda.jar /app/corda.jar + +COPY entrypoint.sh /app/entrypoint.sh + +WORKDIR /app + +EXPOSE 10001 10002 + +ENTRYPOINT ["/bin/sh", "entrypoint.sh"] diff --git a/experimental/kubernetes/build-contexts/node/entrypoint.sh b/experimental/kubernetes/build-contexts/node/entrypoint.sh new file mode 100755 index 0000000000..ca36700d16 --- /dev/null +++ b/experimental/kubernetes/build-contexts/node/entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +set -eux + +export LEGAL_NAME="C=GB,L=London,O=T-$HOSTNAME" + +export P2P_ADDRESS=$(eval "echo $P2P_ADDRESS") + +export RPC_ADDRESS=$(eval "echo $RPC_ADDRESS") + +export ADMIN_ADDRESS=$(eval "echo $ADMIN_ADDRESS") +export COMPATIBILITY_ZONE_URL="http://$DOORMAN_SERVICE_HOST:1300" + +env + +# TODO: fix +cp ${CONFIG_FILE} /app/node.conf + +java -Xmx700m -jar corda.jar --initial-registration \ + --config-file=${CONFIG_FILE} \ + --network-root-truststore /truststore/network-root-truststore.jks \ + --network-root-truststore-password '' + +exec java -Xmx700m -jar corda.jar \ + --config-file=${CONFIG_FILE} \ + --no-local-shell \ + --log-to-console \ + "$@" diff --git a/experimental/kubernetes/build-contexts/notary/Dockerfile b/experimental/kubernetes/build-contexts/notary/Dockerfile new file mode 100644 index 0000000000..c81084d27f --- /dev/null +++ b/experimental/kubernetes/build-contexts/notary/Dockerfile @@ -0,0 +1,11 @@ +FROM openjdk:8u151-jre-alpine + +RUN mkdir /app +COPY corda.jar /app +COPY node.conf /app +COPY init.sh /app +COPY entrypoint.sh /app + +WORKDIR /app + +CMD ["/bin/sh", "entrypoint.sh"] diff --git a/experimental/kubernetes/build-contexts/notary/entrypoint.sh b/experimental/kubernetes/build-contexts/notary/entrypoint.sh new file mode 100644 index 0000000000..67c68d7488 --- /dev/null +++ b/experimental/kubernetes/build-contexts/notary/entrypoint.sh @@ -0,0 +1,11 @@ +set -eux + +export COMPATIBILITY_ZONE_URL=$(eval "echo http://${DOORMAN_PORT_1300_TCP_ADDR}:1300") +export P2P_ADDRESS=$(eval "echo ${HOSTNAME}:10001") +export RPC_ADDRESS=$(eval "echo ${HOSTNAME}:10002") + + +exec java -Xmx1g -jar /app/corda.jar --log-to-console \ + --no-local-shell \ + --base-directory=/data \ + --network-root-truststore=/truststore/network-root-truststore.jks diff --git a/experimental/kubernetes/build-contexts/notary/init.sh b/experimental/kubernetes/build-contexts/notary/init.sh new file mode 100755 index 0000000000..377fc6706d --- /dev/null +++ b/experimental/kubernetes/build-contexts/notary/init.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +set -eux + +export COMPATIBILITY_ZONE_URL=$(eval "echo http://${DOORMAN_PORT_1300_TCP_ADDR}:1300") +export P2P_ADDRESS=$(eval "echo ${HOSTNAME}:10001") +export RPC_ADDRESS=$(eval "echo ${HOSTNAME}:10002") + +cd /data + +rm -rf * + +cp /app/node.conf . + +echo 'hello' +java -Xmx1g -jar /app/corda.jar --initial-registration \ + --base-directory=/data \ + --network-root-truststore /truststore/network-root-truststore.jks \ + --network-root-truststore-password '' + +java -Xmx1g -jar /app/corda.jar --just-generate-node-info --base-directory=/data + +cp nodeInfo* notary-node-info + +echo 'DONE_BOOTSTRAPPING' + +sleep 3600 diff --git a/experimental/kubernetes/build-contexts/notary/node.conf b/experimental/kubernetes/build-contexts/notary/node.conf new file mode 100644 index 0000000000..65ea98c9bd --- /dev/null +++ b/experimental/kubernetes/build-contexts/notary/node.conf @@ -0,0 +1,22 @@ +myLegalName="C=GB,L=London,O=NotaryService0" + +notary { + validating=false +} + +devMode=false + +p2pAddress=${P2P_ADDRESS} +rpcAddress=${RPC_ADDRESS} +rpcSettings { + adminAddress=${HOSTNAME}":7777" +} +rpcUsers=[] + +compatibilityZoneURL=${COMPATIBILITY_ZONE_URL} + +enterpriseConfiguration { + tuning { + flowThreadPoolSize = 4 + } +} diff --git a/experimental/kubernetes/config-files/doorman/doorman-init.conf b/experimental/kubernetes/config-files/doorman/doorman-init.conf new file mode 100644 index 0000000000..df40992452 --- /dev/null +++ b/experimental/kubernetes/config-files/doorman/doorman-init.conf @@ -0,0 +1,32 @@ +basedir = "/data/doorman" +address = ${HOSTNAME}":1300" + +#For local signing +rootStorePath = ${basedir}"/certificates/rootstore.jks" +keystorePath = ${basedir}"/certificates/caKeystore.jks" +keystorePassword = "password" +caPrivateKeyPassword = "password" + +# Database config +dataSourceProperties { + dataSourceClassName = org.h2.jdbcx.JdbcDataSource + "dataSource.url" = "jdbc:h2:file:"${basedir}"/persistence;DB_CLOSE_ON_EXIT=FALSE;LOCK_TIMEOUT=10000;WRITE_DELAY=0;AUTO_SERVER_PORT="${h2port} + "dataSource.user" = sa + "dataSource.password" = "" +} +database { runMigration = true } +h2port = 0 + +# Doorman config +# Comment out this section if running without doorman service +doorman { + approveInterval = 20 + approveAll = true +} + +# Network map config +# Comment out this section if running without network map service +#networkMap { +# cacheTimeout = 1000 +# signInterval = 3000 +#} diff --git a/experimental/kubernetes/config-files/doorman/doorman.conf b/experimental/kubernetes/config-files/doorman/doorman.conf new file mode 100644 index 0000000000..767213bfa2 --- /dev/null +++ b/experimental/kubernetes/config-files/doorman/doorman.conf @@ -0,0 +1,32 @@ +basedir = "/data/doorman" +address = ${HOSTNAME}":1300" + +#For local signing +rootStorePath = ${basedir}"/certificates/rootstore.jks" +keystorePath = ${basedir}"/certificates/caKeystore.jks" +keystorePassword = "password" +caPrivateKeyPassword = "password" + +# Database config +dataSourceProperties { + dataSourceClassName = org.h2.jdbcx.JdbcDataSource + "dataSource.url" = "jdbc:h2:file:"${basedir}"/persistence;DB_CLOSE_ON_EXIT=FALSE;LOCK_TIMEOUT=10000;WRITE_DELAY=0;AUTO_SERVER_PORT="${h2port} + "dataSource.user" = sa + "dataSource.password" = "" +} +database { runMigration = true } +h2port = 0 + +# Doorman config +# Comment out this section if running without doorman service +doorman { + approveInterval = 20 + approveAll = true +} + +# Network map config +# Comment out this section if running without network map service +networkMap { + cacheTimeout = 1000 + signInterval = 3000 +} diff --git a/experimental/kubernetes/config-files/doorman/network-parameters.conf b/experimental/kubernetes/config-files/doorman/network-parameters.conf new file mode 100644 index 0000000000..60694df675 --- /dev/null +++ b/experimental/kubernetes/config-files/doorman/network-parameters.conf @@ -0,0 +1,7 @@ +notaries : [{ + notaryNodeInfoFile: "/data/notary-node-info" + validating: false + }] + minimumPlatformVersion = 1 + maxMessageSize = 10485760 + maxTransactionSize = 10485760 diff --git a/experimental/kubernetes/config-files/hot-warm.conf b/experimental/kubernetes/config-files/hot-warm.conf new file mode 100644 index 0000000000..27b6505069 --- /dev/null +++ b/experimental/kubernetes/config-files/hot-warm.conf @@ -0,0 +1,41 @@ +p2pAddress : "hot-warm:10002" +rpcSettings { + address : ${HOSTNAME}":10003" + adminAddress: ${HOSTNAME}":12000" +} +myLegalName : "O=Corda HA, L=London, C=GB" +keyStorePassword : "cordacadevpass" +trustStorePassword : "trustpass" +devMode : true + +dataSourceProperties = { + dataSourceClassName = "org.postgresql.ds.PGSimpleDataSource" + dataSource.url = "jdbc:postgresql://db:5432/postgres" + dataSource.user = postgres + dataSource.password = "" +} +database = { + transactionIsolationLevel = READ_COMMITTED + runMigration = true +} +enterpriseConfiguration = { + haConfiguration { + hotWarm { + connectString = "zk" + electionPath = "/example/leader" + nodeID = ${HOSTNAME} + priority = 1 + } + } +} +rpcUsers=[ + { + user=demou + password=demop + permissions=[ + ALL + ] + } +] + +compatibilityZoneURL=${COMPATIBILITY_ZONE_URL} diff --git a/experimental/kubernetes/config-files/node.conf b/experimental/kubernetes/config-files/node.conf new file mode 100644 index 0000000000..d098d4a85c --- /dev/null +++ b/experimental/kubernetes/config-files/node.conf @@ -0,0 +1,19 @@ +myLegalName=${LEGAL_NAME} +p2pAddress=${P2P_ADDRESS} +rpcSettings { + address=${RPC_ADDRESS} + adminAddress=${ADMIN_ADDRESS} +} + +jarDirs = ["cordapps"] + +rpcUsers=[ + { + password=demop + permissions=[ + ALL + ] + username=demou + } +] +compatibilityZoneURL=${COMPATIBILITY_ZONE_URL} diff --git a/experimental/kubernetes/generate_config.py b/experimental/kubernetes/generate_config.py new file mode 100644 index 0000000000..5bb2912eab --- /dev/null +++ b/experimental/kubernetes/generate_config.py @@ -0,0 +1,113 @@ +from argparse import ArgumentParser +import glob +import os + +from jinja2 import Environment, FileSystemLoader + + +env = Environment(loader=FileSystemLoader('templates')) +service_template = env.get_template('service.yml.j2') +statefulset_template = env.get_template('statefulset.yml.j2') +loadgenerator_template = env.get_template('load-generator.yml') +distribute_cordapp_tmpl = env.get_template('distribute-healthcheck-cordapp.yml') + + +def main(): + p = ArgumentParser() + p.add_argument("--namespace", "-n", help="namespace where to deploy", required=True) + p.add_argument("--storage-class", "-s", default="standard", help="storage-class for volume claims [default=default]") + args = p.parse_args() + namespace = args.namespace + storage_class = args.storage_class + + load_gen_image = open('built/load_generator.txt').read().strip() + cordapps_image = open('built/cordapps.txt').read().strip() + + for l in open('built/node-images.txt'): + version, image = l.split() + p = os.path.join('config', version) + makedirs(p) + version = version.replace(".", "-").lower() + open(os.path.join(p, 'service.yml'), 'w').write(service_template.render(version=version)) + open(os.path.join(p, 'statefulset.yml'), 'w').write(statefulset_template.render(version=version, image=image, namespace=namespace)) + + # Emit load generator config. + p = os.path.join('config', 'load-generator') + makedirs(p) + # The load generator will hit the node with the last version in the input. + # Since the load generator discovers all nodes via network map snapshot, it + # is not necessary to redeploy the load generator if another load generator + # is already running. + open(os.path.join(p, 'load-generator.yml'), 'w').write(loadgenerator_template.render(version=version, namespace=namespace, image=load_gen_image)) + open(os.path.join(p, 'distribute-cordapp-job.yml'), 'w').write(distribute_cordapp_tmpl.render(image=cordapps_image)) + + # Genrerate persistent volume claims. + env = Environment(loader=FileSystemLoader('templates/persistent-volume-claims')) + p = 'config/persistent-volume-claims' + makedirs(p) + for templ in glob.glob('templates/persistent-volume-claims/*'): + b = os.path.basename(templ) + t = env.get_template(b) + open(p +'/'+ b, 'w').write(t.render(storage_class=storage_class)) + + # Prep for doorman and notary. + services_env = Environment(loader=FileSystemLoader('templates/services')) + pods_env = Environment(loader=FileSystemLoader('templates/pods')) + + # Generate doorman. + doorman_name, doorman_image = open('built/doorman.txt').read().split() + templ = services_env.get_template('doorman.yml') + makedirs('config/doorman') + open('config/doorman/service.yml', 'w').write(templ.render(image=doorman_image, name=doorman_name)) + templ = pods_env.get_template('doorman.yml') + open('config/doorman/pod.yml', 'w').write(templ.render(image=doorman_image, name=doorman_name)) + templ = pods_env.get_template('doorman-init.yml') + open('config/doorman/pod-init.yml', 'w').write(templ.render(image=doorman_image, name=doorman_name)) + + # Generate notary. + notary_name, notary_image = open('built/notary.txt').read().split() + templ = services_env.get_template('notary.yml') + makedirs('config/notary') + open('config/notary/service.yml', 'w').write(templ.render(image=notary_image, name=notary_name)) + templ = pods_env.get_template('notary.yml') + open('config/notary/pod.yml', 'w').write(templ.render(image=notary_image, name=notary_name, namespace=namespace)) + templ = pods_env.get_template('notary-init.yml') + open('config/notary/pod-init.yml', 'w').write(templ.render(image=notary_image, name=notary_name, namespace=namespace)) + + # Template the HA nodes. + try: + makedirs('config/ha') + _, haimage = open('built/ha-node-image.txt').read().split() + templ = pods_env.get_template('ha-node.yml') + open('config/ha/node-a.yml', 'w').write(templ.render(image=haimage, name='hanode-a')) + open('config/ha/node-b.yml', 'w').write(templ.render(image=haimage, name='hanode-b')) + templ = services_env.get_template('ha.yml') + open('config/ha/service.yml', 'w').write(templ.render()) + templ = services_env.get_template('db.yml') + open('config/ha/db-service.yml', 'w').write(templ.render()) + templ = pods_env.get_template('db.yml') + open('config/ha/db-pod.yml', 'w').write(templ.render()) + + # Hot warm + _, hwimage = open('built/hot-warm-image.txt').read().split() + templ = pods_env.get_template('hot-warm.yml') + open('config/ha/hot-warm.yml', 'w').write(templ.render(image=hwimage, name='hot-warm')) + + templ = services_env.get_template('zk.yml') + open('config/ha/zk-service.yml', 'w').write(templ.render()) + templ = pods_env.get_template('zk.yml') + open('config/ha/zk-pod.yml', 'w').write(templ.render()) + except IOError: + pass # Assuming the HA node images are not built. + +def makedirs(p): + try: + os.makedirs(p) + except Exception: # assuming file exists + pass + + +if __name__ == '__main__': + main() + + diff --git a/experimental/kubernetes/ha-testing/Readme.md b/experimental/kubernetes/ha-testing/Readme.md new file mode 100644 index 0000000000..f367ae4c46 --- /dev/null +++ b/experimental/kubernetes/ha-testing/Readme.md @@ -0,0 +1,12 @@ +# HA Testing + +## Hot Cold + +Components: +- Postgres DB service and pod called `db` +- hanode-a and hanode-b pods, a service called `hanode` to load balance between the two +- persistent volumes for artemis, cordapps, doorman and notary + +Registration is currently done with the doorman, the bootstrapper didn't work +for me, because some environment varaibles in the config file didn't resolve +properly. This could be improved by adding defaults. diff --git a/experimental/kubernetes/ha-testing/pods/db.yml b/experimental/kubernetes/ha-testing/pods/db.yml new file mode 100644 index 0000000000..e9857444f9 --- /dev/null +++ b/experimental/kubernetes/ha-testing/pods/db.yml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: db + labels: + app: db +spec: + containers: + - name: db + image: postgres + ports: + - containerPort: 5432 diff --git a/experimental/kubernetes/ha-testing/pods/node-a.yml b/experimental/kubernetes/ha-testing/pods/node-a.yml new file mode 100644 index 0000000000..b3a0a0d18f --- /dev/null +++ b/experimental/kubernetes/ha-testing/pods/node-a.yml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: Pod +metadata: + name: hanode-a + labels: + app: hanode +spec: + imagePullSecrets: + - name: regsecret + containers: + - name: hanode + image: ctesting.azurecr.io/r3/hanode:b3f29942d1 + imagePullPolicy: Always + env: + - name: "COMPATIBILITY_ZONE_URL" + value: "http://doorman:1300" + volumeMounts: + - mountPath: /app/artemis + name: artemis + - mountPath: /app/cordapps + name: cordapps + - mountPath: /app/certificates + name: certificates + - mountPath: /truststore + name: truststore + readOnly: true + ports: + - containerPort: 10002 + - containerPort: 10003 + volumes: + - name: artemis + persistentVolumeClaim: + claimName: artemis + - name: cordapps + persistentVolumeClaim: + claimName: cordapps + - name: certificates + persistentVolumeClaim: + claimName: certificates + - name: truststore + secret: + secretName: truststore-3.0.0 diff --git a/experimental/kubernetes/ha-testing/pods/node-b.yml b/experimental/kubernetes/ha-testing/pods/node-b.yml new file mode 100644 index 0000000000..a61242cf86 --- /dev/null +++ b/experimental/kubernetes/ha-testing/pods/node-b.yml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: Pod +metadata: + name: hanode-b + labels: + app: hanode +spec: + imagePullSecrets: + - name: regsecret + containers: + - name: hanode + image: ctesting.azurecr.io/r3/hanode:b3f29942d1 + imagePullPolicy: Always + env: + - name: "COMPATIBILITY_ZONE_URL" + value: "http://doorman:1300" + volumeMounts: + - mountPath: /app/artemis + name: artemis + - mountPath: /app/cordapps + name: cordapps + - mountPath: /app/certificates + name: certificates + - mountPath: /truststore + name: truststore + readOnly: true + ports: + - containerPort: 10002 + - containerPort: 10003 + volumes: + - name: artemis + persistentVolumeClaim: + claimName: artemis + - name: cordapps + persistentVolumeClaim: + claimName: cordapps + - name: certificates + persistentVolumeClaim: + claimName: certificates + - name: truststore + secret: + secretName: truststore-3.0.0 diff --git a/experimental/kubernetes/ha-testing/pvcs/artemis.yml b/experimental/kubernetes/ha-testing/pvcs/artemis.yml new file mode 100644 index 0000000000..5eba456543 --- /dev/null +++ b/experimental/kubernetes/ha-testing/pvcs/artemis.yml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: artemis +spec: + accessModes: + - ReadWriteMany + storageClassName: standard + resources: + requests: + storage: 200Mi diff --git a/experimental/kubernetes/ha-testing/pvcs/certificates.yml b/experimental/kubernetes/ha-testing/pvcs/certificates.yml new file mode 100644 index 0000000000..4388b521a7 --- /dev/null +++ b/experimental/kubernetes/ha-testing/pvcs/certificates.yml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: certificates +spec: + accessModes: + - ReadWriteMany + storageClassName: standard + resources: + requests: + storage: 200Mi diff --git a/experimental/kubernetes/ha-testing/pvcs/cordapps.yml b/experimental/kubernetes/ha-testing/pvcs/cordapps.yml new file mode 100644 index 0000000000..657d39b1f6 --- /dev/null +++ b/experimental/kubernetes/ha-testing/pvcs/cordapps.yml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: cordapps +spec: + accessModes: + - ReadOnlyMany + storageClassName: standard + resources: + requests: + storage: 200Mi diff --git a/experimental/kubernetes/ha-testing/pvcs/node-infos.yml b/experimental/kubernetes/ha-testing/pvcs/node-infos.yml new file mode 100644 index 0000000000..f0394f7a33 --- /dev/null +++ b/experimental/kubernetes/ha-testing/pvcs/node-infos.yml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: node-infos +spec: + accessModes: + - ReadWriteMany + storageClassName: standard + resources: + requests: + storage: 200Mi diff --git a/experimental/kubernetes/ha-testing/services/db.yml b/experimental/kubernetes/ha-testing/services/db.yml new file mode 100644 index 0000000000..e78227fe43 --- /dev/null +++ b/experimental/kubernetes/ha-testing/services/db.yml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: db +spec: + ports: + - name: postgres + port: 5432 + protocol: TCP + targetPort: 5432 + selector: + app: db diff --git a/experimental/kubernetes/ha-testing/services/hanode.yml b/experimental/kubernetes/ha-testing/services/hanode.yml new file mode 100644 index 0000000000..73926fad16 --- /dev/null +++ b/experimental/kubernetes/ha-testing/services/hanode.yml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: hanode +spec: + ports: + - name: p2p + port: 10002 + protocol: TCP + targetPort: 10002 + - name: rpc + port: 10003 + protocol: TCP + targetPort: 10003 + selector: + app: hanode diff --git a/experimental/kubernetes/ha-testing/start.sh b/experimental/kubernetes/ha-testing/start.sh new file mode 100755 index 0000000000..58294c91d0 --- /dev/null +++ b/experimental/kubernetes/ha-testing/start.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -eux + +kubectl apply -f pvcs/artemis.yml +kubectl apply -f pvcs/certificates.yml +kubectl apply -f services/ +kubectl apply -f pods/db.yml + + diff --git a/experimental/kubernetes/releases.txt b/experimental/kubernetes/releases.txt new file mode 100644 index 0000000000..24c3eb0cd9 --- /dev/null +++ b/experimental/kubernetes/releases.txt @@ -0,0 +1 @@ +release-V3.0.1-DEV-PREVIEW diff --git a/experimental/kubernetes/requirements.txt b/experimental/kubernetes/requirements.txt new file mode 100644 index 0000000000..89ee2b0ffc --- /dev/null +++ b/experimental/kubernetes/requirements.txt @@ -0,0 +1 @@ +Jinja2==2.10 diff --git a/experimental/kubernetes/templates/distribute-healthcheck-cordapp.yml b/experimental/kubernetes/templates/distribute-healthcheck-cordapp.yml new file mode 100644 index 0000000000..f795375161 --- /dev/null +++ b/experimental/kubernetes/templates/distribute-healthcheck-cordapp.yml @@ -0,0 +1,22 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: distribute-cordapps +spec: + template: + spec: + imagePullSecrets: + - name: regsecret + containers: + - name: distribute-healthcheck-cordapp + image: {{ image }} + command: ["/bin/sh", "-c", "rm -f cordapps-mnt/* && cp /cordapps/* /cordapps-mnt/"] + volumeMounts: + - name: cordapps + mountPath: /cordapps-mnt + restartPolicy: Never + volumes: + - name: cordapps + persistentVolumeClaim: + claimName: cordapps + backoffLimit: 4 diff --git a/experimental/kubernetes/templates/load-generator.yml b/experimental/kubernetes/templates/load-generator.yml new file mode 100644 index 0000000000..085a085edb --- /dev/null +++ b/experimental/kubernetes/templates/load-generator.yml @@ -0,0 +1,36 @@ +apiVersion: "apps/v1beta1" +kind: Deployment +metadata: + name: load-generator +spec: + replicas: 0 + selector: + matchLabels: + app: load-generator + template: + metadata: + labels: + app: load-generator + spec: + imagePullSecrets: + - name: regsecret + containers: + - name: notary-healthcheck + image: {{ image }} + imagePullPolicy: Always + env: + - name: "TARGET_HOST" + value: "{{ version }}-0.{{ version }}.{{ namespace }}.svc.cluster.local" + - name: "RPC_PORT" + value: "10003" + - name: "NUM_ITERATIONS" + value: "2" + - name: "SLEEP_MILLIS" + value: "30000" + volumeMounts: + - name: cordapps + mountPath: /cordapps + volumes: + - name: cordapps + persistentVolumeClaim: + claimName: cordapps diff --git a/experimental/kubernetes/templates/persistent-volume-claims/artemis.yml b/experimental/kubernetes/templates/persistent-volume-claims/artemis.yml new file mode 100644 index 0000000000..4b6af1c8a3 --- /dev/null +++ b/experimental/kubernetes/templates/persistent-volume-claims/artemis.yml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: artemis +spec: + accessModes: + - ReadWriteMany + storageClassName: {{ storage_class }} + resources: + requests: + storage: 200Mi diff --git a/experimental/kubernetes/templates/persistent-volume-claims/certificates.yml b/experimental/kubernetes/templates/persistent-volume-claims/certificates.yml new file mode 100644 index 0000000000..a9c40f04a9 --- /dev/null +++ b/experimental/kubernetes/templates/persistent-volume-claims/certificates.yml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: certificates +spec: + accessModes: + - ReadWriteMany + storageClassName: {{ storage_class }} + resources: + requests: + storage: 200Mi diff --git a/experimental/kubernetes/templates/persistent-volume-claims/cordapps.yml b/experimental/kubernetes/templates/persistent-volume-claims/cordapps.yml new file mode 100644 index 0000000000..a131c43c7c --- /dev/null +++ b/experimental/kubernetes/templates/persistent-volume-claims/cordapps.yml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: cordapps +spec: + accessModes: + - ReadOnlyMany + storageClassName: {{ storage_class }} + resources: + requests: + storage: 1Gi diff --git a/experimental/kubernetes/templates/persistent-volume-claims/doorman.yml b/experimental/kubernetes/templates/persistent-volume-claims/doorman.yml new file mode 100644 index 0000000000..20fe0fde90 --- /dev/null +++ b/experimental/kubernetes/templates/persistent-volume-claims/doorman.yml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: doorman +spec: + accessModes: + - ReadWriteMany + storageClassName: {{ storage_class }} + resources: + requests: + storage: 10Gi diff --git a/experimental/kubernetes/templates/persistent-volume-claims/notary.yml b/experimental/kubernetes/templates/persistent-volume-claims/notary.yml new file mode 100644 index 0000000000..227c9bf50e --- /dev/null +++ b/experimental/kubernetes/templates/persistent-volume-claims/notary.yml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: notary +spec: + accessModes: + - ReadWriteMany + storageClassName: {{ storage_class }} + resources: + requests: + storage: 10Gi diff --git a/experimental/kubernetes/templates/pods/db.yml b/experimental/kubernetes/templates/pods/db.yml new file mode 100644 index 0000000000..e9857444f9 --- /dev/null +++ b/experimental/kubernetes/templates/pods/db.yml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: db + labels: + app: db +spec: + containers: + - name: db + image: postgres + ports: + - containerPort: 5432 diff --git a/experimental/kubernetes/templates/pods/doorman-init.yml b/experimental/kubernetes/templates/pods/doorman-init.yml new file mode 100644 index 0000000000..2de5305788 --- /dev/null +++ b/experimental/kubernetes/templates/pods/doorman-init.yml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Pod +metadata: + name: doorman + labels: + app: doorman +spec: + imagePullSecrets: + - name: regsecret + containers: + - name: doorman + image: {{ image }} + imagePullPolicy: Always + volumeMounts: + - mountPath: /data + name: pv + - mountPath: /config + name: config + ports: + - containerPort: 1300 + command: + - '/app/init.sh' + volumes: + - name: config + configMap: + name: doorman + - name: pv + persistentVolumeClaim: + claimName: doorman diff --git a/experimental/kubernetes/templates/pods/doorman.yml b/experimental/kubernetes/templates/pods/doorman.yml new file mode 100644 index 0000000000..ffd0bcc6b9 --- /dev/null +++ b/experimental/kubernetes/templates/pods/doorman.yml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Pod +metadata: + name: doorman + labels: + app: doorman +spec: + imagePullSecrets: + - name: regsecret + containers: + - name: doorman + image: {{ image }} + imagePullPolicy: Always + volumeMounts: + - mountPath: /data + name: pv + - mountPath: /config + name: config + ports: + - containerPort: 1300 + volumes: + - name: config + configMap: + name: doorman + - name: pv + persistentVolumeClaim: + claimName: doorman diff --git a/experimental/kubernetes/templates/pods/ha-node.yml b/experimental/kubernetes/templates/pods/ha-node.yml new file mode 100644 index 0000000000..279dfc41e8 --- /dev/null +++ b/experimental/kubernetes/templates/pods/ha-node.yml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: Pod +metadata: + name: {{ name }} + labels: + app: hanode +spec: + imagePullSecrets: + - name: regsecret + containers: + - name: hanode + image: {{ image }} + imagePullPolicy: Always + env: + - name: "COMPATIBILITY_ZONE_URL" + value: "http://doorman:1300" + volumeMounts: + - mountPath: /app/artemis + name: artemis + - mountPath: /app/cordapps + name: cordapps + - mountPath: /app/certificates + name: certificates + - mountPath: /truststore + name: truststore + readOnly: true + ports: + - containerPort: 10002 + - containerPort: 10003 + volumes: + - name: artemis + persistentVolumeClaim: + claimName: artemis + - name: cordapps + persistentVolumeClaim: + claimName: cordapps + - name: certificates + persistentVolumeClaim: + claimName: certificates + - name: truststore + secret: + secretName: truststore-3.0.0 diff --git a/experimental/kubernetes/templates/pods/hot-warm.yml b/experimental/kubernetes/templates/pods/hot-warm.yml new file mode 100644 index 0000000000..1df66c96e3 --- /dev/null +++ b/experimental/kubernetes/templates/pods/hot-warm.yml @@ -0,0 +1,62 @@ +apiVersion: "apps/v1beta1" +kind: Deployment +metadata: + name: {{ name }} +spec: + replicas: 0 + selector: + matchLabels: + app: {{ name }} + template: + metadata: + labels: + app: {{ name }} + spec: + imagePullSecrets: + - name: regsecret + containers: + - name: node + image: {{ image }} + imagePullPolicy: Always + env: + - name: "CONFIG_FILE" + value: "/config/hot-warm.conf" + - name: "COMPATIBILITY_ZONE_URL" + value: "http://doorman:1300" + volumeMounts: + - mountPath: /config + name: config + readOnly: true + - mountPath: /app/artemis + name: artemis + - mountPath: /app/cordapps + name: cordapps + - mountPath: /app/certificates + name: certificates + - mountPath: /truststore + name: truststore + readOnly: true + ports: + - containerPort: 10002 + - containerPort: 10003 + readinessProbe: + tcpSocket: + port: 10002 + initialDelaySeconds: 5 + periodSeconds: 5 + volumes: + - name: config + configMap: + name: corda + - name: artemis + persistentVolumeClaim: + claimName: artemis + - name: cordapps + persistentVolumeClaim: + claimName: cordapps + - name: certificates + persistentVolumeClaim: + claimName: certificates + - name: truststore + secret: + secretName: truststore-3.0.0 diff --git a/experimental/kubernetes/templates/pods/notary-init.yml b/experimental/kubernetes/templates/pods/notary-init.yml new file mode 100644 index 0000000000..e4d4e5009a --- /dev/null +++ b/experimental/kubernetes/templates/pods/notary-init.yml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: Pod +metadata: + name: notary + labels: + app: notary +spec: + imagePullSecrets: + - name: regsecret + containers: + - name: notary + image: {{ image }} + imagePullPolicy: Always + volumeMounts: + - mountPath: /data + name: pv + - mountPath: /truststore + name: truststore + readOnly: true + ports: + - containerPort: 1300 + env: + - name: "NAMESPACE" + value: "{{ namespace }}" + command: + - '/app/init.sh' + volumes: + - name: pv + persistentVolumeClaim: + claimName: notary + - name: truststore + secret: + secretName: truststore-3.0.0 diff --git a/experimental/kubernetes/templates/pods/notary.yml b/experimental/kubernetes/templates/pods/notary.yml new file mode 100644 index 0000000000..b7e6e8d26d --- /dev/null +++ b/experimental/kubernetes/templates/pods/notary.yml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Pod +metadata: + name: notary + labels: + app: notary +spec: + imagePullSecrets: + - name: regsecret + containers: + - name: notary + image: {{ image }} + imagePullPolicy: Always + volumeMounts: + - mountPath: /data + name: pv + - mountPath: /truststore + name: truststore + readOnly: true + env: + - name: "NAMESPACE" + value: "{{ namespace }}" + ports: + - containerPort: 1300 + volumes: + - name: pv + persistentVolumeClaim: + claimName: notary + - name: truststore + secret: + secretName: truststore-3.0.0 diff --git a/experimental/kubernetes/templates/pods/zk.yml b/experimental/kubernetes/templates/pods/zk.yml new file mode 100644 index 0000000000..0c8fbbcda7 --- /dev/null +++ b/experimental/kubernetes/templates/pods/zk.yml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Pod +metadata: + name: zk + labels: + app: zk +spec: + containers: + - name: zk + image: zookeeper:3.5 + ports: + - containerPort: 2181 + - containerPort: 2888 + - containerPort: 3888 diff --git a/experimental/kubernetes/templates/service.yml.j2 b/experimental/kubernetes/templates/service.yml.j2 new file mode 100644 index 0000000000..96e1354f5a --- /dev/null +++ b/experimental/kubernetes/templates/service.yml.j2 @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ version }} +spec: + clusterIP: None + ports: + - name: p2p + port: 10002 + protocol: TCP + targetPort: 10002 + - name: rpc + port: 10003 + protocol: TCP + targetPort: 10003 + selector: + version: {{ version }} diff --git a/experimental/kubernetes/templates/services/db.yml b/experimental/kubernetes/templates/services/db.yml new file mode 100644 index 0000000000..e78227fe43 --- /dev/null +++ b/experimental/kubernetes/templates/services/db.yml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: db +spec: + ports: + - name: postgres + port: 5432 + protocol: TCP + targetPort: 5432 + selector: + app: db diff --git a/experimental/kubernetes/templates/services/doorman.yml b/experimental/kubernetes/templates/services/doorman.yml new file mode 100644 index 0000000000..886685ba2c --- /dev/null +++ b/experimental/kubernetes/templates/services/doorman.yml @@ -0,0 +1,11 @@ +kind: Service +apiVersion: v1 +metadata: + name: doorman +spec: + selector: + app: doorman + ports: + - protocol: TCP + port: 1300 + targetPort: 1300 diff --git a/experimental/kubernetes/templates/services/ha.yml b/experimental/kubernetes/templates/services/ha.yml new file mode 100644 index 0000000000..73926fad16 --- /dev/null +++ b/experimental/kubernetes/templates/services/ha.yml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: hanode +spec: + ports: + - name: p2p + port: 10002 + protocol: TCP + targetPort: 10002 + - name: rpc + port: 10003 + protocol: TCP + targetPort: 10003 + selector: + app: hanode diff --git a/experimental/kubernetes/templates/services/hotwarm.yml b/experimental/kubernetes/templates/services/hotwarm.yml new file mode 100644 index 0000000000..78c12fed61 --- /dev/null +++ b/experimental/kubernetes/templates/services/hotwarm.yml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: hot-warm +spec: + ports: + - name: p2p + port: 10002 + protocol: TCP + targetPort: 10002 + - name: rpc + port: 10003 + protocol: TCP + targetPort: 10003 + selector: + app: hot-warm diff --git a/experimental/kubernetes/templates/services/notary.yml b/experimental/kubernetes/templates/services/notary.yml new file mode 100644 index 0000000000..25f91008ba --- /dev/null +++ b/experimental/kubernetes/templates/services/notary.yml @@ -0,0 +1,16 @@ +kind: Service +apiVersion: v1 +metadata: + name: notary +spec: + selector: + app: notary + ports: + - protocol: TCP + port: 10001 + targetPort: 10001 + name: p2p + - protocol: TCP + port: 10002 + targetPort: 10002 + name: rpc diff --git a/experimental/kubernetes/templates/services/zk.yml b/experimental/kubernetes/templates/services/zk.yml new file mode 100644 index 0000000000..f3911fc257 --- /dev/null +++ b/experimental/kubernetes/templates/services/zk.yml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: zk +spec: + ports: + - name: client-port + port: 2181 + protocol: TCP + targetPort: 2181 + selector: + app: zk diff --git a/experimental/kubernetes/templates/statefulset.yml.j2 b/experimental/kubernetes/templates/statefulset.yml.j2 new file mode 100644 index 0000000000..51d3d0b2d2 --- /dev/null +++ b/experimental/kubernetes/templates/statefulset.yml.j2 @@ -0,0 +1,54 @@ +apiVersion: "apps/v1beta1" +kind: StatefulSet +metadata: + name: {{ version }} +spec: + serviceName: {{ version }} + replicas: 0 + template: + metadata: + labels: + version: {{ version }} + spec: + imagePullSecrets: + - name: regsecret + containers: + - name: corda + image: {{ image }} + imagePullPolicy: Always + ports: + - name: p2p + containerPort: 10002 + - name: rpc + containerPort: 10003 + - name: admin + containerPort: 10004 + env: + - name: "P2P_ADDRESS" + value: "$HOSTNAME.{{ version }}.{{ namespace }}.svc.cluster.local:10002" + - name: "RPC_ADDRESS" + value: "$HOSTNAME.{{ version }}.{{ namespace }}.svc.cluster.local:10003" + - name: "ADMIN_ADDRESS" + value: "$HOSTNAME.{{ version }}.{{ namespace }}.svc.cluster.local:10004" + - name: "CONFIG_FILE" + value: "/config/node.conf" + volumeMounts: + - name: config + mountPath: /config + readOnly: true + - name: truststore + mountPath: /truststore + readOnly: true + - name: cordapps + mountPath: /app/cordapps + readOnly: true + volumes: + - name: config + configMap: + name: corda + - name: truststore + secret: + secretName: truststore-3.0.0 + - name: cordapps + persistentVolumeClaim: + claimName: cordapps