Merge pull request #5723 from corda/my_merge_branch

Merge OS4.3 -> OS4.4
This commit is contained in:
Stefano Franz 2019-11-18 19:26:07 +00:00 committed by GitHub
commit 5f819c1917
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 169 additions and 48 deletions

View File

@ -60,7 +60,7 @@ pipeline {
pullRequest.createStatus(status: 'success',
context: 'continuous-integration/jenkins/pr-merge/smokeTest',
description: 'Smoke Tests Passed',
targetUrl: "${env.JOB_URL}testResults")
targetUrl: "${env.BUILD_URL}testResults")
}
}
}
@ -71,7 +71,7 @@ pipeline {
pullRequest.createStatus(status: 'failure',
context: 'continuous-integration/jenkins/pr-merge/smokeTest',
description: 'Smoke Tests Failed',
targetUrl: "${env.JOB_URL}testResults")
targetUrl: "${env.BUILD_URL}testResults")
}
}
}

View File

@ -7,6 +7,8 @@ import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.tasks.testing.Test
import org.gradle.api.tasks.testing.TestResult
import org.gradle.internal.impldep.junit.framework.TestFailure
import java.util.stream.Collectors
@ -95,6 +97,7 @@ class DistributedTesting implements Plugin<Project> {
//modify the image building task to depend on the preAllocate task (if specified on the command line) - this prevents gradle running out of order
if (preAllocateTask.name in requestedTaskNames) {
imageBuildTask.dependsOn preAllocateTask
imagePushTask.finalizedBy(deAllocateTask)
}
def userDefinedParallelTask = project.rootProject.tasks.create("userDefined" + testGrouping.getName().capitalize(), KubesTest) {
@ -117,6 +120,8 @@ class DistributedTesting implements Plugin<Project> {
podLogLevel = testGrouping.getLogLevel()
doFirst {
dockerTag = tagToUseForRunningTests ? (ImageBuilding.registryName + ":" + tagToUseForRunningTests) : (imagePushTask.imageName.get() + ":" + imagePushTask.tag.get())
sidecarImage = testGrouping.sidecarImage
additionalArgs = testGrouping.additionalArgs
}
}
def reportOnAllTask = project.rootProject.tasks.create("userDefinedReports${testGrouping.getName().capitalize()}", KubesReporting) {
@ -167,7 +172,8 @@ class DistributedTesting implements Plugin<Project> {
Task deAllocateTask = project.rootProject.tasks.create("deAllocateFor" + testGrouping.getName().capitalize()) {
group = GRADLE_GROUP
doFirst {
String dockerTag = System.getProperty(ImageBuilding.PROVIDE_TAG_FOR_RUNNING_PROPERTY)
String dockerTag = System.getProperty(ImageBuilding.PROVIDE_TAG_FOR_RUNNING_PROPERTY) ?:
System.getProperty(ImageBuilding.PROVIDE_TAG_FOR_BUILDING_PROPERTY)
if (dockerTag == null) {
throw new GradleException("pre allocation cannot be used without a stable docker tag - please provide one using -D" + ImageBuilding.PROVIDE_TAG_FOR_RUNNING_PROPERTY)
}
@ -250,8 +256,10 @@ class DistributedTesting implements Plugin<Project> {
}
afterTest { desc, result ->
executedTestsFile.withWriterAppend { writer ->
writer.writeLine(desc.getClassName() + "." + desc.getName())
if (result.getResultType() == TestResult.ResultType.SUCCESS ) {
executedTestsFile.withWriterAppend { writer ->
writer.writeLine(desc.getClassName() + "." + desc.getName())
}
}
}
}

View File

@ -18,7 +18,6 @@ import io.fabric8.kubernetes.client.dsl.PodResource;
import io.fabric8.kubernetes.client.utils.Serialization;
import net.corda.testing.retry.Retry;
import okhttp3.Response;
import org.apache.commons.compress.utils.IOUtils;
import org.gradle.api.DefaultTask;
import org.gradle.api.tasks.TaskAction;
import org.jetbrains.annotations.NotNull;
@ -27,8 +26,6 @@ import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
@ -36,23 +33,25 @@ import java.io.InputStreamReader;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.math.BigInteger;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Optional;
import java.util.Queue;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@ -65,22 +64,26 @@ public class KubesTest extends DefaultTask {
*/
private static final String REGISTRY_CREDENTIALS_SECRET_NAME = "regcred";
private static int DEFAULT_K8S_TIMEOUT_VALUE_MILLIES = 60 * 1_000;
private static int DEFAULT_K8S_WEBSOCKET_TIMEOUT = DEFAULT_K8S_TIMEOUT_VALUE_MILLIES * 30;
private static int DEFAULT_POD_ALLOCATION_TIMEOUT = 60;
String dockerTag;
String fullTaskToExecutePath;
String taskToExecuteName;
String sidecarImage;
Boolean printOutput = false;
List<String> additionalArgs;
Integer numberOfCoresPerFork = 4;
Integer memoryGbPerFork = 6;
public volatile List<File> testOutput = Collections.emptyList();
public volatile List<KubePodResult> containerResults = Collections.emptyList();
private final List<String> remainingPods = Collections.synchronizedList(new ArrayList());
private final Set<String> remainingPods = Collections.synchronizedSet(new HashSet());
public static String NAMESPACE = "thisisatest";
int k8sTimeout = 50 * 1_000;
int webSocketTimeout = k8sTimeout * 6;
int numberOfPods = 5;
int timeoutInMinutesForPodToStart = 60;
DistributeTestsBy distribution = DistributeTestsBy.METHOD;
PodLogLevel podLogLevel = PodLogLevel.INFO;
@ -138,11 +141,11 @@ public class KubesTest extends DefaultTask {
@NotNull
private KubernetesClient getKubernetesClient() {
io.fabric8.kubernetes.client.Config config = new io.fabric8.kubernetes.client.ConfigBuilder()
.withConnectionTimeout(k8sTimeout)
.withRequestTimeout(k8sTimeout)
.withRollingTimeout(k8sTimeout)
.withWebsocketTimeout(webSocketTimeout)
.withWebsocketPingInterval(webSocketTimeout)
.withConnectionTimeout(DEFAULT_K8S_TIMEOUT_VALUE_MILLIES)
.withRequestTimeout(DEFAULT_K8S_TIMEOUT_VALUE_MILLIES)
.withRollingTimeout(DEFAULT_K8S_TIMEOUT_VALUE_MILLIES)
.withWebsocketTimeout(DEFAULT_K8S_WEBSOCKET_TIMEOUT)
.withWebsocketPingInterval(DEFAULT_K8S_WEBSOCKET_TIMEOUT)
.build();
return new DefaultKubernetesClient(config);
@ -212,7 +215,7 @@ public class KubesTest extends DefaultTask {
});
int podNumber = podIdx + 1;
final AtomicInteger testRetries = new AtomicInteger(0);
try {
// pods might die, so we retry
return Retry.fixed(numberOfRetries).run(() -> {
@ -229,7 +232,7 @@ public class KubesTest extends DefaultTask {
}
}
getProject().getLogger().lifecycle("creating pod: " + podName);
createdPod = client.pods().inNamespace(namespace).create(buildPodRequest(podName, pvc));
createdPod = client.pods().inNamespace(namespace).create(buildPodRequest(podName, pvc, sidecarImage != null));
remainingPods.add(podName);
getProject().getLogger().lifecycle("scheduled pod: " + podName);
}
@ -246,11 +249,20 @@ public class KubesTest extends DefaultTask {
if (!podLogsDirectory.exists()) {
podLogsDirectory.mkdirs();
}
File podOutput = executeBuild(namespace, numberOfPods, podIdx, podName, podLogsDirectory, printOutput, stdOutOs, stdOutIs, errChannelStream, waiter);
File podOutput = executeBuild(namespace, numberOfPods, podIdx, podName, podLogsDirectory, printOutput, stdOutOs, stdOutIs, errChannelStream, waiter);
int resCode = waiter.join();
getProject().getLogger().lifecycle("build has ended on on pod " + podName + " (" + podNumber + "/" + numberOfPods + ") with result " + resCode + " , gathering results");
Collection<File> binaryResults = downloadTestXmlFromPod(namespace, createdPod);
Collection<File> binaryResults;
//we don't retry on the final attempt as this will crash the build and some pods might not get to finish
if (resCode != 0 && testRetries.getAndIncrement() < numberOfRetries - 1) {
downloadTestXmlFromPod(namespace, createdPod);
getProject().getLogger().lifecycle("There are test failures in this pod. Retrying failed tests!!!");
throw new RuntimeException("There are test failures in this pod");
} else {
binaryResults = downloadTestXmlFromPod(namespace, createdPod);
}
getLogger().lifecycle("removing pod " + podName + " (" + podNumber + "/" + numberOfPods + ") after completed build");
try (KubernetesClient client = getKubernetesClient()) {
@ -265,6 +277,8 @@ public class KubesTest extends DefaultTask {
return new KubePodResult(podIdx, resCode, podOutput, binaryResults);
});
} catch (Retry.RetryException e) {
Pod pod = getKubernetesClient().pods().inNamespace(namespace).create(buildPodRequest(podName, pvc, sidecarImage != null));
downloadTestXmlFromPod(namespace, pod);
throw new RuntimeException("Failed to build in pod " + podName + " (" + podNumber + "/" + numberOfPods + ") in " + numberOfRetries + " attempts", e);
}
}
@ -287,20 +301,27 @@ public class KubesTest extends DefaultTask {
String[] buildCommand = getBuildCommand(numberOfPods, podIdx);
getProject().getLogger().quiet("About to execute " + Arrays.stream(buildCommand).reduce("", (s, s2) -> s + " " + s2) + " on pod " + podName);
client.pods().inNamespace(namespace).withName(podName)
.inContainer(podName)
.writingOutput(stdOutOs)
.writingErrorChannel(errChannelStream)
.usingListener(execListener)
.exec(getBuildCommand(numberOfPods, podIdx));
.exec(buildCommand);
return startLogPumping(stdOutIs, podIdx, podLogsDirectory, printOutput);
}
private Pod buildPodRequest(String podName, PersistentVolumeClaim pvc) {
private Pod buildPodRequest(String podName, PersistentVolumeClaim pvc, boolean withDb) {
if (withDb) {
return buildPodRequestWithWorkerNodeAndDbContainer(podName, pvc);
} else {
return buildPodRequestWithOnlyWorkerNode(podName, pvc);
}
}
private Pod buildPodRequestWithOnlyWorkerNode(String podName, PersistentVolumeClaim pvc) {
return new PodBuilder()
.withNewMetadata().withName(podName).endMetadata()
.withNewSpec()
.addNewVolume()
.withName("gradlecache")
.withNewHostPath()
@ -327,19 +348,77 @@ public class KubesTest extends DefaultTask {
.withName(podName)
.withNewResources()
.addToRequests("cpu", new Quantity(numberOfCoresPerFork.toString()))
.addToRequests("memory", new Quantity(memoryGbPerFork.toString() + "Gi"))
.addToRequests("memory", new Quantity(memoryGbPerFork.toString()))
.endResources()
.addNewVolumeMount().withName("gradlecache").withMountPath("/tmp/gradle").endVolumeMount()
.addNewVolumeMount().withName("testruns").withMountPath(TEST_RUN_DIR).endVolumeMount()
.endContainer()
.addNewImagePullSecret(REGISTRY_CREDENTIALS_SECRET_NAME)
.withRestartPolicy("Never")
.endSpec()
.build();
}
private Pod buildPodRequestWithWorkerNodeAndDbContainer(String podName, PersistentVolumeClaim pvc) {
return new PodBuilder()
.withNewMetadata().withName(podName).endMetadata()
.withNewSpec()
.addNewVolume()
.withName("gradlecache")
.withNewHostPath()
.withType("DirectoryOrCreate")
.withPath("/tmp/gradle")
.endHostPath()
.endVolume()
.addNewVolume()
.withName("testruns")
.withNewPersistentVolumeClaim()
.withClaimName(pvc.getMetadata().getName())
.endPersistentVolumeClaim()
.endVolume()
.addNewContainer()
.withImage(dockerTag)
.withCommand("bash")
.withArgs("-c", "sleep 3600")
.addNewEnv()
.withName("DRIVER_NODE_MEMORY")
.withValue("1024m")
.withName("DRIVER_WEB_MEMORY")
.withValue("1024m")
.endEnv()
.withName(podName)
.withNewResources()
.addToRequests("cpu", new Quantity(Integer.valueOf(numberOfCoresPerFork - 1).toString()))
.addToRequests("memory", new Quantity(Integer.valueOf(memoryGbPerFork - 1).toString() + "Gi"))
.endResources()
.addNewVolumeMount().withName("gradlecache").withMountPath("/tmp/gradle").endVolumeMount()
.addNewVolumeMount().withName("testruns").withMountPath(TEST_RUN_DIR).endVolumeMount()
.endContainer()
.addNewContainer()
.withImage(sidecarImage)
.addNewEnv()
.withName("DRIVER_NODE_MEMORY")
.withValue("1024m")
.withName("DRIVER_WEB_MEMORY")
.withValue("1024m")
.endEnv()
.withName(podName + "-pg")
.withNewResources()
.addToRequests("cpu", new Quantity("1"))
.addToRequests("memory", new Quantity("1Gi"))
.endResources()
.endContainer()
.addNewImagePullSecret(REGISTRY_CREDENTIALS_SECRET_NAME)
.withRestartPolicy("Never")
.endSpec()
.build();
}
private File startLogPumping(InputStream stdOutIs, int podIdx, File podLogsDirectory, boolean printOutput) throws IOException {
File outputFile = new File(podLogsDirectory, "container-" + podIdx + ".log");
outputFile.createNewFile();
@ -383,7 +462,7 @@ public class KubesTest extends DefaultTask {
try (KubernetesClient client = getKubernetesClient()) {
getProject().getLogger().lifecycle("Waiting for pod " + pod.getMetadata().getName() + " to start before executing build");
try {
client.pods().inNamespace(pod.getMetadata().getNamespace()).withName(pod.getMetadata().getName()).waitUntilReady(timeoutInMinutesForPodToStart, TimeUnit.MINUTES);
client.pods().inNamespace(pod.getMetadata().getNamespace()).withName(pod.getMetadata().getName()).waitUntilReady(DEFAULT_POD_ALLOCATION_TIMEOUT, TimeUnit.MINUTES);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
@ -405,6 +484,7 @@ public class KubesTest extends DefaultTask {
client.pods()
.inNamespace(namespace)
.withName(podName)
.inContainer(podName)
.dir(resultsInContainerPath)
.copy(tempDir);
}
@ -416,6 +496,7 @@ public class KubesTest extends DefaultTask {
final String gitTargetBranch = " -Dgit.target.branch=" + Properties.getTargetGitBranch();
final String artifactoryUsername = " -Dartifactory.username=" + Properties.getUsername() + " ";
final String artifactoryPassword = " -Dartifactory.password=" + Properties.getPassword() + " ";
final String additionalArgs = this.additionalArgs.isEmpty() ? "" : String.join(" ", this.additionalArgs);
String shellScript = "(let x=1 ; while [ ${x} -ne 0 ] ; do echo \"Waiting for DNS\" ; curl services.gradle.org > /dev/null 2>&1 ; x=$? ; sleep 1 ; done ) && "
+ " cd /tmp/source && " +
@ -425,7 +506,7 @@ public class KubesTest extends DefaultTask {
gitTargetBranch +
artifactoryUsername +
artifactoryPassword +
"-Dkubenetize -PdockerFork=" + podIdx + " -PdockerForks=" + numberOfPods + " " + fullTaskToExecutePath + " " + getLoggingLevel() + " 2>&1) ; " +
"-Dkubenetize -PdockerFork=" + podIdx + " -PdockerForks=" + numberOfPods + " " + fullTaskToExecutePath + " " + additionalArgs + " " + getLoggingLevel() + " 2>&1) ; " +
"let rs=$? ; sleep 10 ; exit ${rs}";
return new String[]{"bash", "-c", shellScript};
}

View File

@ -15,6 +15,8 @@ public class ParallelTestGroup extends DefaultTask {
private int gbOfMemory = 4;
private boolean printToStdOut = true;
private PodLogLevel logLevel = PodLogLevel.INFO;
private String sidecarImage;
private List<String> additionalArgs = new ArrayList<>();
public DistributeTestsBy getDistribution() {
return distribution;
@ -44,6 +46,10 @@ public class ParallelTestGroup extends DefaultTask {
return logLevel;
}
public String getSidecarImage() { return sidecarImage; }
public List<String> getAdditionalArgs() { return additionalArgs; }
public void numberOfShards(int shards) {
this.shardCount = shards;
}
@ -77,4 +83,16 @@ public class ParallelTestGroup extends DefaultTask {
groups.addAll(group);
}
public void sidecarImage(String sidecarImage) {
this.sidecarImage = sidecarImage;
}
public void additionalArgs(String... additionalArgs) {
additionalArgs(Arrays.asList(additionalArgs));
}
private void additionalArgs(List<String> additionalArgs) {
this.additionalArgs.addAll(additionalArgs);
}
}

View File

@ -1,6 +1,5 @@
package net.corda.testing;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.Quantity;
import io.fabric8.kubernetes.api.model.batch.Job;
import io.fabric8.kubernetes.api.model.batch.JobBuilder;
@ -39,36 +38,38 @@ public class PodAllocator {
public void allocatePods(Integer number, Integer coresPerPod, Integer memoryPerPod, String prefix) {
Config config = new ConfigBuilder()
.withConnectionTimeout(CONNECTION_TIMEOUT)
.withRequestTimeout(CONNECTION_TIMEOUT)
.withRollingTimeout(CONNECTION_TIMEOUT)
.withWebsocketTimeout(CONNECTION_TIMEOUT)
.withWebsocketPingInterval(CONNECTION_TIMEOUT)
.build();
Config config = getConfig();
KubernetesClient client = new DefaultKubernetesClient(config);
List<Job> podsToRequest = IntStream.range(0, number).mapToObj(i -> buildJob("pa-" + prefix + i, coresPerPod, memoryPerPod)).collect(Collectors.toList());
podsToRequest.forEach(requestedJob -> {
List<Job> createdJobs = podsToRequest.stream().map(requestedJob -> {
String msg = "PreAllocating " + requestedJob.getMetadata().getName();
if (logger instanceof org.gradle.api.logging.Logger) {
((org.gradle.api.logging.Logger) logger).quiet(msg);
} else {
logger.info(msg);
}
client.batch().jobs().inNamespace(KubesTest.NAMESPACE).create(requestedJob);
});
return client.batch().jobs().inNamespace(KubesTest.NAMESPACE).create(requestedJob);
}).collect(Collectors.toList());
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
KubernetesClient tearDownClient = new DefaultKubernetesClient(getConfig());
tearDownClient.batch().jobs().delete(createdJobs);
}));
}
public void tearDownPods(String prefix) {
io.fabric8.kubernetes.client.Config config = new io.fabric8.kubernetes.client.ConfigBuilder()
private Config getConfig() {
return new ConfigBuilder()
.withConnectionTimeout(CONNECTION_TIMEOUT)
.withRequestTimeout(CONNECTION_TIMEOUT)
.withRollingTimeout(CONNECTION_TIMEOUT)
.withWebsocketTimeout(CONNECTION_TIMEOUT)
.withWebsocketPingInterval(CONNECTION_TIMEOUT)
.build();
}
public void tearDownPods(String prefix) {
io.fabric8.kubernetes.client.Config config = getConfig();
KubernetesClient client = new DefaultKubernetesClient(config);
Stream<Job> jobsToDelete = client.batch().jobs().inNamespace(KubesTest.NAMESPACE).list()
.getItems()

View File

@ -11,7 +11,7 @@ Welcome to the Corda 4.3 release notes. Please read these carefully to understan
Corda 4.3
=========
Corda 4.1 was released with a great suite of new features to build on top of the success of Corda 4. Now, Corda 4.3 extends upon that with some powerful new capabilities. Corda 4.3 over 400 fixes and documentation updates to bring additional stability and quality of life improvements to those developing on the Corda platform.
Corda 4.1 was released with a great suite of new features to build on top of the success of Corda 4. Now, Corda 4.3 extends upon that with some powerful new capabilities. Corda 4.3 contains over 400 fixes and documentation updates to bring additional stability and quality of life improvements to those developing on the Corda platform.
We recommend you upgrade from Corda 4.1 to Corda 4.3 as soon as possible.
@ -88,6 +88,13 @@ There have been several security upgrades, including changes to the Corda webser
* Enhancements to attachment whitelisting: Transactions referencing contracts that are not installed on a node can still be accepted if the contract is signed by a trusted party.
* Updated vulnerable dependency: Jolokia 1.2 to 1.6.0 are vulnerable to system-wide cross-site-request-forgery attacks. Updated to Jolokia 1.6.1
Platform version change
~~~~~~~~~~~~~~~~~~~~~~~
Given the addition of a new API to support the Accounts feature, the platform version of Corda 4.3 has been bumped up from 4 to 5. This is to prevent CorDapps that use it being deployed onto nodes unable to host them. Note that the minimum platform version has not been changed - this means that older Corda nodes can still interoperate with Corda 4.3 nodes. Since the APIs added do not affect the wire protocol or have other zone-level implications, applications can take advantage of these new platform version 5 features even if the Corda 4.3 node is running on a network whose minimum platform version is 4.
For more information on platform version, please see :doc:`versioning`. For more details on upgrading a CorDapp to use platform version 5, please see :doc:`app-upgrade-notes`.
Deprecations
~~~~~~~~~~~~
@ -95,7 +102,7 @@ The Corda Finance library is now deprecated and has been superseded by the Corda
Any confidential identities registered using the old API will not be reflected in the new tables after migration to Corda 4.3. However, the standard APIs work with both old and new confidential identities tables. For this reason, we do not recommend the use of both old and new confidential identities APIs in the same deployment. The old confidential identities API will be deprecated in a future release.
Issued Fixed
Issues Fixed
~~~~~~~~~~~~
* Register custom serializers for jackson as well as amqp [`CORDA-3152 <https://r3-cev.atlassian.net/browse/CORDA-3152>`_]

View File

@ -2,10 +2,14 @@ package net.corda.testing.driver
import net.corda.core.concurrent.CordaFuture
import net.corda.core.identity.CordaX500Name
import net.corda.core.internal.*
import net.corda.core.internal.CertRole
import net.corda.core.internal.concurrent.fork
import net.corda.core.internal.concurrent.openFuture
import net.corda.core.internal.concurrent.transpose
import net.corda.core.internal.div
import net.corda.core.internal.isRegularFile
import net.corda.core.internal.list
import net.corda.core.internal.readLines
import net.corda.core.utilities.getOrThrow
import net.corda.node.internal.NodeStartup
import net.corda.testing.common.internal.ProjectStructure.projectRootDir
@ -15,7 +19,9 @@ import net.corda.testing.core.DUMMY_BANK_B_NAME
import net.corda.testing.http.HttpApi
import net.corda.testing.node.internal.addressMustBeBound
import net.corda.testing.node.internal.addressMustNotBeBound
import org.assertj.core.api.Assertions.*
import org.assertj.core.api.Assertions.assertThat
import org.assertj.core.api.Assertions.assertThatCode
import org.assertj.core.api.Assertions.assertThatIllegalArgumentException
import org.json.simple.JSONObject
import org.junit.Test
import java.util.*