From f4c1119727b7cee6f99a73eb7d6b36373d5034b8 Mon Sep 17 00:00:00 2001 From: Razvan Codreanu <52859362+Schife@users.noreply.github.com> Date: Fri, 17 Apr 2020 15:53:10 +0100 Subject: [PATCH 1/3] INFRA-284 switching from local k8s label (#6156) --- .ci/dev/nightly-regression/Jenkinsfile | 2 +- .ci/dev/on-demand-tests/Jenkinsfile | 2 +- .ci/dev/regression/Jenkinsfile | 2 +- .ci/dev/smoke/Jenkinsfile | 2 +- .ci/dev/unit/Jenkinsfile | 2 +- Jenkinsfile | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.ci/dev/nightly-regression/Jenkinsfile b/.ci/dev/nightly-regression/Jenkinsfile index de26a41c90..7dd4301440 100644 --- a/.ci/dev/nightly-regression/Jenkinsfile +++ b/.ci/dev/nightly-regression/Jenkinsfile @@ -4,7 +4,7 @@ import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) pipeline { - agent { label 'local-k8s' } + agent { label 'k8s' } options { timestamps() overrideIndexTriggers(false) diff --git a/.ci/dev/on-demand-tests/Jenkinsfile b/.ci/dev/on-demand-tests/Jenkinsfile index f59d3d67d0..25127ef133 100644 --- a/.ci/dev/on-demand-tests/Jenkinsfile +++ b/.ci/dev/on-demand-tests/Jenkinsfile @@ -3,4 +3,4 @@ import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) -onDemandTestPipeline('local-k8s', '.ci/dev/on-demand-tests/commentMappings.yml') +onDemandTestPipeline('k8s', '.ci/dev/on-demand-tests/commentMappings.yml') diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index ed550bd401..41cc2ad218 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -4,7 +4,7 @@ import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) pipeline { - agent { label 'local-k8s' } + agent { label 'k8s' } options { timestamps() buildDiscarder(logRotator(daysToKeepStr: '7', artifactDaysToKeepStr: '7')) diff --git a/.ci/dev/smoke/Jenkinsfile b/.ci/dev/smoke/Jenkinsfile index 05aec41e59..3ddc3cdce8 100644 --- a/.ci/dev/smoke/Jenkinsfile +++ b/.ci/dev/smoke/Jenkinsfile @@ -4,7 +4,7 @@ import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) pipeline { - agent { label 'local-k8s' } + agent { label 'k8s' } options { timestamps() overrideIndexTriggers(false) diff --git a/.ci/dev/unit/Jenkinsfile b/.ci/dev/unit/Jenkinsfile index 98f43b4428..65a6cc08ae 100644 --- a/.ci/dev/unit/Jenkinsfile +++ b/.ci/dev/unit/Jenkinsfile @@ -5,7 +5,7 @@ import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) pipeline { - agent { label 'local-k8s' } + agent { label 'k8s' } options { timestamps() timeout(time: 3, unit: 'HOURS') diff --git a/Jenkinsfile b/Jenkinsfile index b81f50ed61..5f02f89de0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,7 @@ import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) pipeline { - agent { label 'local-k8s' } + agent { label 'k8s' } options { timestamps() timeout(time: 3, unit: 'HOURS') From 02d21c7bac51238f52be694716439561ea51c97e Mon Sep 17 00:00:00 2001 From: nikinagy <61757742+nikinagy@users.noreply.github.com> Date: Wed, 22 Apr 2020 13:34:17 +0100 Subject: [PATCH 2/3] making sure hibernate uses UTC time zone (#6168) --- .../corda/nodeapi/internal/persistence/HibernateConfiguration.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt index 0fca5b23bd..ebd023f155 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt @@ -91,6 +91,7 @@ class HibernateConfiguration( .setProperty("hibernate.hbm2ddl.auto", hbm2dll) .setProperty("javax.persistence.validation.mode", "none") .setProperty("hibernate.connection.isolation", databaseConfig.transactionIsolationLevel.jdbcValue.toString()) + .setProperty("hibernate.jdbc.time_zone", "UTC") schemas.forEach { schema -> // TODO: require mechanism to set schemaOptions (databaseSchema, tablePrefix) which are not global to session From 69a4f80cd22f0885f82ecbb61efd22b3e71c7358 Mon Sep 17 00:00:00 2001 From: Joseph Zuniga-Daly <59851625+josephzunigadaly@users.noreply.github.com> Date: Thu, 23 Apr 2020 13:11:23 +0100 Subject: [PATCH 3/3] ENT-5141: Fix ConcurrentModificationException in FetchDataFlow (#6176) * ENT-5141: Fix ConcurrentModificationException in FetchDataFlow * Make detekt happy * Make CheckpointSerializationEnvironmentRule inheritable --- .../kryo/DefaultKryoCustomizer.kt | 4 + .../serialization/kryo/IteratorSerializer.kt | 52 ++++++++ ...yListItrConcurrentModificationException.kt | 122 ++++++++++++++++++ 3 files changed, 178 insertions(+) create mode 100644 node/src/main/kotlin/net/corda/node/serialization/kryo/IteratorSerializer.kt create mode 100644 node/src/test/kotlin/net/corda/node/serialization/kryo/ArrayListItrConcurrentModificationException.kt diff --git a/node/src/main/kotlin/net/corda/node/serialization/kryo/DefaultKryoCustomizer.kt b/node/src/main/kotlin/net/corda/node/serialization/kryo/DefaultKryoCustomizer.kt index b8130dce5f..504b17aff6 100644 --- a/node/src/main/kotlin/net/corda/node/serialization/kryo/DefaultKryoCustomizer.kt +++ b/node/src/main/kotlin/net/corda/node/serialization/kryo/DefaultKryoCustomizer.kt @@ -129,6 +129,10 @@ object DefaultKryoCustomizer { register(ContractUpgradeWireTransaction::class.java, ContractUpgradeWireTransactionSerializer) register(ContractUpgradeFilteredTransaction::class.java, ContractUpgradeFilteredTransactionSerializer) + addDefaultSerializer(Iterator::class.java) {kryo, type -> + IteratorSerializer(type, CompatibleFieldSerializer>(kryo, type).apply { setIgnoreSyntheticFields(false) }) + } + for (whitelistProvider in serializationWhitelists) { val types = whitelistProvider.whitelist require(types.toSet().size == types.size) { diff --git a/node/src/main/kotlin/net/corda/node/serialization/kryo/IteratorSerializer.kt b/node/src/main/kotlin/net/corda/node/serialization/kryo/IteratorSerializer.kt new file mode 100644 index 0000000000..382ae840c5 --- /dev/null +++ b/node/src/main/kotlin/net/corda/node/serialization/kryo/IteratorSerializer.kt @@ -0,0 +1,52 @@ +package net.corda.node.serialization.kryo + +import com.esotericsoftware.kryo.Kryo +import com.esotericsoftware.kryo.Serializer +import com.esotericsoftware.kryo.io.Input +import com.esotericsoftware.kryo.io.Output +import java.lang.reflect.Field + +class IteratorSerializer(type: Class<*>, private val serializer: Serializer>) : Serializer>(false, false) { + + private val iterableReferenceField = findField(type, "this\$0")?.apply { isAccessible = true } + private val expectedModCountField = findField(type, "expectedModCount")?.apply { isAccessible = true } + private val iterableReferenceFieldType = iterableReferenceField?.type + private val modCountField = when (iterableReferenceFieldType) { + null -> null + else -> findField(iterableReferenceFieldType, "modCount")?.apply { isAccessible = true } + } + + override fun write(kryo: Kryo, output: Output, obj: Iterator<*>) { + serializer.write(kryo, output, obj) + } + + override fun read(kryo: Kryo, input: Input, type: Class>): Iterator<*> { + val iterator = serializer.read(kryo, input, type) + return fixIterator(iterator) + } + + private fun fixIterator(iterator: Iterator<*>) : Iterator<*> { + + // Set expectedModCount of iterator + val iterableInstance = iterableReferenceField?.get(iterator) ?: return iterator + val modCountValue = modCountField?.getInt(iterableInstance) ?: return iterator + expectedModCountField?.setInt(iterator, modCountValue) + + return iterator + } + + /** + * Find field in clazz or any superclass + */ + private fun findField(clazz: Class<*>, fieldName: String): Field? { + return clazz.declaredFields.firstOrNull { x -> x.name == fieldName } ?: when { + clazz.superclass != null -> { + // Look in superclasses + findField(clazz.superclass, fieldName) + } + else -> null // Not found + } + } +} + + diff --git a/node/src/test/kotlin/net/corda/node/serialization/kryo/ArrayListItrConcurrentModificationException.kt b/node/src/test/kotlin/net/corda/node/serialization/kryo/ArrayListItrConcurrentModificationException.kt new file mode 100644 index 0000000000..44a48c793d --- /dev/null +++ b/node/src/test/kotlin/net/corda/node/serialization/kryo/ArrayListItrConcurrentModificationException.kt @@ -0,0 +1,122 @@ +package net.corda.node.serialization.kryo + +import com.nhaarman.mockito_kotlin.doReturn +import com.nhaarman.mockito_kotlin.whenever +import net.corda.core.serialization.EncodingWhitelist +import net.corda.core.serialization.internal.CheckpointSerializationContext +import net.corda.core.serialization.internal.checkpointDeserialize +import net.corda.core.serialization.internal.checkpointSerialize +import net.corda.serialization.internal.AllWhitelist +import net.corda.serialization.internal.CheckpointSerializationContextImpl +import net.corda.serialization.internal.CordaSerializationEncoding +import net.corda.testing.core.internal.CheckpointSerializationEnvironmentRule +import net.corda.testing.internal.rigorousMock +import org.assertj.core.api.Assertions.assertThat +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.Parameterized +import org.junit.runners.Parameterized.Parameters +import java.util.* +import kotlin.collections.ArrayList +import kotlin.collections.HashMap +import kotlin.collections.HashSet +import kotlin.collections.LinkedHashMap +import kotlin.collections.LinkedHashSet + +@RunWith(Parameterized::class) +class ArrayListItrConcurrentModificationException(private val compression: CordaSerializationEncoding?) { + companion object { + @Parameters(name = "{0}") + @JvmStatic + fun compression() = arrayOf(null) + CordaSerializationEncoding.values() + } + + @get:Rule + val serializationRule = CheckpointSerializationEnvironmentRule(inheritable = true) + private lateinit var context: CheckpointSerializationContext + + @Before + fun setup() { + context = CheckpointSerializationContextImpl( + deserializationClassLoader = javaClass.classLoader, + whitelist = AllWhitelist, + properties = emptyMap(), + objectReferencesEnabled = true, + encoding = compression, + encodingWhitelist = rigorousMock().also { + if (compression != null) doReturn(true).whenever(it).acceptEncoding(compression) + }) + } + + @Test(timeout=300_000) + fun `ArrayList iterator can checkpoint without error`() { + runTestWithCollection(ArrayList()) + } + + @Test(timeout=300_000) + fun `HashSet iterator can checkpoint without error`() { + runTestWithCollection(HashSet()) + } + + @Test(timeout=300_000) + fun `LinkedHashSet iterator can checkpoint without error`() { + runTestWithCollection(LinkedHashSet()) + } + + @Test(timeout=300_000) + fun `HashMap iterator can checkpoint without error`() { + runTestWithCollection(HashMap()) + } + + @Test(timeout=300_000) + fun `LinkedHashMap iterator can checkpoint without error`() { + runTestWithCollection(LinkedHashMap()) + } + + @Test(timeout=300_000) + fun `LinkedList iterator can checkpoint without error`() { + runTestWithCollection(LinkedList()) + } + + private data class TestCheckpoint(val list: C, val iterator: I) + + private fun runTestWithCollection(collection: MutableCollection) { + + for (i in 1..100) { + collection.add(i) + } + + val iterator = collection.iterator() + iterator.next() + + val checkpoint = TestCheckpoint(collection, iterator) + + val serializedBytes = checkpoint.checkpointSerialize(context) + val deserializedCheckpoint = serializedBytes.checkpointDeserialize(context) + + assertThat(deserializedCheckpoint.list).isEqualTo(collection) + assertThat(deserializedCheckpoint.iterator.next()).isEqualTo(2) + assertThat(deserializedCheckpoint.iterator.hasNext()).isTrue() + } + + private fun runTestWithCollection(collection: MutableMap) { + + for (i in 1..100) { + collection[i] = i + } + + val iterator = collection.iterator() + iterator.next() + + val checkpoint = TestCheckpoint(collection, iterator) + + val serializedBytes = checkpoint.checkpointSerialize(context) + val deserializedCheckpoint = serializedBytes.checkpointDeserialize(context) + + assertThat(deserializedCheckpoint.list).isEqualTo(collection) + assertThat(deserializedCheckpoint.iterator.next().key).isEqualTo(2) + assertThat(deserializedCheckpoint.iterator.hasNext()).isTrue() + } +}