diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 13fc420f42..3be2d6bd9e 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -108,6 +108,7 @@ see changes to this list. * Lars Stage Thomsen (Danske Bank) * Lee Braine (Barclays) * Lucas Salmen (Itau) +* Lulu Ren (S-Labs) * Maksymilian Pawlak (R3) * Marek Scocovsky (ABSA) * marekdapps @@ -138,7 +139,6 @@ see changes to this list. * Przemyslaw Bak (R3) * quiark * RangerOfFire -* renlulu * Rex Maudsley (Société Générale) * Rhett Brewer (Goldman Sachs) * Richard Crook (RBS) diff --git a/bridge/build.gradle b/bridge/build.gradle index a02dabca6a..545afed651 100644 --- a/bridge/build.gradle +++ b/bridge/build.gradle @@ -54,8 +54,7 @@ dependencies { compile "io.reactivex:rxjava:$rxjava_version" compile("org.apache.activemq:artemis-core-client:${artemis_version}") compile "org.apache.activemq:artemis-commons:${artemis_version}" - // Netty: All of it! Dn't depend upon ActiveMQ to pull it in correctly - compile "io.netty:netty-all:$netty_version" + compile "io.netty:netty-handler-proxy:$netty_version" // TypeSafe Config: for simple and human friendly config files. compile "com.typesafe:config:$typesafe_config_version" @@ -68,7 +67,9 @@ dependencies { // Seems to be needed? compile "com.github.ben-manes.caffeine:caffeine:$caffeine_version" // Pulled in by whitelist - compile "com.esotericsoftware:kryo:4.0.0" + compile ("com.esotericsoftware:kryo:4.0.0") { + transitive = false + } // Log4J: logging framework (with SLF4J bindings) compile "org.apache.logging.log4j:log4j-slf4j-impl:$log4j_version" diff --git a/build.gradle b/build.gradle index 3d2f519807..c170c89e5c 100644 --- a/build.gradle +++ b/build.gradle @@ -276,12 +276,23 @@ allprojects { force "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" force "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version" force "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version" + + // Demand that everything uses our given version of Netty. + eachDependency { details -> + if (details.requested.group == 'io.netty' && details.requested.name.startsWith('netty-')) { + details.useVersion netty_version + } + } } } compile { // We want to use SLF4J's version of these bindings: jcl-over-slf4j // Remove any transitive dependency on Apache's version. exclude group: 'commons-logging', module: 'commons-logging' + + // Netty-All is an uber-jar which contains every Netty module. + // Exclude it to force us to use the individual Netty modules instead. + exclude group: 'io.netty', module: 'netty-all' } runtime { // We never want isolated.jar on classPath, since we want to test jar being dynamically loaded as an attachment diff --git a/core/src/main/kotlin/net/corda/core/flows/FlowLogic.kt b/core/src/main/kotlin/net/corda/core/flows/FlowLogic.kt index 42cb8d518d..c06712dfea 100644 --- a/core/src/main/kotlin/net/corda/core/flows/FlowLogic.kt +++ b/core/src/main/kotlin/net/corda/core/flows/FlowLogic.kt @@ -444,7 +444,8 @@ abstract class FlowLogic { */ var stateMachine: FlowStateMachine<*> @CordaInternal - get() = _stateMachine ?: throw IllegalStateException("This can only be done after the flow has been started.") + get() = _stateMachine ?: throw IllegalStateException( + "You cannot access the flow's state machine until the flow has been started.") @CordaInternal set(value) { _stateMachine = value diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 52fff25e44..810f4a0433 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -6,16 +6,14 @@ release, see :doc:`upgrade-notes`. Unreleased ========== +* Set co.paralleluniverse.fibers.verifyInstrumentation=true in devMode. + * Node will now gracefully fail to start if one of the required ports is already in use. * Node will now gracefully fail to start if ``devMode`` is true and ``compatibilityZoneURL`` is specified. * Added smart detection logic for the development mode setting and an option to override it from the command line. -* Fixed an error thrown by NodeVaultService upon recording a transaction with a number of inputs greater than the default page size. - -* Fixed incorrect computation of ``totalStates`` from ``otherResults`` in ``NodeVaultService``. - * Changes to the JSON/YAML serialisation format from ``JacksonSupport``, which also applies to the node shell: * ``Instant`` and ``Date`` objects are serialised as ISO-8601 formatted strings rather than timestamps diff --git a/node-api/build.gradle b/node-api/build.gradle index ea17afa9f1..15bcdfc640 100644 --- a/node-api/build.gradle +++ b/node-api/build.gradle @@ -30,8 +30,7 @@ dependencies { compile "org.apache.activemq:artemis-core-client:${artemis_version}" compile "org.apache.activemq:artemis-commons:${artemis_version}" - // Netty: All of it. - compile "io.netty:netty-all:$netty_version" + compile "io.netty:netty-handler-proxy:$netty_version" // For adding serialisation of file upload streams to RPC // TODO: Remove this dependency and the code that requires it diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/SerializationScheme.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/SerializationScheme.kt index ab153b0725..2514b93652 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/SerializationScheme.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/SerializationScheme.kt @@ -102,7 +102,12 @@ internal class AttachmentsClassLoaderBuilder(private val properties: Map, SerializationScheme> +) : SerializationFactory() { + constructor() : this(ConcurrentHashMap()) + companion object { val magicSize = sequenceOf(kryoMagic, amqpMagic).map { it.size }.distinct().single() } @@ -113,9 +118,6 @@ open class SerializationFactoryImpl : SerializationFactory() { private val logger = LoggerFactory.getLogger(javaClass) - // TODO: This is read-mostly. Probably a faster implementation to be found. - private val schemes: ConcurrentHashMap, SerializationScheme> = ConcurrentHashMap() - private fun schemeFor(byteSequence: ByteSequence, target: SerializationContext.UseCase): Pair { // truncate sequence to at most magicSize, and make sure it's a copy to avoid holding onto large ByteArrays val magic = CordaSerializationMagic(byteSequence.slice(end = magicSize).copyBytes()) diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializationScheme.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializationScheme.kt index d65067b58a..27bb08bc38 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializationScheme.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializationScheme.kt @@ -45,10 +45,11 @@ interface SerializerFactoryFactory { } abstract class AbstractAMQPSerializationScheme( - private val cordappCustomSerializers: Set>, - val sff: SerializerFactoryFactory = createSerializerFactoryFactory() + private val cordappCustomSerializers: Set>, + private val serializerFactoriesForContexts: MutableMap, SerializerFactory>, + val sff: SerializerFactoryFactory = createSerializerFactoryFactory() ) : SerializationScheme { - constructor(cordapps: List) : this(cordapps.customSerializers) + constructor(cordapps: List) : this(cordapps.customSerializers, ConcurrentHashMap()) // TODO: This method of initialisation for the Whitelist and plugin serializers will have to change // when we have per-cordapp contexts and dynamic app reloading but for now it's the easiest way @@ -138,8 +139,6 @@ abstract class AbstractAMQPSerializationScheme( } } - private val serializerFactoriesForContexts = ConcurrentHashMap, SerializerFactory>() - protected abstract fun rpcClientSerializerFactory(context: SerializationContext): SerializerFactory protected abstract fun rpcServerSerializerFactory(context: SerializationContext): SerializerFactory protected open val publicKeySerializer: CustomSerializer.Implements = net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer @@ -175,9 +174,13 @@ abstract class AbstractAMQPSerializationScheme( } // TODO: This will eventually cover server RPC as well and move to node module, but for now this is not implemented -class AMQPServerSerializationScheme(cordappCustomSerializers: Set> = emptySet()) - : AbstractAMQPSerializationScheme(cordappCustomSerializers) { - constructor(cordapps: List) : this(cordapps.customSerializers) +class AMQPServerSerializationScheme( + cordappCustomSerializers: Set>, + serializerFactoriesForContexts: MutableMap, SerializerFactory> +) : AbstractAMQPSerializationScheme(cordappCustomSerializers, serializerFactoriesForContexts) { + constructor(cordapps: List) : this(cordapps.customSerializers, ConcurrentHashMap()) + + constructor() : this(emptySet(), ConcurrentHashMap()) override fun rpcClientSerializerFactory(context: SerializationContext): SerializerFactory { throw UnsupportedOperationException() @@ -195,9 +198,13 @@ class AMQPServerSerializationScheme(cordappCustomSerializers: Set> = emptySet()) - : AbstractAMQPSerializationScheme(cordappCustomSerializers) { - constructor(cordapps: List) : this(cordapps.customSerializers) +class AMQPClientSerializationScheme( + cordappCustomSerializers: Set>, + serializerFactoriesForContexts: MutableMap, SerializerFactory> +) : AbstractAMQPSerializationScheme(cordappCustomSerializers, serializerFactoriesForContexts) { + constructor(cordapps: List) : this(cordapps.customSerializers, ConcurrentHashMap()) + + constructor() : this(emptySet(), ConcurrentHashMap()) override fun rpcClientSerializerFactory(context: SerializationContext): SerializerFactory { throw UnsupportedOperationException() diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializerFactories.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializerFactories.kt index 884c6b057e..677f8a2c18 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializerFactories.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializerFactories.kt @@ -1,5 +1,13 @@ +/* + * R3 Proprietary and Confidential + * + * Copyright (c) 2018 R3 Limited. All rights reserved. + * + * The intellectual and technical concepts contained herein are proprietary to R3 and its suppliers and are protected by trade secret law. + * + * Distribution of this file or any portion thereof via any medium without the express permission of R3 is strictly prohibited. + */ @file:JvmName("AMQPSerializerFactories") - package net.corda.nodeapi.internal.serialization.amqp import net.corda.core.serialization.SerializationContext diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CustomSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CustomSerializer.kt index 68cfaf941c..536c6807f3 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CustomSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CustomSerializer.kt @@ -68,7 +68,7 @@ abstract class CustomSerializer : AMQPSerializer, SerializerFor { * subclass in the schema, so that we can distinguish between subclasses. */ // TODO: should this be a custom serializer at all, or should it just be a plain AMQPSerializer? - class SubClass(protected val clazz: Class<*>, protected val superClassSerializer: CustomSerializer) : CustomSerializer() { + class SubClass(private val clazz: Class<*>, private val superClassSerializer: CustomSerializer) : CustomSerializer() { // TODO: should this be empty or contain the schema of the super? override val schemaForDocumentation = Schema(emptyList()) diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt index f0479a7351..1d15b9ceed 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt @@ -249,7 +249,7 @@ class EvolutionSerializerGetter : EvolutionSerializerGetterBase() { typeNotation: TypeNotation, newSerializer: AMQPSerializer, schemas: SerializationSchemas): AMQPSerializer { - return factory.getSerializersByDescriptor().computeIfAbsent(typeNotation.descriptor.name!!) { + return factory.serializersByDescriptor.computeIfAbsent(typeNotation.descriptor.name!!) { when (typeNotation) { is CompositeType -> EvolutionSerializer.make(typeNotation, newSerializer as ObjectSerializer, factory) is RestrictedType -> { diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt index 3b19ca35e7..b0eb38e94e 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt @@ -50,32 +50,40 @@ open class SerializerFactory( val whitelist: ClassWhitelist, val classCarpenter: ClassCarpenter, private val evolutionSerializerGetter: EvolutionSerializerGetterBase = EvolutionSerializerGetter(), - val fingerPrinter: FingerPrinter = SerializerFingerPrinter()) { + val fingerPrinter: FingerPrinter = SerializerFingerPrinter(), + private val serializersByType: MutableMap>, + val serializersByDescriptor: MutableMap>, + private val customSerializers: MutableList, + val transformsCache: MutableMap>>) { + constructor(whitelist: ClassWhitelist, + classCarpenter: ClassCarpenter, + evolutionSerializerGetter: EvolutionSerializerGetterBase = EvolutionSerializerGetter(), + fingerPrinter: FingerPrinter = SerializerFingerPrinter() + ) : this(whitelist, classCarpenter, evolutionSerializerGetter, fingerPrinter, + serializersByType = ConcurrentHashMap(), + serializersByDescriptor = ConcurrentHashMap(), + customSerializers = CopyOnWriteArrayList(), + transformsCache = ConcurrentHashMap()) constructor(whitelist: ClassWhitelist, classLoader: ClassLoader, evolutionSerializerGetter: EvolutionSerializerGetterBase = EvolutionSerializerGetter(), fingerPrinter: FingerPrinter = SerializerFingerPrinter() - ) : this(whitelist, ClassCarpenterImpl(classLoader, whitelist), evolutionSerializerGetter, fingerPrinter) + ) : this(whitelist, ClassCarpenterImpl(classLoader, whitelist), evolutionSerializerGetter, fingerPrinter, + serializersByType = ConcurrentHashMap(), + serializersByDescriptor = ConcurrentHashMap(), + customSerializers = CopyOnWriteArrayList(), + transformsCache = ConcurrentHashMap()) init { fingerPrinter.setOwner(this) } - private val serializersByType = ConcurrentHashMap>() - private val serializersByDescriptor = ConcurrentHashMap>() - private val customSerializers = CopyOnWriteArrayList() - private val transformsCache = ConcurrentHashMap>>() - val classloader: ClassLoader get() = classCarpenter.classloader private fun getEvolutionSerializer(typeNotation: TypeNotation, newSerializer: AMQPSerializer, schemas: SerializationSchemas) = evolutionSerializerGetter.getEvolutionSerializer(this, typeNotation, newSerializer, schemas) - fun getSerializersByDescriptor() = serializersByDescriptor - - fun getTransformsCache() = transformsCache - /** * Look up, and manufacture if necessary, a serializer for the given type. * @@ -229,7 +237,7 @@ open class SerializerFactory( /** * Iterate over an AMQP schema, for each type ascertain whether it's on ClassPath of [classloader] and, - * if not, use the [ClassCarpenter] to generate a class to use in it's place. + * if not, use the [ClassCarpenter] to generate a class to use in its place. */ private fun processSchema(schemaAndDescriptor: FactorySchemaAndDescriptor, sentinel: Boolean = false) { val metaSchema = CarpenterMetaSchema.newInstance() @@ -249,24 +257,28 @@ open class SerializerFactory( } if (metaSchema.isNotEmpty()) { - val mc = MetaCarpenter(metaSchema, classCarpenter) - try { - mc.build() - } catch (e: MetaCarpenterException) { - // preserve the actual message locally - loggerFor().apply { - error("${e.message} [hint: enable trace debugging for the stack trace]") - trace("", e) - } - - // prevent carpenter exceptions escaping into the world, convert things into a nice - // NotSerializableException for when this escapes over the wire - throw NotSerializableException(e.name) - } - processSchema(schemaAndDescriptor, true) + runCarpentry(schemaAndDescriptor, metaSchema) } } + private fun runCarpentry(schemaAndDescriptor: FactorySchemaAndDescriptor, metaSchema: CarpenterMetaSchema) { + val mc = MetaCarpenter(metaSchema, classCarpenter) + try { + mc.build() + } catch (e: MetaCarpenterException) { + // preserve the actual message locally + loggerFor().apply { + error("${e.message} [hint: enable trace debugging for the stack trace]") + trace("", e) + } + + // prevent carpenter exceptions escaping into the world, convert things into a nice + // NotSerializableException for when this escapes over the wire + throw NotSerializableException(e.name) + } + processSchema(schemaAndDescriptor, true) + } + private fun processSchemaEntry(typeNotation: TypeNotation) = when (typeNotation) { is CompositeType -> processCompositeType(typeNotation) // java.lang.Class (whether a class or interface) is RestrictedType -> processRestrictedType(typeNotation) // Collection / Map, possibly with generics diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/TransformsSchema.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/TransformsSchema.kt index b471e66809..aa54a7136e 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/TransformsSchema.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/TransformsSchema.kt @@ -210,7 +210,7 @@ data class TransformsSchema(val types: Map>(TransformTypes::class.java) try { val clazz = sf.classloader.loadClass(name) diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/carpenter/ClassCarpenter.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/carpenter/ClassCarpenter.kt index ce74d2fe77..80fcd147be 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/carpenter/ClassCarpenter.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/carpenter/ClassCarpenter.kt @@ -58,7 +58,7 @@ private val toStringHelper: String = Type.getInternalName(MoreObjects.ToStringHe // Allow us to create alternative ClassCarpenters. interface ClassCarpenter { val whitelist: ClassWhitelist - val classloader: CarpenterClassLoader + val classloader: ClassLoader fun build(schema: Schema): Class<*> } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/carpenter/SchemaFields.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/carpenter/SchemaFields.kt index b78bcceb01..0602205ddd 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/carpenter/SchemaFields.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/carpenter/SchemaFields.kt @@ -40,8 +40,8 @@ abstract class ClassField(field: Class) : Field(field) { abstract val nullabilityAnnotation: String abstract fun nullTest(mv: MethodVisitor, slot: Int) - override var descriptor = Type.getDescriptor(this.field) - override val type: String get() = if (this.field.isPrimitive) this.descriptor else "Ljava/lang/Object;" + override var descriptor: String? = Type.getDescriptor(this.field) + override val type: String get() = if (this.field.isPrimitive) this.descriptor!! else "Ljava/lang/Object;" fun addNullabilityAnnotation(mv: MethodVisitor) { mv.visitAnnotation(nullabilityAnnotation, true).visitEnd() diff --git a/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaPrivatePropertyTests.java b/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaPrivatePropertyTests.java index fbc86a1f89..0b9ae422e2 100644 --- a/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaPrivatePropertyTests.java +++ b/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaPrivatePropertyTests.java @@ -17,7 +17,7 @@ import static org.junit.Assert.*; import java.io.NotSerializableException; import java.lang.reflect.Field; -import java.util.concurrent.ConcurrentHashMap; +import java.util.Map; public class JavaPrivatePropertyTests { static class C { @@ -126,7 +126,7 @@ public class JavaPrivatePropertyTests { B3 b2 = des.deserialize(ser.serialize(b, TestSerializationContext.testSerializationContext), B3.class, TestSerializationContext.testSerializationContext); // since we can't find a getter for b (isb != isB) then we won't serialize that parameter - assertEquals (null, b2.b); + assertNull (b2.b); } @Test @@ -164,8 +164,7 @@ public class JavaPrivatePropertyTests { Field f = SerializerFactory.class.getDeclaredField("serializersByDescriptor"); f.setAccessible(true); - ConcurrentHashMap> serializersByDescriptor = - (ConcurrentHashMap>) f.get(factory); + Map> serializersByDescriptor = (Map>) f.get(factory); assertEquals(1, serializersByDescriptor.size()); ObjectSerializer cSerializer = ((ObjectSerializer)serializersByDescriptor.values().toArray()[0]); @@ -195,8 +194,7 @@ public class JavaPrivatePropertyTests { // Field f = SerializerFactory.class.getDeclaredField("serializersByDescriptor"); f.setAccessible(true); - ConcurrentHashMap> serializersByDescriptor = - (ConcurrentHashMap>) f.get(factory); + Map> serializersByDescriptor = (Map>) f.get(factory); assertEquals(1, serializersByDescriptor.size()); ObjectSerializer cSerializer = ((ObjectSerializer)serializersByDescriptor.values().toArray()[0]); diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt index a254bde448..974b8d0e1b 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt @@ -20,7 +20,6 @@ import net.corda.core.identity.CordaX500Name import net.corda.nodeapi.internal.serialization.amqp.testutils.* import net.corda.testing.core.TestIdentity import java.util.* -import java.util.concurrent.ConcurrentHashMap import kotlin.test.assertEquals data class TestContractState( @@ -46,7 +45,7 @@ class GenericsTests { private fun BytesAndSchemas.printSchema() = if (VERBOSE) println("${this.schema}\n") else Unit - private fun ConcurrentHashMap>.printKeyToType() { + private fun MutableMap>.printKeyToType() { if (!VERBOSE) return forEach { @@ -63,11 +62,11 @@ class GenericsTests { val bytes1 = SerializationOutput(factory).serializeAndReturnSchema(G("hi")).apply { printSchema() } - factory.getSerializersByDescriptor().printKeyToType() + factory.serializersByDescriptor.printKeyToType() val bytes2 = SerializationOutput(factory).serializeAndReturnSchema(G(121)).apply { printSchema() } - factory.getSerializersByDescriptor().printKeyToType() + factory.serializersByDescriptor.printKeyToType() listOf(factory, testDefaultFactory()).forEach { f -> DeserializationInput(f).deserialize(bytes1.obj).apply { assertEquals("hi", this.a) } @@ -100,14 +99,14 @@ class GenericsTests { val bytes = ser.serializeAndReturnSchema(G("hi")).apply { printSchema() } - factory.getSerializersByDescriptor().printKeyToType() + factory.serializersByDescriptor.printKeyToType() assertEquals("hi", DeserializationInput(factory).deserialize(bytes.obj).a) assertEquals("hi", DeserializationInput(altContextFactory).deserialize(bytes.obj).a) val bytes2 = ser.serializeAndReturnSchema(Wrapper(1, G("hi"))).apply { printSchema() } - factory.getSerializersByDescriptor().printKeyToType() + factory.serializersByDescriptor.printKeyToType() printSeparator() @@ -159,21 +158,21 @@ class GenericsTests { ser.serialize(Wrapper(Container(InnerA(1)))).apply { factories.forEach { DeserializationInput(it).deserialize(this).apply { assertEquals(1, c.b.a_a) } - it.getSerializersByDescriptor().printKeyToType(); printSeparator() + it.serializersByDescriptor.printKeyToType(); printSeparator() } } ser.serialize(Wrapper(Container(InnerB(1)))).apply { factories.forEach { DeserializationInput(it).deserialize(this).apply { assertEquals(1, c.b.a_b) } - it.getSerializersByDescriptor().printKeyToType(); printSeparator() + it.serializersByDescriptor.printKeyToType(); printSeparator() } } ser.serialize(Wrapper(Container(InnerC("Ho ho ho")))).apply { factories.forEach { DeserializationInput(it).deserialize(this).apply { assertEquals("Ho ho ho", c.b.a_c) } - it.getSerializersByDescriptor().printKeyToType(); printSeparator() + it.serializersByDescriptor.printKeyToType(); printSeparator() } } } @@ -209,7 +208,7 @@ class GenericsTests { a: ForceWildcard<*>, factory: SerializerFactory = SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader())): SerializedBytes<*> { val bytes = SerializationOutput(factory).serializeAndReturnSchema(a) - factory.getSerializersByDescriptor().printKeyToType() + factory.serializersByDescriptor.printKeyToType() bytes.printSchema() return bytes.obj } diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationSchemaTests.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationSchemaTests.kt index a9c75359b7..cd49520e42 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationSchemaTests.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationSchemaTests.kt @@ -4,6 +4,7 @@ import net.corda.core.serialization.* import net.corda.core.utilities.ByteSequence import net.corda.nodeapi.internal.serialization.* import org.junit.Test +import java.util.concurrent.ConcurrentHashMap import kotlin.test.assertEquals // Make sure all serialization calls in this test don't get stomped on by anything else @@ -43,7 +44,7 @@ class TestSerializerFactoryFactory : SerializerFactoryFactoryImpl() { } } -class AMQPTestSerializationScheme : AbstractAMQPSerializationScheme(emptySet(), TestSerializerFactoryFactory()) { +class AMQPTestSerializationScheme : AbstractAMQPSerializationScheme(emptySet(), ConcurrentHashMap(), TestSerializerFactoryFactory()) { override fun rpcClientSerializerFactory(context: SerializationContext): SerializerFactory { throw UnsupportedOperationException() } diff --git a/node/build.gradle b/node/build.gradle index 6beecc4333..7800ff24c5 100644 --- a/node/build.gradle +++ b/node/build.gradle @@ -29,13 +29,6 @@ buildscript { //noinspection GroovyAssignabilityCheck configurations { - compile { - // We don't need these because we already include netty-all. - exclude group: 'io.netty', module: 'netty-transport' - exclude group: 'io.netty', module: 'netty-handler' - exclude group: 'io.netty', module: 'netty-common' - } - integrationTestCompile.extendsFrom testCompile integrationTestRuntime.extendsFrom testRuntime } diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index 217b261f5c..ef829fbbb5 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -135,10 +135,7 @@ import net.corda.node.services.transactions.SimpleNotaryService import net.corda.node.services.transactions.ValidatingNotaryService import net.corda.node.services.upgrade.ContractUpgradeServiceImpl import net.corda.node.services.vault.NodeVaultService -import net.corda.node.utilities.AffinityExecutor -import net.corda.node.utilities.JVMAgentRegistry -import net.corda.node.utilities.NamedThreadFactory -import net.corda.node.utilities.NodeBuildProperties +import net.corda.node.utilities.* import net.corda.nodeapi.internal.DevIdentityGenerator import net.corda.nodeapi.internal.NodeInfoAndSigned import net.corda.nodeapi.internal.SignedNodeInfo @@ -282,6 +279,7 @@ abstract class AbstractNode(val configuration: NodeConfiguration, open fun start(): StartedNode { check(started == null) { "Node has already been started" } if (configuration.devMode) { + System.setProperty("co.paralleluniverse.fibers.verifyInstrumentation", "true") Emoji.renderIfSupported { Node.printWarning("This node is running in developer mode! ${Emoji.developer} This is not safe for production deployment.") } } log.info("Node starting up ...") diff --git a/node/src/main/kotlin/net/corda/node/services/identity/IdentityServiceUtil.kt b/node/src/main/kotlin/net/corda/node/services/identity/IdentityServiceUtil.kt new file mode 100644 index 0000000000..b3000262ac --- /dev/null +++ b/node/src/main/kotlin/net/corda/node/services/identity/IdentityServiceUtil.kt @@ -0,0 +1,33 @@ +/* + * R3 Proprietary and Confidential + * + * Copyright (c) 2018 R3 Limited. All rights reserved. + * + * The intellectual and technical concepts contained herein are proprietary to R3 and its suppliers and are protected by trade secret law. + * + * Distribution of this file or any portion thereof via any medium without the express permission of R3 is strictly prohibited. + */ +package net.corda.node.services.identity + +import net.corda.core.identity.CordaX500Name +import net.corda.core.identity.Party + + +fun partiesFromName(query: String, exactMatch: Boolean, x500name: CordaX500Name, results: LinkedHashSet, party: Party) { + + val components = listOfNotNull(x500name.commonName, x500name.organisationUnit, x500name.organisation, x500name.locality, x500name.state, x500name.country) + components.forEach { component -> + if (exactMatch && component == query) { + results += party + } else if (!exactMatch) { + // We can imagine this being a query over a lucene index in future. + // + // Kostas says: We can easily use the Jaro-Winkler distance metric as it is best suited for short + // strings such as entity/company names, and to detect small typos. We can also apply it for city + // or any keyword related search in lists of records (not raw text - for raw text we need indexing) + // and we can return results in hierarchical order (based on normalised String similarity 0.0-1.0). + if (component.contains(query, ignoreCase = true)) + results += party + } + } +} \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/identity/InMemoryIdentityService.kt b/node/src/main/kotlin/net/corda/node/services/identity/InMemoryIdentityService.kt index bea6ddcf43..a9e16f4da2 100644 --- a/node/src/main/kotlin/net/corda/node/services/identity/InMemoryIdentityService.kt +++ b/node/src/main/kotlin/net/corda/node/services/identity/InMemoryIdentityService.kt @@ -113,22 +113,7 @@ class InMemoryIdentityService(identities: Array, override fun partiesFromName(query: String, exactMatch: Boolean): Set { val results = LinkedHashSet() for ((x500name, partyAndCertificate) in principalToParties) { - val party = partyAndCertificate.party - val components = listOfNotNull(x500name.commonName, x500name.organisationUnit, x500name.organisation, x500name.locality, x500name.state, x500name.country) - components.forEach { component -> - if (exactMatch && component == query) { - results += party - } else if (!exactMatch) { - // We can imagine this being a query over a lucene index in future. - // - // Kostas says: We can easily use the Jaro-Winkler distance metric as it is best suited for short - // strings such as entity/company names, and to detect small typos. We can also apply it for city - // or any keyword related search in lists of records (not raw text - for raw text we need indexing) - // and we can return results in hierarchical order (based on normalised String similarity 0.0-1.0). - if (component.contains(query, ignoreCase = true)) - results += party - } - } + partiesFromName(query, exactMatch, x500name, results, partyAndCertificate.party) } return results } diff --git a/node/src/main/kotlin/net/corda/node/services/identity/PersistentIdentityService.kt b/node/src/main/kotlin/net/corda/node/services/identity/PersistentIdentityService.kt index 4010de4a7a..bf7abd3608 100644 --- a/node/src/main/kotlin/net/corda/node/services/identity/PersistentIdentityService.kt +++ b/node/src/main/kotlin/net/corda/node/services/identity/PersistentIdentityService.kt @@ -190,22 +190,7 @@ class PersistentIdentityService(override val trustRoot: X509Certificate, override fun partiesFromName(query: String, exactMatch: Boolean): Set { val results = LinkedHashSet() for ((x500name, partyId) in principalToParties.allPersisted()) { - val party = keyToParties[partyId]!!.party - val components = listOfNotNull(x500name.commonName, x500name.organisationUnit, x500name.organisation, x500name.locality, x500name.state, x500name.country) - components.forEach { component -> - if (exactMatch && component == query) { - results += party - } else if (!exactMatch) { - // We can imagine this being a query over a lucene index in future. - // - // Kostas says: We can easily use the Jaro-Winkler distance metric as it is best suited for short - // strings such as entity/company names, and to detect small typos. We can also apply it for city - // or any keyword related search in lists of records (not raw text - for raw text we need indexing) - // and we can return results in hierarchical order (based on normalised String similarity 0.0-1.0). - if (component.contains(query, ignoreCase = true)) - results += party - } - } + partiesFromName(query, exactMatch, x500name, results, keyToParties[partyId]!!.party) } return results } diff --git a/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt b/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt index 0e3a9989ec..6b5056acd9 100644 --- a/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt +++ b/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt @@ -407,13 +407,18 @@ class NodeVaultService( @Throws(VaultQueryException::class) override fun _queryBy(criteria: QueryCriteria, paging: PageSpecification, sorting: Sort, contractStateType: Class): Vault.Page { + return _queryBy(criteria, paging, sorting, contractStateType, false) + } + + @Throws(VaultQueryException::class) + private fun _queryBy(criteria: QueryCriteria, paging: PageSpecification, sorting: Sort, contractStateType: Class, skipPagingChecks: Boolean): Vault.Page { log.debug {"Vault Query for contract type: $contractStateType, criteria: $criteria, pagination: $paging, sorting: $sorting" } // calculate total results where a page specification has been defined var totalStates = -1L - if (!paging.isDefault) { + if (!skipPagingChecks && !paging.isDefault) { val count = builder { VaultSchemaV1.VaultStates::recordedTime.count() } val countCriteria = QueryCriteria.VaultCustomQueryCriteria(count, Vault.StateStatus.ALL) - val results = queryBy(contractStateType, criteria.and(countCriteria)) + val results = _queryBy(criteria.and(countCriteria), PageSpecification(), Sort(emptyList()), contractStateType, true) // only skip pagination checks for total results count query totalStates = results.otherResults.last() as Long } @@ -432,7 +437,7 @@ class NodeVaultService( val query = session.createQuery(criteriaQuery) // pagination checks - if (!paging.isDefault) { + if (!skipPagingChecks && !paging.isDefault) { // pagination if (paging.pageNumber < DEFAULT_PAGE_NUM) throw VaultQueryException("Page specification: invalid page number ${paging.pageNumber} [page numbers start from $DEFAULT_PAGE_NUM]") if (paging.pageSize < 1) throw VaultQueryException("Page specification: invalid page size ${paging.pageSize} [must be a value between 1 and $MAX_PAGE_SIZE]") @@ -445,7 +450,7 @@ class NodeVaultService( val results = query.resultList // final pagination check (fail-fast on too many results when no pagination specified) - if (paging.isDefault && results.size > DEFAULT_PAGE_SIZE) + if (!skipPagingChecks && paging.isDefault && results.size > DEFAULT_PAGE_SIZE) throw VaultQueryException("Please specify a `PageSpecification` as there are more results [${results.size}] than the default page size [$DEFAULT_PAGE_SIZE]") val statesAndRefs: MutableList> = mutableListOf() diff --git a/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt b/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt index 582c2d86bf..d053cb457f 100644 --- a/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt @@ -38,10 +38,7 @@ import net.corda.nodeapi.internal.persistence.DatabaseTransaction import net.corda.testing.core.* import net.corda.testing.internal.TEST_TX_TIME import net.corda.testing.internal.rigorousMock -import net.corda.testing.internal.vault.DUMMY_LINEAR_CONTRACT_PROGRAM_ID -import net.corda.testing.internal.vault.DummyLinearContract -import net.corda.testing.internal.vault.DummyLinearStateSchemaV1 -import net.corda.testing.internal.vault.VaultFiller +import net.corda.testing.internal.vault.* import net.corda.testing.node.MockServices import net.corda.testing.node.MockServices.Companion.makeTestDatabaseAndMockServices import net.corda.testing.node.makeTestIdentityService @@ -1190,6 +1187,30 @@ abstract class VaultQueryTestsBase : VaultQueryParties { } } + // test paging with aggregate function and group by clause + @Test + fun `test paging with aggregate function and group by clause`() { + database.transaction { + (0..200).forEach { + vaultFiller.fillWithSomeTestLinearStates(1, linearNumber = it.toLong(), linearString = it.toString()) + } + val max = builder { DummyLinearStateSchemaV1.PersistentDummyLinearState::linearTimestamp.max( + groupByColumns = listOf(DummyLinearStateSchemaV1.PersistentDummyLinearState::linearNumber) + ) + } + val maxCriteria = VaultCustomQueryCriteria(max) + val pageSpec = PageSpecification(DEFAULT_PAGE_NUM, MAX_PAGE_SIZE) + + val results = vaultService.queryBy(maxCriteria, paging = pageSpec) + println("Total states available: ${results.totalStatesAvailable}") + results.otherResults.forEachIndexed { index, any -> + println("$index : $any") + } + assertThat(results.otherResults.size).isEqualTo(402) + assertThat(results.otherResults.last()).isEqualTo(200L) + } + } + // sorting @Test fun `sorting - all states sorted by contract type, state status, consumed time`() { diff --git a/verify-enclave/src/main/kotlin/com/r3/enclaves/txverify/EnclaveletSerializationScheme.kt b/verify-enclave/src/main/kotlin/com/r3/enclaves/txverify/EnclaveletSerializationScheme.kt index d047a1215c..391bd28c1e 100644 --- a/verify-enclave/src/main/kotlin/com/r3/enclaves/txverify/EnclaveletSerializationScheme.kt +++ b/verify-enclave/src/main/kotlin/com/r3/enclaves/txverify/EnclaveletSerializationScheme.kt @@ -32,7 +32,7 @@ private class EnclaveletSerializationScheme { private companion object { init { nodeSerializationEnv = SerializationEnvironmentImpl( - SerializationFactoryImpl().apply { + SerializationFactoryImpl(HashMap()).apply { registerScheme(KryoVerifierSerializationScheme) registerScheme(AMQPVerifierSerializationScheme) }, @@ -59,7 +59,7 @@ private object KryoVerifierSerializationScheme : AbstractKryoSerializationScheme override fun rpcServerKryoPool(context: SerializationContext) = throw UnsupportedOperationException() } -private object AMQPVerifierSerializationScheme : AbstractAMQPSerializationScheme(emptyList()) { +private object AMQPVerifierSerializationScheme : AbstractAMQPSerializationScheme(emptySet(), HashMap()) { override fun canDeserializeVersion(magic: CordaSerializationMagic, target: SerializationContext.UseCase): Boolean { return magic == amqpMagic && target == SerializationContext.UseCase.P2P }