From ea22a10b3e96dc73cffac942c8adc3483d6cdf73 Mon Sep 17 00:00:00 2001 From: Christian Sailer Date: Fri, 13 Mar 2020 14:26:24 +0000 Subject: [PATCH] ENT-4494 Harmonize network stack (#6059) * ENT-4494 harmonize proton wrapper with ENT * Harmonise Artemis and Bridge implementation * Move tests across * detekt changes * Fix AMQP tests in node --- detekt-baseline.xml | 41 +- node-api/build.gradle | 2 + .../internal/ArtemisMessagingClient.kt | 47 ++- .../internal/ArtemisMessagingComponent.kt | 6 + .../nodeapi/internal/ArtemisTcpTransport.kt | 31 +- .../corda/nodeapi/internal/ArtemisUtils.kt | 2 +- .../nodeapi/internal/ClientSessionUtils.kt | 8 + .../corda/nodeapi/internal/ConcurrentBox.kt | 17 + .../internal/RoundRobinConnectionPolicy.kt | 18 + .../internal/bridging/AMQPBridgeManager.kt | 395 ++++++++++++++---- .../bridging/BridgeControlListener.kt | 205 +++++++-- .../bridging/BridgeControlMessages.kt | 11 +- .../internal/bridging/BridgeManager.kt | 7 +- .../bridging/LoopbackBridgeManager.kt | 223 ++++++++++ .../MessagingServerConnectionConfiguration.kt | 63 +++ .../engine/ConnectionStateMachine.kt | 11 +- .../protonwrapper/engine/EventProcessor.kt | 4 +- .../messages/ApplicationMessage.kt | 1 + .../messages/impl/ReceivedMessageImpl.kt | 19 +- .../messages/impl/SendableMessageImpl.kt | 10 +- .../protonwrapper/netty/AMQPChannelHandler.kt | 218 +++++++--- .../protonwrapper/netty/AMQPClient.kt | 183 ++++++-- .../protonwrapper/netty/AMQPConfiguration.kt | 45 +- .../protonwrapper/netty/AMQPServer.kt | 75 +++- .../netty/AliasProvidingKeyMangerWrapper.kt | 60 +++ .../netty/AllowAllRevocationChecker.kt | 34 ++ .../CertHoldingKeyManagerFactoryWrapper.kt | 81 ++++ .../protonwrapper/netty/ExternalCrlSource.kt | 12 + .../netty/ModeSelectingChannel.kt | 76 ++++ .../netty/NettyServerEventLogger.kt | 73 ++++ .../protonwrapper/netty/RevocationConfig.kt | 83 ++++ .../protonwrapper/netty/SNIKeyManager.kt | 112 +++++ .../internal/protonwrapper/netty/SSLHelper.kt | 230 ++++++++-- .../protonwrapper/netty/ServerSNIMatcher.kt | 47 +++ .../netty/TrustManagerFactoryWrapper.kt | 40 ++ .../ExternalSourceRevocationChecker.kt | 88 ++++ .../internal/config/ConfigParsingTest.kt | 1 - .../protonwrapper/netty/SSLHelperTest.kt | 15 +- .../ExternalSourceRevocationCheckerTest.kt | 56 +++ .../internal/protonwrapper/netty/Readme.txt | 3 + .../internal/protonwrapper/netty/doorman.crl | Bin 0 -> 576 bytes .../netty/sslkeystore_Revoked.jks | Bin 0 -> 3594 bytes .../serialization/amqp/networkParamsWrite | Bin 0 -> 3066 bytes .../net/corda/node/amqp/AMQPBridgeTest.kt | 16 +- .../CertificateRevocationListNodeTests.kt | 8 +- .../net/corda/node/amqp/ProtonWrapperTests.kt | 11 +- .../kotlin/net/corda/node/internal/Node.kt | 11 +- .../services/messaging/P2PMessagingClient.kt | 13 +- 48 files changed, 2372 insertions(+), 340 deletions(-) create mode 100644 node-api/src/main/kotlin/net/corda/nodeapi/internal/ClientSessionUtils.kt create mode 100644 node-api/src/main/kotlin/net/corda/nodeapi/internal/ConcurrentBox.kt create mode 100644 node-api/src/main/kotlin/net/corda/nodeapi/internal/RoundRobinConnectionPolicy.kt create mode 100644 node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/LoopbackBridgeManager.kt create mode 100644 node-api/src/main/kotlin/net/corda/nodeapi/internal/config/MessagingServerConnectionConfiguration.kt create mode 100644 node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AliasProvidingKeyMangerWrapper.kt create mode 100644 node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AllowAllRevocationChecker.kt create mode 100644 node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/CertHoldingKeyManagerFactoryWrapper.kt create mode 100644 node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/ExternalCrlSource.kt create mode 100644 node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/ModeSelectingChannel.kt create mode 100644 node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/NettyServerEventLogger.kt create mode 100644 node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/RevocationConfig.kt create mode 100644 node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SNIKeyManager.kt create mode 100644 node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/ServerSNIMatcher.kt create mode 100644 node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/TrustManagerFactoryWrapper.kt create mode 100644 node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/revocation/ExternalSourceRevocationChecker.kt create mode 100644 node-api/src/test/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/revocation/ExternalSourceRevocationCheckerTest.kt create mode 100644 node-api/src/test/resources/net/corda/nodeapi/internal/protonwrapper/netty/Readme.txt create mode 100644 node-api/src/test/resources/net/corda/nodeapi/internal/protonwrapper/netty/doorman.crl create mode 100644 node-api/src/test/resources/net/corda/nodeapi/internal/protonwrapper/netty/sslkeystore_Revoked.jks create mode 100644 node-api/src/test/resources/net/corda/nodeapi/internal/serialization/amqp/networkParamsWrite diff --git a/detekt-baseline.xml b/detekt-baseline.xml index 3e147108a6..20c1cb21e4 100644 --- a/detekt-baseline.xml +++ b/detekt-baseline.xml @@ -91,7 +91,6 @@ ComplexCondition:WireTransaction.kt$WireTransaction$notary != null && (inputs.isNotEmpty() || references.isNotEmpty() || timeWindow != null) ComplexMethod:AMQPBridgeManager.kt$AMQPBridgeManager.AMQPBridge$private fun clientArtemisMessageHandler(artemisMessage: ClientMessage) ComplexMethod:AMQPBridgeTest.kt$AMQPBridgeTest$@Test(timeout=300_000) fun `test acked and nacked messages`() - ComplexMethod:AMQPChannelHandler.kt$AMQPChannelHandler$override fun userEventTriggered(ctx: ChannelHandlerContext, evt: Any) ComplexMethod:AMQPTypeIdentifierParser.kt$AMQPTypeIdentifierParser$// Make sure our inputs aren't designed to blow things up. private fun validate(typeString: String) ComplexMethod:ANSIProgressRenderer.kt$ANSIProgressRenderer$// Returns number of lines rendered. private fun renderLevel(ansi: Ansi, error: Boolean): Int ComplexMethod:ANSIProgressRenderer.kt$ANSIProgressRenderer$@Synchronized protected fun draw(moveUp: Boolean, error: Throwable? = null) @@ -124,13 +123,10 @@ ComplexMethod:ConfigUtilities.kt$// For Iterables figure out the type parameter and apply the same logic as above on the individual elements. private fun Iterable<*>.toConfigIterable(field: Field): Iterable<Any?> ComplexMethod:ConfigUtilities.kt$// TODO Move this to KeyStoreConfigHelpers. fun MutualSslConfiguration.configureDevKeyAndTrustStores(myLegalName: CordaX500Name, signingCertificateStore: FileBasedCertificateStoreSupplier, certificatesDirectory: Path, cryptoService: CryptoService? = null) ComplexMethod:ConfigUtilities.kt$@Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN") // Reflect over the fields of the receiver and generate a value Map that can use to create Config object. private fun Any.toConfigMap(): Map<String, Any> - ComplexMethod:ConfigUtilities.kt$private fun Config.getCollectionValue(path: String, type: KType, onUnknownKeys: (Set<String>, logger: Logger) -> Unit, nestedPath: String?, baseDirectory: Path?): Collection<Any> - ComplexMethod:ConfigUtilities.kt$private fun Config.getSingleValue(path: String, type: KType, onUnknownKeys: (Set<String>, logger: Logger) -> Unit, nestedPath: String?, baseDirectory: Path?): Any? ComplexMethod:ConfigUtilities.kt$private fun convertValue(value: Any): Any ComplexMethod:ConnectionStateMachine.kt$ConnectionStateMachine$override fun onConnectionFinal(event: Event) ComplexMethod:ConnectionStateMachine.kt$ConnectionStateMachine$override fun onDelivery(event: Event) ComplexMethod:ConstraintsUtils.kt$ fun AttachmentConstraint.canBeTransitionedFrom(input: AttachmentConstraint, attachment: ContractAttachment): Boolean - ComplexMethod:CordaCliWrapper.kt$fun CordaCliWrapper.start(args: Array<String>) ComplexMethod:CordaPersistence.kt$CordaPersistence$private fun <T> inTopLevelTransaction(isolationLevel: TransactionIsolationLevel, recoverableFailureTolerance: Int, recoverAnyNestedSQLException: Boolean, statement: DatabaseTransaction.() -> T): T ComplexMethod:CordaRPCClient.kt$CordaRPCClientConfiguration$override fun equals(other: Any?): Boolean ComplexMethod:CordaRPCClientTest.kt$CordaRPCClientTest$@Test(timeout=300_000) fun `shutdown command stops the node`() @@ -182,7 +178,6 @@ ComplexMethod:RPCClientProxyHandler.kt$RPCClientProxyHandler$// This is the general function that transforms a client side RPC to internal Artemis messages. override fun invoke(proxy: Any, method: Method, arguments: Array<out Any?>?): Any? ComplexMethod:RPCClientProxyHandler.kt$RPCClientProxyHandler$private fun attemptReconnect() ComplexMethod:RPCServer.kt$RPCServer$private fun clientArtemisMessageHandler(artemisMessage: ClientMessage) - ComplexMethod:ReconnectingCordaRPCOps.kt$ReconnectingCordaRPCOps.ErrorInterceptingHandler$ private fun doInvoke(method: Method, args: Array<out Any>?, maxNumberOfAttempts: Int): Any? ComplexMethod:ReconnectingCordaRPCOps.kt$ReconnectingCordaRPCOps.ReconnectingRPCConnection$ private tailrec fun establishConnectionWithRetry( retryInterval: Duration, roundRobinIndex: Int = 0, retries: Int = -1 ): CordaRPCConnection? ComplexMethod:RemoteTypeCarpenter.kt$SchemaBuildingRemoteTypeCarpenter$override fun carpent(typeInformation: RemoteTypeInformation): Type ComplexMethod:RpcReconnectTests.kt$RpcReconnectTests$ @Test(timeout=300_000) fun `test that the RPC client is able to reconnect and proceed after node failure, restart, or connection reset`() @@ -215,7 +210,6 @@ EmptyDefaultConstructor:FlowRetryTest.kt$AsyncRetryFlow$() EmptyDefaultConstructor:FlowRetryTest.kt$RetryFlow$() EmptyDefaultConstructor:FlowRetryTest.kt$ThrowingFlow$() - EmptyElseBlock:CordaCliWrapper.kt${ } EmptyIfBlock:ContentSignerBuilder.kt$ContentSignerBuilder.SignatureOutputStream$if (alreadySigned) throw IllegalStateException("Cannot write to already signed object") EmptyIfBlock:InMemoryIdentityService.kt$InMemoryIdentityService${ } EmptyKtFile:KryoHook.kt$.KryoHook.kt @@ -331,7 +325,6 @@ ForbiddenComment:DigitalSignatureWithCert.kt$// TODO: Rename this to DigitalSignature.WithCert once we're happy for it to be public API. The methods will need documentation ForbiddenComment:DriverDSLImpl.kt$DriverDSLImpl$// TODO: Derive name from the full picked name, don't just wrap the common name ForbiddenComment:DriverDSLImpl.kt$DriverDSLImpl$//TODO: remove this once we can bundle quasar properly. - ForbiddenComment:DriverDSLImpl.kt$DriverDSLImpl.Companion$// TODO: There is pending work to fix this issue without custom blacklisting. See: https://r3-cev.atlassian.net/browse/CORDA-2164. ForbiddenComment:DriverDSLImpl.kt$DriverDSLImpl.LocalNetworkMap$// TODO: this object will copy NodeInfo files from started nodes to other nodes additional-node-infos/ ForbiddenComment:DummyFungibleContract.kt$DummyFungibleContract$// TODO: This doesn't work with the trader demo, so use the underlying key instead ForbiddenComment:E2ETestKeyManagementService.kt$E2ETestKeyManagementService$// TODO: A full KeyManagementService implementation needs to record activity to the Audit Service and to limit @@ -388,6 +381,7 @@ ForbiddenComment:LegalNameValidator.kt$LegalNameValidator.Rule.Companion$// TODO: Implement confusable character detection if we add more scripts. ForbiddenComment:LocalTypeInformationBuilder.kt$// TODO: Revisit this when Kotlin issue is fixed. ForbiddenComment:LoggingBuyerFlow.kt$LoggingBuyerFlow$// TODO: This is potentially very expensive, and requires transaction details we may no longer have once + ForbiddenComment:LoopbackBridgeManager.kt$LoopbackBridgeManager.LoopbackBridge$// TODO: refactor MDC support, duplicated in AMQPBridgeManager. ForbiddenComment:MockServices.kt$MockServices.Companion$// TODO: Can we use an X509 principal generator here? ForbiddenComment:NetParams.kt$NetParamsSigner$// TODO: not supported ForbiddenComment:NetworkBootstrapper.kt$NetworkBootstrapper$// TODO: pass a commandline parameter to the bootstrapper instead. Better yet, a notary config map @@ -630,8 +624,6 @@ FunctionNaming:VersionExtractorTest.kt$VersionExtractorTest$@Test(timeout=300_000) fun version_header_extraction_no_metadata() FunctionNaming:VersionExtractorTest.kt$VersionExtractorTest$@Test(timeout=300_000) fun version_header_extraction_no_value() FunctionNaming:VersionExtractorTest.kt$VersionExtractorTest$@Test(timeout=300_000) fun version_header_extraction_present() - FunctionNaming:VersionedParsingExampleTest.kt$VersionedParsingExampleTest$@Test(timeout=300_000) fun correct_parsing_function_is_used_for_present_version() - FunctionNaming:VersionedParsingExampleTest.kt$VersionedParsingExampleTest$@Test(timeout=300_000) fun default_value_is_used_for_absent_version() LargeClass:AbstractNode.kt$AbstractNode<S> : SingletonSerializeAsToken LargeClass:SingleThreadedStateMachineManager.kt$SingleThreadedStateMachineManager : StateMachineManagerStateMachineManagerInternal LongMethod:FlowCookbook.kt$InitiatorFlow$@Suppress("RemoveExplicitTypeArguments") @Suspendable override fun call() @@ -655,7 +647,6 @@ LongParameterList:CertificateRevocationListNodeTests.kt$CertificateRevocationListNodeTests$(port: Int, name: CordaX500Name = ALICE_NAME, crlCheckSoftFail: Boolean, nodeCrlDistPoint: String = "http://${server.hostAndPort}/crl/node.crl", tlsCrlDistPoint: String? = "http://${server.hostAndPort}/crl/empty.crl", maxMessageSize: Int = MAX_MESSAGE_SIZE) LongParameterList:CertificateRevocationListNodeTests.kt$CertificateRevocationListNodeTests.Companion$(clrServer: CrlServer, signatureAlgorithm: String, caCertificate: X509Certificate, caPrivateKey: PrivateKey, endpoint: String, indirect: Boolean, vararg serialNumbers: BigInteger) LongParameterList:CertificateStoreStubs.kt$CertificateStoreStubs.P2P.Companion$(baseDirectory: Path, certificatesDirectoryName: String = DEFAULT_CERTIFICATES_DIRECTORY_NAME, keyStoreFileName: String = KeyStore.DEFAULT_STORE_FILE_NAME, keyStorePassword: String = KeyStore.DEFAULT_STORE_PASSWORD, keyPassword: String = keyStorePassword, trustStoreFileName: String = TrustStore.DEFAULT_STORE_FILE_NAME, trustStorePassword: String = TrustStore.DEFAULT_STORE_PASSWORD) - LongParameterList:CertificateStoreStubs.kt$CertificateStoreStubs.P2P.Companion$(certificatesDirectory: Path, keyStoreFileName: String = KeyStore.DEFAULT_STORE_FILE_NAME, keyStorePassword: String = KeyStore.DEFAULT_STORE_PASSWORD, keyPassword: String = keyStorePassword, trustStoreFileName: String = TrustStore.DEFAULT_STORE_FILE_NAME, trustStorePassword: String = TrustStore.DEFAULT_STORE_PASSWORD, trustStoreKeyPassword: String = TrustStore.DEFAULT_KEY_PASSWORD, @Suppress("UNUSED_PARAMETER") useOpenSsl: Boolean = false) LongParameterList:ContractAttachment.kt$ContractAttachment.Companion$(attachment: Attachment, contract: ContractClassName, additionalContracts: Set<ContractClassName> = emptySet(), uploader: String? = null, signerKeys: List<PublicKey> = emptyList(), version: Int = DEFAULT_CORDAPP_VERSION) LongParameterList:ContractFunctions.kt$(expiry: String, notional: BigDecimal, strike: BigDecimal, foreignCurrency: Currency, domesticCurrency: Currency, partyA: Party, partyB: Party) LongParameterList:ContractFunctions.kt$(expiry: String, notional: Long, strike: Double, foreignCurrency: Currency, domesticCurrency: Currency, partyA: Party, partyB: Party) @@ -925,7 +916,6 @@ MagicNumber:IrsDemoWebApplication.kt$IrsDemoWebApplication$1000 MagicNumber:JarScanningCordappLoader.kt$CordappLoaderTemplate$36 MagicNumber:JarScanningCordappLoader.kt$CordappLoaderTemplate$64 - MagicNumber:JarScanningCordappLoader.kt$JarScanningCordappLoader$1000 MagicNumber:JarSignatureCollector.kt$JarSignatureCollector$1024 MagicNumber:JarSignatureTestUtils.kt$JarSignatureTestUtils$14 MagicNumber:KMSUtils.kt$3650 @@ -1229,8 +1219,6 @@ MatchingDeclarationName:Query.kt$net.corda.webserver.api.Query.kt MatchingDeclarationName:ReceiveAllFlowTests.kt$net.corda.coretests.flows.ReceiveAllFlowTests.kt MatchingDeclarationName:ReferenceInputStateTests.kt$net.corda.coretests.transactions.ReferenceInputStateTests.kt - MatchingDeclarationName:RigorousMock.kt$net.corda.testing.internal.RigorousMock.kt - MatchingDeclarationName:RpcServerCordaFutureSerialiser.kt$net.corda.node.serialization.amqp.RpcServerCordaFutureSerialiser.kt MatchingDeclarationName:SSLHelper.kt$net.corda.nodeapi.internal.protonwrapper.netty.SSLHelper.kt MatchingDeclarationName:SampleData.kt$net.corda.deterministic.verifier.SampleData.kt MatchingDeclarationName:SerializationHelper.kt$net.corda.networkbuilder.serialization.SerializationHelper.kt @@ -1285,7 +1273,6 @@ NestedBlockDepth:RPCClientProxyHandler.kt$RPCClientProxyHandler$// The handler for Artemis messages. private fun artemisMessageHandler(message: ClientMessage) NestedBlockDepth:ShutdownManager.kt$ShutdownManager$fun shutdown() NestedBlockDepth:SpringDriver.kt$SpringBootDriverDSL$private fun queryWebserver(handle: NodeHandle, process: Process, checkUrl: String): WebserverHandle - NestedBlockDepth:StartedFlowTransition.kt$StartedFlowTransition$private fun TransitionBuilder.sendToSessionsTransition(sourceSessionIdToMessage: Map<SessionId, SerializedBytes<Any>>) NestedBlockDepth:StatusTransitions.kt$StatusTransitions$ fun verify(tx: LedgerTransaction) NestedBlockDepth:ThrowableSerializer.kt$ThrowableSerializer$override fun fromProxy(proxy: ThrowableProxy): Throwable NestedBlockDepth:TransactionVerifierServiceInternal.kt$Verifier$ private fun verifyConstraintsValidity(contractAttachmentsByContract: Map<ContractClassName, ContractAttachment>) @@ -1316,7 +1303,6 @@ SpreadOperator:ConfigUtilities.kt$(*pairs) SpreadOperator:Configuration.kt$Configuration.Validation.Error$(*(containingPath.toList() + this.containingPath).toTypedArray()) SpreadOperator:ContractJarTestUtils.kt$ContractJarTestUtils$(jarName, *contractNames.map{ "${it.replace(".", "/")}.class" }.toTypedArray()) - SpreadOperator:CordaCliWrapper.kt$(RunLast().useOut(System.out).useAnsi(defaultAnsiMode), DefaultExceptionHandler<List<Any>>().useErr(System.err).useAnsi(defaultAnsiMode).andExit(ExitCodes.FAILURE), *args) SpreadOperator:CordaRPCOpsImpl.kt$CordaRPCOpsImpl$(logicType, context(), *args) SpreadOperator:CordaX500Name.kt$CordaX500Name.Companion$(*Locale.getISOCountries(), unspecifiedCountry) SpreadOperator:CustomCordapp.kt$CustomCordapp$(*classes.map { it.name }.toTypedArray()) @@ -1326,7 +1312,6 @@ SpreadOperator:DockerInstantiator.kt$DockerInstantiator$(*it.toTypedArray()) SpreadOperator:DummyContract.kt$DummyContract.Companion$( /* INPUTS */ *priors.toTypedArray(), /* COMMAND */ Command(cmd, priorState.owner.owningKey), /* OUTPUT */ StateAndContract(state, PROGRAM_ID) ) SpreadOperator:DummyContract.kt$DummyContract.Companion$(*items) - SpreadOperator:DummyContractV2.kt$DummyContractV2.Companion$( /* INPUTS */ *priors.toTypedArray(), /* COMMAND */ Command(cmd, priorState.owners.map { it.owningKey }), /* OUTPUT */ StateAndContract(state, DummyContractV2.PROGRAM_ID) ) SpreadOperator:ExceptionsErrorCodeFunctions.kt$(*fields) SpreadOperator:ExceptionsErrorCodeFunctions.kt$(*fields, cause.staticLocationBasedHash(hashedFields, visited + cause)) SpreadOperator:ExceptionsErrorCodeFunctions.kt$(*hashedFields.invoke(this)) @@ -1491,7 +1476,6 @@ TooGenericExceptionCaught:BankOfCordaWebApi.kt$BankOfCordaWebApi$e: Exception TooGenericExceptionCaught:BlobInspector.kt$BlobInspector$e: Exception TooGenericExceptionCaught:BootstrapperView.kt$BootstrapperView$e: Exception - TooGenericExceptionCaught:BridgeControlListener.kt$BridgeControlListener$ex: Exception TooGenericExceptionCaught:BrokerJaasLoginModule.kt$BrokerJaasLoginModule$e: Exception TooGenericExceptionCaught:CertRole.kt$CertRole.Companion$ex: ArrayIndexOutOfBoundsException TooGenericExceptionCaught:CheckpointAgent.kt$CheckpointAgent.Companion$e: Exception @@ -1506,9 +1490,6 @@ TooGenericExceptionCaught:ContractUpgradeTransactions.kt$ContractUpgradeWireTransaction$e: Exception TooGenericExceptionCaught:CordaAuthenticationPlugin.kt$CordaAuthenticationPlugin$e: Exception TooGenericExceptionCaught:CordaClassResolver.kt$LoggingWhitelist.Companion$ioEx: Exception - TooGenericExceptionCaught:CordaFutureImpl.kt$CordaFutureImpl$e: Exception - TooGenericExceptionCaught:CordaFutureImpl.kt$ValueOrException$e: Exception - TooGenericExceptionCaught:CordaFutureImpl.kt$e: Exception TooGenericExceptionCaught:CordaPersistence.kt$CordaPersistence$e: Exception TooGenericExceptionCaught:CordaRPCClientTest.kt$CordaRPCClientTest$e: Exception TooGenericExceptionCaught:CordaRPCOpsImpl.kt$CordaRPCOpsImpl$e: Exception @@ -1644,7 +1625,6 @@ TooGenericExceptionCaught:ValidatingNotaryFlow.kt$ValidatingNotaryFlow$e: Exception TooGenericExceptionCaught:VaultStateMigration.kt$VaultStateIterator$e: Exception TooGenericExceptionCaught:VaultStateMigration.kt$VaultStateMigration$e: Exception - TooGenericExceptionCaught:VersionedParsingExampleTest.kt$VersionedParsingExampleTest.RpcSettingsSpec$e: Exception TooGenericExceptionCaught:WebServer.kt$WebServer$e: Exception TooGenericExceptionCaught:WebServer.kt$e: Exception TooGenericExceptionCaught:WebServer.kt$ex: Exception @@ -1699,7 +1679,6 @@ TooManyFunctions:CryptoUtils.kt$net.corda.core.crypto.CryptoUtils.kt TooManyFunctions:Currencies.kt$net.corda.finance.Currencies.kt TooManyFunctions:Driver.kt$DriverParameters - TooManyFunctions:DriverDSLImpl.kt$DriverDSLImpl : InternalDriverDSL TooManyFunctions:EncodingUtils.kt$net.corda.core.utilities.EncodingUtils.kt TooManyFunctions:FlowLogic.kt$FlowLogic<out T> TooManyFunctions:FlowStateMachineImpl.kt$FlowStateMachineImpl<R> : FiberFlowStateMachineFlowFiber @@ -1885,8 +1864,6 @@ VariableNaming:VaultQueryTests.kt$VaultQueryTestsBase$// Beware: do not use `MyContractClass::class.qualifiedName` as this returns a fully qualified name using "dot" notation for enclosed class val MYCONTRACT_ID = "net.corda.node.services.vault.VaultQueryTestsBase\$MyContractClass" VariableNaming:ZeroCouponBond.kt$ZeroCouponBond$val TEST_TX_TIME_1: Instant get() = Instant.parse("2017-09-02T12:00:00.00Z") WildcardImport:AMQPClient.kt$import io.netty.channel.* - WildcardImport:AMQPClientSerializationScheme.kt$import net.corda.serialization.internal.* - WildcardImport:AMQPClientSerializationScheme.kt$import net.corda.serialization.internal.amqp.* WildcardImport:AMQPRemoteTypeModel.kt$import net.corda.serialization.internal.model.* WildcardImport:AMQPSerializationScheme.kt$import net.corda.core.serialization.* WildcardImport:AMQPServerSerializationScheme.kt$import net.corda.serialization.internal.amqp.* @@ -2023,9 +2000,6 @@ WildcardImport:CordaModule.kt$import net.corda.core.crypto.* WildcardImport:CordaModule.kt$import net.corda.core.identity.* WildcardImport:CordaModule.kt$import net.corda.core.transactions.* - WildcardImport:CordaRPCClientTest.kt$import net.corda.core.context.* - WildcardImport:CordaRPCClientTest.kt$import net.corda.core.messaging.* - WildcardImport:CordaRPCClientTest.kt$import net.corda.testing.core.* WildcardImport:CordaRPCOps.kt$import net.corda.core.node.services.vault.* WildcardImport:CordaRPCOpsImplTest.kt$import net.corda.core.messaging.* WildcardImport:CordaRPCOpsImplTest.kt$import org.assertj.core.api.Assertions.* @@ -2121,8 +2095,6 @@ WildcardImport:FlowStateMachineImpl.kt$import net.corda.core.flows.* WildcardImport:FlowStateMachineImpl.kt$import net.corda.core.internal.* WildcardImport:FlowsDrainingModeContentionTest.kt$import net.corda.core.flows.* - WildcardImport:FxTransactionBuildTutorial.kt$import net.corda.core.contracts.* - WildcardImport:FxTransactionBuildTutorial.kt$import net.corda.core.flows.* WildcardImport:FxTransactionBuildTutorialTest.kt$import net.corda.finance.* WildcardImport:GenericsTests.kt$import net.corda.serialization.internal.amqp.testutils.* WildcardImport:Gui.kt$import tornadofx.* @@ -2168,10 +2140,7 @@ WildcardImport:InternalMockNetwork.kt$import net.corda.core.internal.* WildcardImport:InternalMockNetwork.kt$import net.corda.node.services.config.* WildcardImport:InternalMockNetwork.kt$import net.corda.testing.node.* - WildcardImport:InternalSerializationTestHelpers.kt$import net.corda.serialization.internal.* WildcardImport:InternalTestUtils.kt$import net.corda.core.contracts.* - WildcardImport:InternalUtils.kt$import java.security.cert.* - WildcardImport:InternalUtils.kt$import net.corda.core.crypto.* WildcardImport:IssuerModel.kt$import tornadofx.* WildcardImport:JVMConfig.kt$import tornadofx.* WildcardImport:JacksonSupport.kt$import com.fasterxml.jackson.core.* @@ -2249,7 +2218,6 @@ WildcardImport:NetworkBootstrapper.kt$import net.corda.nodeapi.internal.* WildcardImport:NetworkBootstrapperRunnerTests.kt$import org.junit.* WildcardImport:NetworkBootstrapperTest.kt$import net.corda.core.internal.* - WildcardImport:NetworkBootstrapperTest.kt$import net.corda.testing.core.* WildcardImport:NetworkBuilder.kt$import net.corda.networkbuilder.nodes.* WildcardImport:NetworkIdentityModel.kt$import net.corda.client.jfx.utils.* WildcardImport:NetworkMapServer.kt$import javax.ws.rs.* @@ -2362,7 +2330,6 @@ WildcardImport:PathUtils.kt$import java.io.* WildcardImport:PathUtils.kt$import java.nio.file.* WildcardImport:PersistentIdentityMigrationNewTableTest.kt$import net.corda.testing.core.* - WildcardImport:PersistentIdentityServiceTests.kt$import net.corda.testing.core.* WildcardImport:PersistentNetworkMapCacheTest.kt$import net.corda.testing.core.* WildcardImport:PersistentStateServiceTests.kt$import net.corda.core.contracts.* WildcardImport:Portfolio.kt$import net.corda.core.contracts.* @@ -2387,8 +2354,6 @@ WildcardImport:QueryCriteriaUtils.kt$import net.corda.core.node.services.vault.LikenessOperator.* WildcardImport:RPCMultipleInterfacesTests.kt$import org.junit.Assert.* WildcardImport:RPCSecurityManagerImpl.kt$import org.apache.shiro.authc.* - WildcardImport:RPCServer.kt$import net.corda.core.utilities.* - WildcardImport:RPCServer.kt$import org.apache.activemq.artemis.api.core.client.* WildcardImport:ReceiveFinalityFlowTest.kt$import net.corda.node.services.statemachine.StaffedFlowHospital.* WildcardImport:ReceiveFinalityFlowTest.kt$import net.corda.testing.node.internal.* WildcardImport:ReceiveTransactionFlow.kt$import net.corda.core.contracts.* @@ -2427,7 +2392,6 @@ WildcardImport:SearchField.kt$import tornadofx.* WildcardImport:SecureHashTest.kt$import org.junit.Assert.* WildcardImport:SendTransactionFlow.kt$import net.corda.core.internal.* - WildcardImport:SerializationEnvironmentRule.kt$import net.corda.testing.internal.* WildcardImport:SerializationHelper.kt$import java.lang.reflect.* WildcardImport:SerializationHelper.kt$import net.corda.core.serialization.* WildcardImport:SerializationOutputTests.kt$import java.time.* @@ -2572,7 +2536,6 @@ WildcardImport:VaultWithCashTest.kt$import net.corda.testing.core.* WildcardImport:VaultWithCashTest.kt$import net.corda.testing.internal.vault.* WildcardImport:VerifyTransactionTest.kt$import net.corda.finance.contracts.asset.Cash.Commands.* - WildcardImport:VersionedParsingExampleTest.kt$import net.corda.common.configuration.parsing.internal.* WildcardImport:WebServerController.kt$import tornadofx.* WildcardImport:WhitelistBasedTypeModelConfiguration.kt$import org.apache.qpid.proton.amqp.* WildcardImport:WhitelistGenerator.kt$import net.corda.core.internal.* @@ -2581,8 +2544,6 @@ WildcardImport:WireTransaction.kt$import net.corda.core.internal.* WildcardImport:WithFinality.kt$import net.corda.core.flows.* WildcardImport:WithMockNet.kt$import com.natpryce.hamkrest.* - WildcardImport:WorkflowTransactionBuildTutorial.kt$import net.corda.core.contracts.* - WildcardImport:WorkflowTransactionBuildTutorial.kt$import net.corda.core.flows.* WildcardImport:X509CRLSerializer.kt$import net.corda.serialization.internal.amqp.* WildcardImport:X509CertificateSerializer.kt$import net.corda.serialization.internal.amqp.* WildcardImport:X509EdDSAEngine.kt$import java.security.* diff --git a/node-api/build.gradle b/node-api/build.gradle index 2f5f774b9a..a8a8607cf6 100644 --- a/node-api/build.gradle +++ b/node-api/build.gradle @@ -19,6 +19,8 @@ dependencies { compile "org.apache.activemq:artemis-core-client:${artemis_version}" compile "org.apache.activemq:artemis-commons:${artemis_version}" + compile "io.netty:netty-handler-proxy:$netty_version" + // TypeSafe Config: for simple and human friendly config files. compile "com.typesafe:config:$typesafe_config_version" diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisMessagingClient.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisMessagingClient.kt index b0957af4e4..1206cbe8ec 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisMessagingClient.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisMessagingClient.kt @@ -4,6 +4,9 @@ import net.corda.core.serialization.internal.nodeSerializationEnv import net.corda.core.utilities.NetworkHostAndPort import net.corda.core.utilities.loggerFor import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.NODE_P2P_USER +import net.corda.nodeapi.internal.ArtemisTcpTransport.Companion.p2pConnectorTcpTransport +import net.corda.nodeapi.internal.ArtemisTcpTransport.Companion.p2pConnectorTcpTransportFromList +import net.corda.nodeapi.internal.config.MessagingServerConnectionConfiguration import net.corda.nodeapi.internal.config.MutualSslConfiguration import org.apache.activemq.artemis.api.core.client.* import org.apache.activemq.artemis.api.core.client.ActiveMQClient.DEFAULT_ACK_BATCH_SIZE @@ -17,28 +20,55 @@ interface ArtemisSessionProvider { class ArtemisMessagingClient(private val config: MutualSslConfiguration, private val serverAddress: NetworkHostAndPort, private val maxMessageSize: Int, - private val failoverCallback: ((FailoverEventType) -> Unit)? = null) : ArtemisSessionProvider { + private val autoCommitSends: Boolean = true, + private val autoCommitAcks: Boolean = true, + private val confirmationWindowSize: Int = -1, + private val messagingServerConnectionConfig: MessagingServerConnectionConfiguration? = null, + private val backupServerAddressPool: List = emptyList(), + private val failoverCallback: ((FailoverEventType) -> Unit)? = null +) : ArtemisSessionProvider { companion object { private val log = loggerFor() + const val CORDA_ARTEMIS_CALL_TIMEOUT_PROP_NAME = "net.corda.nodeapi.artemismessagingclient.CallTimeout" + const val CORDA_ARTEMIS_CALL_TIMEOUT_DEFAULT = 5000L } - class Started(val sessionFactory: ClientSessionFactory, val session: ClientSession, val producer: ClientProducer) + class Started(val serverLocator: ServerLocator, val sessionFactory: ClientSessionFactory, val session: ClientSession, val producer: ClientProducer) override var started: Started? = null private set override fun start(): Started = synchronized(this) { check(started == null) { "start can't be called twice" } + val tcpTransport = p2pConnectorTcpTransport(serverAddress, config) + val backupTransports = p2pConnectorTcpTransportFromList(backupServerAddressPool, config) + log.info("Connecting to message broker: $serverAddress") - // TODO Add broker CN to config for host verification in case the embedded broker isn't used - val tcpTransport = ArtemisTcpTransport.p2pConnectorTcpTransport(serverAddress, config) - val locator = ActiveMQClient.createServerLocatorWithoutHA(tcpTransport).apply { + if (backupTransports.isNotEmpty()) { + log.info("Back-up message broker addresses: $backupServerAddressPool") + } + // If back-up artemis addresses are configured, the locator will be created using HA mode. + @Suppress("SpreadOperator") + val locator = ActiveMQClient.createServerLocator(backupTransports.isNotEmpty(), *(listOf(tcpTransport) + backupTransports).toTypedArray()).apply { // Never time out on our loopback Artemis connections. If we switch back to using the InVM transport this // would be the default and the two lines below can be deleted. connectionTTL = 60000 clientFailureCheckPeriod = 30000 + callFailoverTimeout = java.lang.Long.getLong(CORDA_ARTEMIS_CALL_TIMEOUT_PROP_NAME, CORDA_ARTEMIS_CALL_TIMEOUT_DEFAULT) + callTimeout = java.lang.Long.getLong(CORDA_ARTEMIS_CALL_TIMEOUT_PROP_NAME, CORDA_ARTEMIS_CALL_TIMEOUT_DEFAULT) minLargeMessageSize = maxMessageSize isUseGlobalPools = nodeSerializationEnv != null + confirmationWindowSize = this@ArtemisMessagingClient.confirmationWindowSize + producerWindowSize = -1 + messagingServerConnectionConfig?.let { + connectionLoadBalancingPolicyClassName = RoundRobinConnectionPolicy::class.java.canonicalName + reconnectAttempts = messagingServerConnectionConfig.reconnectAttempts(isHA) + retryInterval = messagingServerConnectionConfig.retryInterval().toMillis() + retryIntervalMultiplier = messagingServerConnectionConfig.retryIntervalMultiplier() + maxRetryInterval = messagingServerConnectionConfig.maxRetryInterval(isHA).toMillis() + isFailoverOnInitialConnection = messagingServerConnectionConfig.failoverOnInitialAttempt(isHA) + initialConnectAttempts = messagingServerConnectionConfig.initialConnectAttempts(isHA) + } addIncomingInterceptor(ArtemisMessageSizeChecksInterceptor(maxMessageSize)) } val sessionFactory = locator.createSessionFactory() @@ -50,23 +80,24 @@ class ArtemisMessagingClient(private val config: MutualSslConfiguration, // using our TLS certificate. // Note that the acknowledgement of messages is not flushed to the Artermis journal until the default buffer // size of 1MB is acknowledged. - val session = sessionFactory!!.createSession(NODE_P2P_USER, NODE_P2P_USER, false, true, true, false, DEFAULT_ACK_BATCH_SIZE) + val session = sessionFactory!!.createSession(NODE_P2P_USER, NODE_P2P_USER, false, autoCommitSends, autoCommitAcks, false, DEFAULT_ACK_BATCH_SIZE) session.start() // Create a general purpose producer. val producer = session.createProducer() - return Started(sessionFactory, session, producer).also { started = it } + return Started(locator, sessionFactory, session, producer).also { started = it } } override fun stop() = synchronized(this) { started?.run { producer.close() // Since we are leaking the session outside of this class it may well be already closed. - if(!session.isClosed) { + if (session.stillOpen()) { // Ensure any trailing messages are committed to the journal session.commit() } // Closing the factory closes all the sessions it produced as well. sessionFactory.close() + serverLocator.close() } started = null } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisMessagingComponent.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisMessagingComponent.kt index 14052c1789..0da47bc2df 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisMessagingComponent.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisMessagingComponent.kt @@ -34,6 +34,7 @@ class ArtemisMessagingComponent { // This is a rough guess on the extra space needed on top of maxMessageSize to store the journal. // TODO: we might want to make this value configurable. const val JOURNAL_HEADER_SIZE = 1024 + object P2PMessagingHeaders { // This is a "property" attached to an Artemis MQ message object, which contains our own notion of "topic". // We should probably try to unify our notion of "topic" (really, just a string that identifies an endpoint @@ -123,6 +124,11 @@ class ArtemisMessagingComponent { require(address.startsWith(PEERS_PREFIX)) { "Failed to map address: $address to a remote topic as it is not in the $PEERS_PREFIX namespace" } return P2P_PREFIX + address.substring(PEERS_PREFIX.length) } + + fun translateInboxAddressToLocalQueue(address: String): String { + require(address.startsWith(P2P_PREFIX)) { "Failed to map topic: $address to a local address as it is not in the $P2P_PREFIX namespace" } + return PEERS_PREFIX + address.substring(P2P_PREFIX.length) + } } override val queueName: String = "$P2P_PREFIX${identity.toStringShort()}" diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisTcpTransport.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisTcpTransport.kt index d1ae947fc3..d3122c9dc8 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisTcpTransport.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisTcpTransport.kt @@ -100,35 +100,43 @@ class ArtemisTcpTransport { fun p2pAcceptorTcpTransport(hostAndPort: NetworkHostAndPort, config: MutualSslConfiguration?, enableSSL: Boolean = true): TransportConfiguration { - return p2pAcceptorTcpTransport(hostAndPort, config?.keyStore, config?.trustStore, enableSSL = enableSSL) + return p2pAcceptorTcpTransport(hostAndPort, config?.keyStore, config?.trustStore, enableSSL = enableSSL, useOpenSsl = config?.useOpenSsl ?: false) } - fun p2pConnectorTcpTransport(hostAndPort: NetworkHostAndPort, config: MutualSslConfiguration?, enableSSL: Boolean = true): TransportConfiguration { + fun p2pConnectorTcpTransport(hostAndPort: NetworkHostAndPort, config: MutualSslConfiguration?, enableSSL: Boolean = true, keyStoreProvider: String? = null): TransportConfiguration { - return p2pConnectorTcpTransport(hostAndPort, config?.keyStore, config?.trustStore, enableSSL = enableSSL) + return p2pConnectorTcpTransport(hostAndPort, config?.keyStore, config?.trustStore, enableSSL = enableSSL, useOpenSsl = config?.useOpenSsl ?: false, keyStoreProvider = keyStoreProvider) } - fun p2pAcceptorTcpTransport(hostAndPort: NetworkHostAndPort, keyStore: FileBasedCertificateStoreSupplier?, trustStore: FileBasedCertificateStoreSupplier?, enableSSL: Boolean = true): TransportConfiguration { + fun p2pAcceptorTcpTransport(hostAndPort: NetworkHostAndPort, keyStore: FileBasedCertificateStoreSupplier?, trustStore: FileBasedCertificateStoreSupplier?, enableSSL: Boolean = true, useOpenSsl: Boolean = false): TransportConfiguration { val options = defaultArtemisOptions(hostAndPort).toMutableMap() if (enableSSL) { options.putAll(defaultSSLOptions) (keyStore to trustStore).addToTransportOptions(options) + options[TransportConstants.SSL_PROVIDER] = if (useOpenSsl) TransportConstants.OPENSSL_PROVIDER else TransportConstants.DEFAULT_SSL_PROVIDER } options[TransportConstants.HANDSHAKE_TIMEOUT] = 0 // Suppress core.server.lambda$channelActive$0 - AMQ224088 error from load balancer type connections return TransportConfiguration(acceptorFactoryClassName, options) } - fun p2pConnectorTcpTransport(hostAndPort: NetworkHostAndPort, keyStore: FileBasedCertificateStoreSupplier?, trustStore: FileBasedCertificateStoreSupplier?, enableSSL: Boolean = true): TransportConfiguration { + @Suppress("LongParameterList") + fun p2pConnectorTcpTransport(hostAndPort: NetworkHostAndPort, keyStore: FileBasedCertificateStoreSupplier?, trustStore: FileBasedCertificateStoreSupplier?, enableSSL: Boolean = true, useOpenSsl: Boolean = false, keyStoreProvider: String? = null): TransportConfiguration { val options = defaultArtemisOptions(hostAndPort).toMutableMap() if (enableSSL) { options.putAll(defaultSSLOptions) (keyStore to trustStore).addToTransportOptions(options) + options[TransportConstants.SSL_PROVIDER] = if (useOpenSsl) TransportConstants.OPENSSL_PROVIDER else TransportConstants.DEFAULT_SSL_PROVIDER + keyStoreProvider?.let { options.put(TransportConstants.KEYSTORE_PROVIDER_PROP_NAME, keyStoreProvider) } } return TransportConfiguration(connectorFactoryClassName, options) } + fun p2pConnectorTcpTransportFromList(hostAndPortList: List, config: MutualSslConfiguration?, enableSSL: Boolean = true, keyStoreProvider: String? = null): List = hostAndPortList.map { + p2pConnectorTcpTransport(it, config, enableSSL, keyStoreProvider) + } + fun rpcAcceptorTcpTransport(hostAndPort: NetworkHostAndPort, config: BrokerRpcSslOptions?, enableSSL: Boolean = true): TransportConfiguration { val options = defaultArtemisOptions(hostAndPort).toMutableMap() @@ -156,12 +164,17 @@ class ArtemisTcpTransport { rpcConnectorTcpTransport(it, config, enableSSL) } - fun rpcInternalClientTcpTransport(hostAndPort: NetworkHostAndPort, config: SslConfiguration): TransportConfiguration { - return TransportConfiguration(connectorFactoryClassName, defaultArtemisOptions(hostAndPort) + defaultSSLOptions + config.toTransportOptions()) + fun rpcInternalClientTcpTransport(hostAndPort: NetworkHostAndPort, config: SslConfiguration, keyStoreProvider: String? = null): TransportConfiguration { + return TransportConfiguration(connectorFactoryClassName, defaultArtemisOptions(hostAndPort) + defaultSSLOptions + config.toTransportOptions() + asMap(keyStoreProvider)) } - fun rpcInternalAcceptorTcpTransport(hostAndPort: NetworkHostAndPort, config: SslConfiguration): TransportConfiguration { - return TransportConfiguration(acceptorFactoryClassName, defaultArtemisOptions(hostAndPort) + defaultSSLOptions + config.toTransportOptions() + (TransportConstants.HANDSHAKE_TIMEOUT to 0)) + fun rpcInternalAcceptorTcpTransport(hostAndPort: NetworkHostAndPort, config: SslConfiguration, keyStoreProvider: String? = null): TransportConfiguration { + return TransportConfiguration(acceptorFactoryClassName, defaultArtemisOptions(hostAndPort) + defaultSSLOptions + + config.toTransportOptions() + (TransportConstants.HANDSHAKE_TIMEOUT to 0) + asMap(keyStoreProvider)) + } + + private fun asMap(keyStoreProvider: String?): Map { + return keyStoreProvider?.let {mutableMapOf(TransportConstants.KEYSTORE_PROVIDER_PROP_NAME to it)} ?: emptyMap() } } } \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisUtils.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisUtils.kt index 97abc0d024..23bb9d1428 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisUtils.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisUtils.kt @@ -1,5 +1,4 @@ @file:JvmName("ArtemisUtils") - package net.corda.nodeapi.internal import java.nio.file.FileSystems @@ -16,3 +15,4 @@ fun Path.requireOnDefaultFileSystem() { fun requireMessageSize(messageSize: Int, limit: Int) { require(messageSize <= limit) { "Message exceeds maxMessageSize network parameter, maxMessageSize: [$limit], message size: [$messageSize]" } } + diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ClientSessionUtils.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ClientSessionUtils.kt new file mode 100644 index 0000000000..1b7ae2aea7 --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ClientSessionUtils.kt @@ -0,0 +1,8 @@ +package net.corda.nodeapi.internal + +import org.apache.activemq.artemis.api.core.client.ClientSession +import org.apache.activemq.artemis.core.client.impl.ClientSessionInternal + +fun ClientSession.stillOpen(): Boolean { + return (!isClosed && (this as? ClientSessionInternal)?.isClosing != false) +} \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ConcurrentBox.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ConcurrentBox.kt new file mode 100644 index 0000000000..1ae2807c4d --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ConcurrentBox.kt @@ -0,0 +1,17 @@ +package net.corda.nodeapi.internal + +import java.util.concurrent.locks.ReentrantReadWriteLock +import kotlin.concurrent.read +import kotlin.concurrent.write + +/** + * A [ConcurrentBox] allows the implementation of track() with reduced contention. [concurrent] may be run from several + * threads (which means it MUST be threadsafe!), while [exclusive] stops the world until the tracking has been set up. + * Internally [ConcurrentBox] is implemented simply as a read-write lock. + */ +class ConcurrentBox(val content: T) { + val lock = ReentrantReadWriteLock() + + inline fun concurrent(block: T.() -> R): R = lock.read { block(content) } + inline fun exclusive(block: T.() -> R): R = lock.write { block(content) } +} diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/RoundRobinConnectionPolicy.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/RoundRobinConnectionPolicy.kt new file mode 100644 index 0000000000..9f2b0135db --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/RoundRobinConnectionPolicy.kt @@ -0,0 +1,18 @@ +package net.corda.nodeapi.internal + +import org.apache.activemq.artemis.api.core.client.loadbalance.ConnectionLoadBalancingPolicy + +/** + * Implementation of an Artemis load balancing policy. It does round-robin always starting from the first position, whereas + * the current [RoundRobinConnectionLoadBalancingPolicy] in Artemis picks the starting position randomly. This can lead to + * attempting to connect to an inactive broker on the first attempt, which can cause start-up delays depending on what connection + * settings are used. + */ +class RoundRobinConnectionPolicy : ConnectionLoadBalancingPolicy { + private var pos = 0 + + override fun select(max: Int): Int { + pos = if (pos >= max) 0 else pos + return pos++ + } +} \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/AMQPBridgeManager.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/AMQPBridgeManager.kt index ed990acd03..40523033f2 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/AMQPBridgeManager.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/AMQPBridgeManager.kt @@ -1,5 +1,8 @@ +@file:Suppress("TooGenericExceptionCaught") // needs to catch and handle/rethrow *all* exceptions in many places package net.corda.nodeapi.internal.bridging +import com.google.common.util.concurrent.ThreadFactoryBuilder +import io.netty.channel.EventLoop import io.netty.channel.EventLoopGroup import io.netty.channel.nio.NioEventLoopGroup import net.corda.core.identity.CordaX500Name @@ -11,11 +14,14 @@ import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.NODE_P2P_U import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.P2PMessagingHeaders import net.corda.nodeapi.internal.ArtemisMessagingComponent.RemoteInboxAddress.Companion.translateLocalQueueToInboxAddress import net.corda.nodeapi.internal.ArtemisSessionProvider +import net.corda.nodeapi.internal.ArtemisConstants.MESSAGE_ID_KEY import net.corda.nodeapi.internal.config.CertificateStore -import net.corda.nodeapi.internal.config.MutualSslConfiguration import net.corda.nodeapi.internal.protonwrapper.messages.MessageStatus import net.corda.nodeapi.internal.protonwrapper.netty.AMQPClient import net.corda.nodeapi.internal.protonwrapper.netty.AMQPConfiguration +import net.corda.nodeapi.internal.protonwrapper.netty.ProxyConfig +import net.corda.nodeapi.internal.protonwrapper.netty.RevocationConfig +import org.apache.activemq.artemis.api.core.ActiveMQObjectClosedException import org.apache.activemq.artemis.api.core.SimpleString import org.apache.activemq.artemis.api.core.client.ActiveMQClient.DEFAULT_ACK_BATCH_SIZE import org.apache.activemq.artemis.api.core.client.ClientConsumer @@ -23,6 +29,10 @@ import org.apache.activemq.artemis.api.core.client.ClientMessage import org.apache.activemq.artemis.api.core.client.ClientSession import org.slf4j.MDC import rx.Subscription +import java.util.concurrent.Executors +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit import java.util.concurrent.locks.ReentrantLock import kotlin.concurrent.withLock @@ -34,33 +44,46 @@ import kotlin.concurrent.withLock * The Netty thread pool used by the AMQPBridges is also shared and managed by the AMQPBridgeManager. */ @VisibleForTesting -class AMQPBridgeManager(config: MutualSslConfiguration, - maxMessageSize: Int, - crlCheckSoftFail: Boolean, - private val artemisMessageClientFactory: () -> ArtemisSessionProvider, - private val bridgeMetricsService: BridgeMetricsService? = null) : BridgeManager { +open class AMQPBridgeManager(keyStore: CertificateStore, + trustStore: CertificateStore, + useOpenSSL: Boolean, + proxyConfig: ProxyConfig? = null, + maxMessageSize: Int, + revocationConfig: RevocationConfig, + enableSNI: Boolean, + private val artemisMessageClientFactory: () -> ArtemisSessionProvider, + private val bridgeMetricsService: BridgeMetricsService? = null, + trace: Boolean, + sslHandshakeTimeout: Long?, + private val bridgeConnectionTTLSeconds: Int) : BridgeManager { private val lock = ReentrantLock() private val queueNamesToBridgesMap = mutableMapOf>() - private class AMQPConfigurationImpl private constructor(override val keyStore: CertificateStore, - override val trustStore: CertificateStore, - override val maxMessageSize: Int, - override val crlCheckSoftFail: Boolean) : AMQPConfiguration { - constructor(config: MutualSslConfiguration, maxMessageSize: Int, crlCheckSoftFail: Boolean) : this(config.keyStore.get(), config.trustStore.get(), maxMessageSize, crlCheckSoftFail) + private class AMQPConfigurationImpl(override val keyStore: CertificateStore, + override val trustStore: CertificateStore, + override val proxyConfig: ProxyConfig?, + override val maxMessageSize: Int, + override val revocationConfig: RevocationConfig, + override val useOpenSsl: Boolean, + override val enableSNI: Boolean, + override val sourceX500Name: String? = null, + override val trace: Boolean, + private val _sslHandshakeTimeout: Long?) : AMQPConfiguration { + override val sslHandshakeTimeout: Long + get() = _sslHandshakeTimeout ?: super.sslHandshakeTimeout } - private val amqpConfig: AMQPConfiguration = AMQPConfigurationImpl(config, maxMessageSize, crlCheckSoftFail) + private val amqpConfig: AMQPConfiguration = AMQPConfigurationImpl(keyStore, trustStore, proxyConfig, maxMessageSize, revocationConfig,useOpenSSL, enableSNI, trace = trace, _sslHandshakeTimeout = sslHandshakeTimeout) private var sharedEventLoopGroup: EventLoopGroup? = null private var artemis: ArtemisSessionProvider? = null - constructor(config: MutualSslConfiguration, - p2pAddress: NetworkHostAndPort, - maxMessageSize: Int, - crlCheckSoftFail: Boolean) : this(config, maxMessageSize, crlCheckSoftFail, { ArtemisMessagingClient(config, p2pAddress, maxMessageSize) }) - companion object { - private const val NUM_BRIDGE_THREADS = 0 // Default sized pool + + private const val CORDA_NUM_BRIDGE_THREADS_PROP_NAME = "net.corda.nodeapi.amqpbridgemanager.NumBridgeThreads" + + private val NUM_BRIDGE_THREADS = Integer.getInteger(CORDA_NUM_BRIDGE_THREADS_PROP_NAME, 0) // Default 0 means Netty default sized pool + private const val ARTEMIS_RETRY_BACKOFF = 5000L } /** @@ -71,13 +94,16 @@ class AMQPBridgeManager(config: MutualSslConfiguration, * If the delivery fails the session is rolled back to prevent loss of the message. This may cause duplicate delivery, * however Artemis and the remote Corda instanced will deduplicate these messages. */ - private class AMQPBridge(val queueName: String, + @Suppress("TooManyFunctions") + private class AMQPBridge(val sourceX500Name: String, + val queueName: String, val targets: List, val legalNames: Set, private val amqpConfig: AMQPConfiguration, sharedEventGroup: EventLoopGroup, private val artemis: ArtemisSessionProvider, - private val bridgeMetricsService: BridgeMetricsService?) { + private val bridgeMetricsService: BridgeMetricsService?, + private val bridgeConnectionTTLSeconds: Int) { companion object { private val log = contextLogger() } @@ -86,6 +112,7 @@ class AMQPBridgeManager(config: MutualSslConfiguration, val oldMDC = MDC.getCopyOfContextMap() ?: emptyMap() try { MDC.put("queueName", queueName) + MDC.put("source", amqpConfig.sourceX500Name) MDC.put("targets", targets.joinToString(separator = ";") { it.toString() }) MDC.put("legalNames", legalNames.joinToString(separator = ";") { it.toString() }) MDC.put("maxMessageSize", amqpConfig.maxMessageSize.toString()) @@ -106,10 +133,80 @@ class AMQPBridgeManager(config: MutualSslConfiguration, private fun logWarnWithMDC(msg: String) = withMDC { log.warn(msg) } val amqpClient = AMQPClient(targets, legalNames, amqpConfig, sharedThreadPool = sharedEventGroup) - private val lock = ReentrantLock() // lock to serialise session level access private var session: ClientSession? = null private var consumer: ClientConsumer? = null private var connectedSubscription: Subscription? = null + @Volatile + private var messagesReceived: Boolean = false + private val eventLoop: EventLoop = sharedEventGroup.next() + private var artemisState: ArtemisState = ArtemisState.STOPPED + set(value) { + logDebugWithMDC { "State change $field to $value" } + field = value + } + @Suppress("MagicNumber") + private var artemisHeartbeatPlusBackoff = TimeUnit.SECONDS.toMillis(90) + private var amqpRestartEvent: ScheduledFuture? = null + private var scheduledExecutorService: ScheduledExecutorService + = Executors.newSingleThreadScheduledExecutor(ThreadFactoryBuilder().setNameFormat("bridge-connection-reset-%d").build()) + + @Suppress("ClassNaming") + private sealed class ArtemisState { + object STARTING : ArtemisState() + data class STARTED(override val pending: ScheduledFuture) : ArtemisState() + + object CHECKING : ArtemisState() + object RESTARTED : ArtemisState() + object RECEIVING : ArtemisState() + + object AMQP_STOPPED : ArtemisState() + object AMQP_STARTING : ArtemisState() + object AMQP_STARTED : ArtemisState() + object AMQP_RESTARTED : ArtemisState() + + object STOPPING : ArtemisState() + object STOPPED : ArtemisState() + data class STOPPED_AMQP_START_SCHEDULED(override val pending: ScheduledFuture) : ArtemisState() + + open val pending: ScheduledFuture? = null + + override fun toString(): String = javaClass.simpleName + } + + private fun artemis(inProgress: ArtemisState, block: (precedingState: ArtemisState) -> ArtemisState) { + val runnable = { + synchronized(artemis) { + try { + val precedingState = artemisState + artemisState.pending?.cancel(false) + artemisState = inProgress + artemisState = block(precedingState) + } catch (ex: Exception) { + withMDC { log.error("Unexpected error in Artemis processing in state $artemisState.", ex) } + } + } + } + if (eventLoop.inEventLoop()) { + runnable() + } else { + eventLoop.execute(runnable) + } + } + + private fun scheduledArtemis(delay: Long, unit: TimeUnit, inProgress: ArtemisState, block: (precedingState: ArtemisState) -> ArtemisState): ScheduledFuture { + return eventLoop.schedule({ + artemis(inProgress, block) + }, delay, unit) + } + + private fun scheduledArtemisInExecutor(delay: Long, unit: TimeUnit, inProgress: ArtemisState, nextState: ArtemisState, block: () -> Unit): ScheduledFuture { + return scheduledExecutorService.schedule({ + artemis(inProgress) { + nextState + } + block() + }, delay, unit) + } fun start() { logInfoWithMDC("Create new AMQP bridge") @@ -119,55 +216,196 @@ class AMQPBridgeManager(config: MutualSslConfiguration, fun stop() { logInfoWithMDC("Stopping AMQP bridge") - lock.withLock { - synchronized(artemis) { - consumer?.close() - consumer = null - session?.stop() - session = null - } - } - amqpClient.stop() - connectedSubscription?.unsubscribe() - connectedSubscription = null - } - - private fun onSocketConnected(connected: Boolean) { - lock.withLock { - synchronized(artemis) { - if (connected) { - logInfoWithMDC("Bridge Connected") - bridgeMetricsService?.bridgeConnected(targets, legalNames) - val sessionFactory = artemis.started!!.sessionFactory - val session = sessionFactory.createSession(NODE_P2P_USER, NODE_P2P_USER, false, true, true, false, DEFAULT_ACK_BATCH_SIZE) - this.session = session - val consumer = session.createConsumer(queueName) - this.consumer = consumer - consumer.setMessageHandler(this@AMQPBridge::clientArtemisMessageHandler) - session.start() - } else { - logInfoWithMDC("Bridge Disconnected") - bridgeMetricsService?.bridgeDisconnected(targets, legalNames) - consumer?.close() - consumer = null - session?.stop() + artemis(ArtemisState.STOPPING) { + logInfoWithMDC("Stopping Artemis because stopping AMQP bridge") + closeConsumer() + consumer = null + eventLoop.execute { + artemis(ArtemisState.STOPPING) { + stopSession() session = null + ArtemisState.STOPPED } } + ArtemisState.STOPPING } + bridgeMetricsService?.bridgeDisconnected(targets, legalNames) + connectedSubscription?.unsubscribe() + connectedSubscription = null + // Do this last because we already scheduled the Artemis stop, so it's okay to unsubscribe onConnected first. + amqpClient.stop() + } + + @Suppress("ComplexMethod") + private fun onSocketConnected(connected: Boolean) { + if (connected) { + logInfoWithMDC("Bridge Connected") + + bridgeMetricsService?.bridgeConnected(targets, legalNames) + if (bridgeConnectionTTLSeconds > 0) { + // AMQP outbound connection will be restarted periodically with bridgeConnectionTTLSeconds interval + amqpRestartEvent = scheduledArtemisInExecutor(bridgeConnectionTTLSeconds.toLong(), TimeUnit.SECONDS, + ArtemisState.AMQP_STOPPED, ArtemisState.AMQP_RESTARTED) { + logInfoWithMDC("Bridge connection time to live exceeded. Restarting AMQP connection") + stopAndStartOutbound(ArtemisState.AMQP_RESTARTED) + } + } + artemis(ArtemisState.STARTING) { + val startedArtemis = artemis.started + if (startedArtemis == null) { + logInfoWithMDC("Bridge Connected but Artemis is disconnected") + ArtemisState.STOPPED + } else { + logInfoWithMDC("Bridge Connected so starting Artemis") + artemisHeartbeatPlusBackoff = startedArtemis.serverLocator.connectionTTL + ARTEMIS_RETRY_BACKOFF + try { + createSessionAndConsumer(startedArtemis) + ArtemisState.STARTED(scheduledArtemis(artemisHeartbeatPlusBackoff, TimeUnit.MILLISECONDS, ArtemisState.CHECKING) { + if (!messagesReceived) { + logInfoWithMDC("No messages received on new bridge. Restarting Artemis session") + if (restartSession()) { + ArtemisState.RESTARTED + } else { + logInfoWithMDC("Artemis session restart failed. Aborting by restarting AMQP connection.") + stopAndStartOutbound() + } + } else { + ArtemisState.RECEIVING + } + }) + } catch (ex: Exception) { + // Now, bounce the AMQP connection to restart the sequence of establishing the connectivity back from the beginning. + withMDC { log.warn("Create Artemis start session error. Restarting AMQP connection", ex) } + stopAndStartOutbound() + } + } + } + } else { + logInfoWithMDC("Bridge Disconnected") + amqpRestartEvent?.cancel(false) + if (artemisState != ArtemisState.AMQP_STARTING && artemisState != ArtemisState.STOPPED) { + bridgeMetricsService?.bridgeDisconnected(targets, legalNames) + } + artemis(ArtemisState.STOPPING) { precedingState: ArtemisState -> + logInfoWithMDC("Stopping Artemis because AMQP bridge disconnected") + closeConsumer() + consumer = null + eventLoop.execute { + artemis(ArtemisState.STOPPING) { + stopSession() + session = null + when (precedingState) { + ArtemisState.AMQP_STOPPED -> + ArtemisState.STOPPED_AMQP_START_SCHEDULED(scheduledArtemis(artemisHeartbeatPlusBackoff, + TimeUnit.MILLISECONDS, ArtemisState.AMQP_STARTING) { startOutbound() }) + ArtemisState.AMQP_RESTARTED -> { + artemis(ArtemisState.AMQP_STARTING) { startOutbound() } + ArtemisState.AMQP_STARTING + } + else -> ArtemisState.STOPPED + } + } + } + ArtemisState.STOPPING + } + } + } + + private fun startOutbound(): ArtemisState { + logInfoWithMDC("Starting AMQP client") + amqpClient.start() + return ArtemisState.AMQP_STARTED + } + + private fun stopAndStartOutbound(state: ArtemisState = ArtemisState.AMQP_STOPPED): ArtemisState { + amqpClient.stop() + // Bridge disconnect will detect this state and schedule an AMQP start. + return state + } + + private fun createSessionAndConsumer(startedArtemis: ArtemisMessagingClient.Started): ClientSession { + logInfoWithMDC("Creating session and consumer.") + val sessionFactory = startedArtemis.sessionFactory + val session = sessionFactory.createSession(NODE_P2P_USER, NODE_P2P_USER, false, true, + true, false, DEFAULT_ACK_BATCH_SIZE) + this.session = session + // Several producers (in the case of shared bridge) can put messages in the same outbound p2p queue. + // The consumers are created using the source x500 name as a filter + val consumer = if (amqpConfig.enableSNI) { + session.createConsumer(queueName, "hyphenated_props:sender-subject-name = '${amqpConfig.sourceX500Name}'") + } else { + session.createConsumer(queueName) + } + this.consumer = consumer + session.start() + consumer.setMessageHandler(this@AMQPBridge::clientArtemisMessageHandler) + return session + } + + private fun closeConsumer(): Boolean { + var closed = false + try { + consumer?.apply { + if (!isClosed) { + close() + } + } + closed = true + } catch (ex: Exception) { + withMDC { log.warn("Close artemis consumer error", ex) } + } finally { + return closed + } + } + + private fun stopSession(): Boolean { + var stopped = false + try { + session?.apply { + if (!isClosed) { + stop() + } + } + stopped = true + } catch (ex: Exception) { + withMDC { log.warn("Stop Artemis session error", ex) } + } finally { + return stopped + } + } + + private fun restartSession(): Boolean { + if (!stopSession()) { + // Session timed out stopping. The request/responses can be out of sequence on the session now, so abandon it. + session = null + // The consumer is also dead now too as attached to the dead session. + consumer = null + return false + } + try { + // Does not wait for a response. + this.session?.start() + } catch (ex: Exception) { + withMDC { log.error("Start Artemis session error", ex) } + } + return true } private fun clientArtemisMessageHandler(artemisMessage: ClientMessage) { + messagesReceived = true if (artemisMessage.bodySize > amqpConfig.maxMessageSize) { val msg = "Message exceeds maxMessageSize network parameter, maxMessageSize: [${amqpConfig.maxMessageSize}], message size: [${artemisMessage.bodySize}], " + - "dropping message, uuid: ${artemisMessage.getObjectProperty("_AMQ_DUPL_ID")}" + "dropping message, uuid: ${artemisMessage.getObjectProperty(MESSAGE_ID_KEY)}" logWarnWithMDC(msg) bridgeMetricsService?.packetDropEvent(artemisMessage, msg) // Ack the message to prevent same message being sent to us again. - artemisMessage.individualAcknowledge() + try { + artemisMessage.individualAcknowledge() + } catch (ex: ActiveMQObjectClosedException) { + log.warn("Artemis message was closed") + } return } - val data = ByteArray(artemisMessage.bodySize).apply { artemisMessage.bodyBuffer.readBytes(this) } val properties = HashMap() for (key in P2PMessagingHeaders.whitelistedHeaders) { if (artemisMessage.containsProperty(key)) { @@ -178,18 +416,22 @@ class AMQPBridgeManager(config: MutualSslConfiguration, properties[key] = value } } - logDebugWithMDC { "Bridged Send to ${legalNames.first()} uuid: ${artemisMessage.getObjectProperty("_AMQ_DUPL_ID")}" } + logDebugWithMDC { "Bridged Send to ${legalNames.first()} uuid: ${artemisMessage.getObjectProperty(MESSAGE_ID_KEY)}" } val peerInbox = translateLocalQueueToInboxAddress(queueName) - val sendableMessage = amqpClient.createMessage(data, peerInbox, + val sendableMessage = amqpClient.createMessage(artemisMessage.payload(), peerInbox, legalNames.first().toString(), properties) sendableMessage.onComplete.then { logDebugWithMDC { "Bridge ACK ${sendableMessage.onComplete.get()}" } - lock.withLock { + eventLoop.submit { if (sendableMessage.onComplete.get() == MessageStatus.Acknowledged) { - artemisMessage.individualAcknowledge() + try { + artemisMessage.individualAcknowledge() + } catch (ex: ActiveMQObjectClosedException) { + log.warn("Artemis message was closed") + } } else { - logInfoWithMDC("Rollback rejected message uuid: ${artemisMessage.getObjectProperty("_AMQ_DUPL_ID")}") + logInfoWithMDC("Rollback rejected message uuid: ${artemisMessage.getObjectProperty(MESSAGE_ID_KEY)}") // We need to commit any acknowledged messages before rolling back the failed // (unacknowledged) message. session?.commit() @@ -202,9 +444,9 @@ class AMQPBridgeManager(config: MutualSslConfiguration, } catch (ex: IllegalStateException) { // Attempting to send a message while the AMQP client is disconnected may cause message loss. // The failed message is rolled back after committing acknowledged messages. - lock.withLock { - ex.message?.let { logInfoWithMDC(it)} - logInfoWithMDC("Rollback rejected message uuid: ${artemisMessage.getObjectProperty("_AMQ_DUPL_ID")}") + eventLoop.submit { + ex.message?.let { logInfoWithMDC(it) } + logInfoWithMDC("Rollback rejected message uuid: ${artemisMessage.getObjectProperty(MESSAGE_ID_KEY)}") session?.commit() session?.rollback(false) } @@ -213,20 +455,22 @@ class AMQPBridgeManager(config: MutualSslConfiguration, } } - override fun deployBridge(queueName: String, targets: List, legalNames: Set) { - val newBridge = lock.withLock { + override fun deployBridge(sourceX500Name: String, queueName: String, targets: List, legalNames: Set) { + lock.withLock { val bridges = queueNamesToBridgesMap.getOrPut(queueName) { mutableListOf() } for (target in targets) { - if (bridges.any { it.targets.contains(target) }) { + if (bridges.any { it.targets.contains(target) && it.sourceX500Name == sourceX500Name }) { return } } - val newBridge = AMQPBridge(queueName, targets, legalNames, amqpConfig, sharedEventLoopGroup!!, artemis!!, bridgeMetricsService) + val newAMQPConfig = with(amqpConfig) { AMQPConfigurationImpl(keyStore, trustStore, proxyConfig, maxMessageSize, + revocationConfig, useOpenSsl, enableSNI, sourceX500Name, trace, sslHandshakeTimeout) } + val newBridge = AMQPBridge(sourceX500Name, queueName, targets, legalNames, newAMQPConfig, sharedEventLoopGroup!!, artemis!!, + bridgeMetricsService, bridgeConnectionTTLSeconds) bridges += newBridge bridgeMetricsService?.bridgeCreated(targets, legalNames) newBridge - } - newBridge.start() + }.start() } override fun destroyBridge(queueName: String, targets: List) { @@ -246,6 +490,17 @@ class AMQPBridgeManager(config: MutualSslConfiguration, } } + fun destroyAllBridges(queueName: String): Map { + return lock.withLock { + // queueNamesToBridgesMap returns a mutable list, .toList converts it to a immutable list so it won't be changed by the [destroyBridge] method. + val bridges = queueNamesToBridgesMap[queueName]?.toList() + destroyBridge(queueName, bridges?.flatMap { it.targets } ?: emptyList()) + bridges?.map { + it.sourceX500Name to BridgeEntry(it.queueName, it.targets, it.legalNames.toList(), serviceAddress = false) + }?.toMap() ?: emptyMap() + } + } + override fun start() { sharedEventLoopGroup = NioEventLoopGroup(NUM_BRIDGE_THREADS) val artemis = artemisMessageClientFactory() diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/BridgeControlListener.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/BridgeControlListener.kt index c3c829b94a..2a37649667 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/BridgeControlListener.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/BridgeControlListener.kt @@ -1,51 +1,124 @@ +@file:Suppress("TooGenericExceptionCaught") // needs to catch and handle/rethrow *all* exceptions package net.corda.nodeapi.internal.bridging +import net.corda.core.identity.CordaX500Name import net.corda.core.serialization.SerializationDefaults import net.corda.core.serialization.deserialize import net.corda.core.serialization.serialize +import net.corda.core.utilities.NetworkHostAndPort import net.corda.core.utilities.contextLogger +import net.corda.nodeapi.internal.ArtemisMessagingClient import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.BRIDGE_CONTROL import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.BRIDGE_NOTIFY import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.P2P_PREFIX import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.PEERS_PREFIX import net.corda.nodeapi.internal.ArtemisSessionProvider +import net.corda.nodeapi.internal.config.CertificateStore import net.corda.nodeapi.internal.config.MutualSslConfiguration +import net.corda.nodeapi.internal.crypto.x509 +import net.corda.nodeapi.internal.protonwrapper.netty.ProxyConfig +import net.corda.nodeapi.internal.protonwrapper.netty.RevocationConfig +import org.apache.activemq.artemis.api.core.ActiveMQNonExistentQueueException +import org.apache.activemq.artemis.api.core.ActiveMQQueueExistsException import org.apache.activemq.artemis.api.core.RoutingType import org.apache.activemq.artemis.api.core.SimpleString import org.apache.activemq.artemis.api.core.client.ClientConsumer import org.apache.activemq.artemis.api.core.client.ClientMessage +import org.apache.activemq.artemis.api.core.client.ClientSession +import rx.Observable +import rx.subjects.PublishSubject import java.util.* -class BridgeControlListener(val config: MutualSslConfiguration, +class BridgeControlListener(private val keyStore: CertificateStore, + trustStore: CertificateStore, + useOpenSSL: Boolean, + proxyConfig: ProxyConfig? = null, maxMessageSize: Int, - crlCheckSoftFail: Boolean, + revocationConfig: RevocationConfig, + enableSNI: Boolean, private val artemisMessageClientFactory: () -> ArtemisSessionProvider, - bridgeMetricsService: BridgeMetricsService? = null) : AutoCloseable { + bridgeMetricsService: BridgeMetricsService? = null, + trace: Boolean = false, + sslHandshakeTimeout: Long? = null, + bridgeConnectionTTLSeconds: Int = 0) : AutoCloseable { private val bridgeId: String = UUID.randomUUID().toString() - private val bridgeManager: BridgeManager = AMQPBridgeManager( - config, - maxMessageSize, - crlCheckSoftFail, - artemisMessageClientFactory, - bridgeMetricsService) + private var bridgeControlQueue = "$BRIDGE_CONTROL.$bridgeId" + private var bridgeNotifyQueue = "$BRIDGE_NOTIFY.$bridgeId" private val validInboundQueues = mutableSetOf() + private val bridgeManager = if (enableSNI) { + LoopbackBridgeManager(keyStore, trustStore, useOpenSSL, proxyConfig, maxMessageSize, revocationConfig, enableSNI, + artemisMessageClientFactory, bridgeMetricsService, this::validateReceiveTopic, trace, sslHandshakeTimeout, + bridgeConnectionTTLSeconds) + } else { + AMQPBridgeManager(keyStore, trustStore, useOpenSSL, proxyConfig, maxMessageSize, revocationConfig, enableSNI, + artemisMessageClientFactory, bridgeMetricsService, trace, sslHandshakeTimeout, bridgeConnectionTTLSeconds) + } private var artemis: ArtemisSessionProvider? = null private var controlConsumer: ClientConsumer? = null + private var notifyConsumer: ClientConsumer? = null + + constructor(config: MutualSslConfiguration, + p2pAddress: NetworkHostAndPort, + maxMessageSize: Int, + revocationConfig: RevocationConfig, + enableSNI: Boolean, + proxy: ProxyConfig? = null) : this(config.keyStore.get(), config.trustStore.get(), config.useOpenSsl, proxy, maxMessageSize, revocationConfig, enableSNI, { ArtemisMessagingClient(config, p2pAddress, maxMessageSize) }) companion object { private val log = contextLogger() } + val active: Boolean + get() = validInboundQueues.isNotEmpty() + + private val _activeChange = PublishSubject.create().toSerialized() + val activeChange: Observable + get() = _activeChange + + private val _failure = PublishSubject.create().toSerialized() + val failure: Observable + get() = _failure + fun start() { - stop() - bridgeManager.start() - val artemis = artemisMessageClientFactory() - this.artemis = artemis - artemis.start() - val artemisClient = artemis.started!! - val artemisSession = artemisClient.session - val bridgeControlQueue = "$BRIDGE_CONTROL.$bridgeId" - artemisSession.createTemporaryQueue(BRIDGE_CONTROL, RoutingType.MULTICAST, bridgeControlQueue) + try { + stop() + + val queueDisambiguityId = UUID.randomUUID().toString() + bridgeControlQueue = "$BRIDGE_CONTROL.$queueDisambiguityId" + bridgeNotifyQueue = "$BRIDGE_NOTIFY.$queueDisambiguityId" + + bridgeManager.start() + val artemis = artemisMessageClientFactory() + this.artemis = artemis + artemis.start() + val artemisClient = artemis.started!! + val artemisSession = artemisClient.session + registerBridgeControlListener(artemisSession) + registerBridgeDuplicateChecker(artemisSession) + // Attempt to read available inboxes directly from Artemis before requesting updates from connected nodes + validInboundQueues.addAll(artemisSession.addressQuery(SimpleString("$P2P_PREFIX#")).queueNames.map { it.toString() }) + log.info("Found inboxes: $validInboundQueues") + if (active) { + _activeChange.onNext(true) + } + val startupMessage = BridgeControl.BridgeToNodeSnapshotRequest(bridgeId).serialize(context = SerializationDefaults.P2P_CONTEXT) + .bytes + val bridgeRequest = artemisSession.createMessage(false) + bridgeRequest.writeBodyBufferBytes(startupMessage) + artemisClient.producer.send(BRIDGE_NOTIFY, bridgeRequest) + } catch (e: Exception) { + log.error("Failure to start BridgeControlListener", e) + _failure.onNext(this) + } + } + + private fun registerBridgeControlListener(artemisSession: ClientSession) { + try { + artemisSession.createTemporaryQueue(BRIDGE_CONTROL, RoutingType.MULTICAST, bridgeControlQueue) + } catch (ex: ActiveMQQueueExistsException) { + // Ignore if there is a queue still not cleaned up + } + val control = artemisSession.createConsumer(bridgeControlQueue) controlConsumer = control control.setMessageHandler { msg -> @@ -53,22 +126,64 @@ class BridgeControlListener(val config: MutualSslConfiguration, processControlMessage(msg) } catch (ex: Exception) { log.error("Unable to process bridge control message", ex) + _failure.onNext(this) + } + msg.acknowledge() + } + } + + private fun registerBridgeDuplicateChecker(artemisSession: ClientSession) { + try { + artemisSession.createTemporaryQueue(BRIDGE_NOTIFY, RoutingType.MULTICAST, bridgeNotifyQueue) + } catch (ex: ActiveMQQueueExistsException) { + // Ignore if there is a queue still not cleaned up + } + val notify = artemisSession.createConsumer(bridgeNotifyQueue) + notifyConsumer = notify + notify.setMessageHandler { msg -> + try { + val data: ByteArray = ByteArray(msg.bodySize).apply { msg.bodyBuffer.readBytes(this) } + val notifyMessage = data.deserialize(context = SerializationDefaults.P2P_CONTEXT) + if (notifyMessage.bridgeIdentity != bridgeId) { + log.error("Fatal Error! Two bridges have been configured simultaneously! Check the enterpriseConfiguration.externalBridge status") + System.exit(1) + } + } catch (ex: Exception) { + log.error("Unable to process bridge notification message", ex) + _failure.onNext(this) } msg.acknowledge() } - val startupMessage = BridgeControl.BridgeToNodeSnapshotRequest(bridgeId).serialize(context = SerializationDefaults.P2P_CONTEXT).bytes - val bridgeRequest = artemisSession.createMessage(false) - bridgeRequest.writeBodyBufferBytes(startupMessage) - artemisClient.producer.send(BRIDGE_NOTIFY, bridgeRequest) } fun stop() { - validInboundQueues.clear() - controlConsumer?.close() - controlConsumer = null - artemis?.stop() - artemis = null - bridgeManager.stop() + try { + if (active) { + _activeChange.onNext(false) + } + validInboundQueues.clear() + controlConsumer?.close() + controlConsumer = null + notifyConsumer?.close() + notifyConsumer = null + artemis?.apply { + try { + started?.session?.deleteQueue(bridgeControlQueue) + } catch (e: ActiveMQNonExistentQueueException) { + log.warn("Queue $bridgeControlQueue does not exist and it can't be deleted") + } + try { + started?.session?.deleteQueue(bridgeNotifyQueue) + } catch (e: ActiveMQNonExistentQueueException) { + log.warn("Queue $bridgeNotifyQueue does not exist and it can't be deleted") + } + stop() + } + artemis = null + bridgeManager.stop() + } catch (e: Exception) { + log.error("Failure to stop BridgeControlListener", e) + } } override fun close() = stop() @@ -91,6 +206,10 @@ class BridgeControlListener(val config: MutualSslConfiguration, log.info("Received bridge control message $controlMessage") when (controlMessage) { is BridgeControl.NodeToBridgeSnapshot -> { + if (!isConfigured(controlMessage.nodeIdentity)) { + log.error("Fatal error! Bridge not configured with keystore for node with legal name ${controlMessage.nodeIdentity}.") + System.exit(1) + } if (!controlMessage.inboxQueues.all { validateInboxQueueName(it) }) { log.error("Invalid queue names in control message $controlMessage") return @@ -99,10 +218,20 @@ class BridgeControlListener(val config: MutualSslConfiguration, log.error("Invalid queue names in control message $controlMessage") return } - for (outQueue in controlMessage.sendQueues) { - bridgeManager.deployBridge(outQueue.queueName, outQueue.targets, outQueue.legalNames.toSet()) - } + + val wasActive = active validInboundQueues.addAll(controlMessage.inboxQueues) + for (outQueue in controlMessage.sendQueues) { + bridgeManager.deployBridge(controlMessage.nodeIdentity, outQueue.queueName, outQueue.targets, outQueue.legalNames.toSet()) + } + log.info("Added inbox: ${controlMessage.inboxQueues}. Current inboxes: $validInboundQueues.") + if (bridgeManager is LoopbackBridgeManager) { + // Notify loopback bridge manager inboxes has changed. + bridgeManager.inboxesAdded(controlMessage.inboxQueues) + } + if (!wasActive && active) { + _activeChange.onNext(true) + } } is BridgeControl.BridgeToNodeSnapshotRequest -> { log.error("Message from Bridge $controlMessage detected on wrong topic!") @@ -112,7 +241,7 @@ class BridgeControlListener(val config: MutualSslConfiguration, log.error("Invalid queue names in control message $controlMessage") return } - bridgeManager.deployBridge(controlMessage.bridgeInfo.queueName, controlMessage.bridgeInfo.targets, controlMessage.bridgeInfo.legalNames.toSet()) + bridgeManager.deployBridge(controlMessage.nodeIdentity, controlMessage.bridgeInfo.queueName, controlMessage.bridgeInfo.targets, controlMessage.bridgeInfo.legalNames.toSet()) } is BridgeControl.Delete -> { if (!controlMessage.bridgeInfo.queueName.startsWith(PEERS_PREFIX)) { @@ -121,7 +250,19 @@ class BridgeControlListener(val config: MutualSslConfiguration, } bridgeManager.destroyBridge(controlMessage.bridgeInfo.queueName, controlMessage.bridgeInfo.targets) } + is BridgeControl.BridgeHealthCheck -> { + log.warn("Not currently doing anything on BridgeHealthCheck") + return + } } } + private fun isConfigured(sourceX500Name: String): Boolean { + val keyStore = keyStore.value.internal + return keyStore.aliases().toList().any { alias -> + val x500Name = keyStore.getCertificate(alias).x509.subjectX500Principal + val cordaX500Name = CordaX500Name.build(x500Name) + cordaX500Name.toString() == sourceX500Name + } + } } \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/BridgeControlMessages.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/BridgeControlMessages.kt index 6a2f30bcd4..7c820e4770 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/BridgeControlMessages.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/BridgeControlMessages.kt @@ -11,7 +11,7 @@ import net.corda.core.utilities.NetworkHostAndPort * @property legalNames The list of acceptable [CordaX500Name] names that should be presented as subject of the validated peer TLS certificate. */ @CordaSerializable -data class BridgeEntry(val queueName: String, val targets: List, val legalNames: List) +data class BridgeEntry(val queueName: String, val targets: List, val legalNames: List, val serviceAddress: Boolean) sealed class BridgeControl { /** @@ -47,4 +47,13 @@ sealed class BridgeControl { */ @CordaSerializable data class Delete(val nodeIdentity: String, val bridgeInfo: BridgeEntry) : BridgeControl() + + /** + * This message is sent to Bridge to check the health of it. + * @property requestId The identifier for the health check request as health check is likely to be produced repeatedly. + * @property command Allows to specify the sort fo health check that needs to be performed. + * @property bridgeInfo The connection details of the new bridge (optional). + */ + @CordaSerializable + data class BridgeHealthCheck(val requestId: Long, val command: String, val bridgeInfo: BridgeEntry?) : BridgeControl() } \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/BridgeManager.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/BridgeManager.kt index 69b1509550..7bf1f150da 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/BridgeManager.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/BridgeManager.kt @@ -3,17 +3,20 @@ package net.corda.nodeapi.internal.bridging import net.corda.core.identity.CordaX500Name import net.corda.core.internal.VisibleForTesting import net.corda.core.utilities.NetworkHostAndPort +import org.apache.activemq.artemis.api.core.client.ClientMessage /** * Provides an internal interface that the [BridgeControlListener] delegates to for Bridge activities. */ @VisibleForTesting interface BridgeManager : AutoCloseable { - fun deployBridge(queueName: String, targets: List, legalNames: Set) + fun deployBridge(sourceX500Name: String, queueName: String, targets: List, legalNames: Set) fun destroyBridge(queueName: String, targets: List) fun start() fun stop() -} \ No newline at end of file +} + +fun ClientMessage.payload() = ByteArray(bodySize).apply { bodyBuffer.readBytes(this) } \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/LoopbackBridgeManager.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/LoopbackBridgeManager.kt new file mode 100644 index 0000000000..fc27029584 --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/LoopbackBridgeManager.kt @@ -0,0 +1,223 @@ +package net.corda.nodeapi.internal.bridging + +import net.corda.nodeapi.internal.ConcurrentBox +import net.corda.core.identity.CordaX500Name +import net.corda.core.internal.VisibleForTesting +import net.corda.core.utilities.NetworkHostAndPort +import net.corda.core.utilities.contextLogger +import net.corda.nodeapi.internal.ArtemisMessagingComponent +import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.NODE_P2P_USER +import net.corda.nodeapi.internal.ArtemisMessagingComponent.RemoteInboxAddress.Companion.translateInboxAddressToLocalQueue +import net.corda.nodeapi.internal.ArtemisMessagingComponent.RemoteInboxAddress.Companion.translateLocalQueueToInboxAddress +import net.corda.nodeapi.internal.ArtemisSessionProvider +import net.corda.nodeapi.internal.ArtemisConstants.MESSAGE_ID_KEY +import net.corda.nodeapi.internal.config.CertificateStore +import net.corda.nodeapi.internal.protonwrapper.messages.impl.SendableMessageImpl +import net.corda.nodeapi.internal.protonwrapper.netty.ProxyConfig +import net.corda.nodeapi.internal.protonwrapper.netty.RevocationConfig +import net.corda.nodeapi.internal.stillOpen +import org.apache.activemq.artemis.api.core.SimpleString +import org.apache.activemq.artemis.api.core.client.ActiveMQClient.DEFAULT_ACK_BATCH_SIZE +import org.apache.activemq.artemis.api.core.client.ClientConsumer +import org.apache.activemq.artemis.api.core.client.ClientMessage +import org.apache.activemq.artemis.api.core.client.ClientProducer +import org.apache.activemq.artemis.api.core.client.ClientSession +import org.slf4j.MDC + +/** + * The LoopbackBridgeManager holds the list of independent LoopbackBridge objects that actively loopback messages to local Artemis + * inboxes. + */ +@VisibleForTesting +class LoopbackBridgeManager(keyStore: CertificateStore, + trustStore: CertificateStore, + useOpenSSL: Boolean, + proxyConfig: ProxyConfig? = null, + maxMessageSize: Int, + revocationConfig: RevocationConfig, + enableSNI: Boolean, + private val artemisMessageClientFactory: () -> ArtemisSessionProvider, + private val bridgeMetricsService: BridgeMetricsService? = null, + private val isLocalInbox: (String) -> Boolean, + trace: Boolean, + sslHandshakeTimeout: Long? = null, + bridgeConnectionTTLSeconds: Int = 0) : AMQPBridgeManager(keyStore, trustStore, useOpenSSL, proxyConfig, + maxMessageSize, revocationConfig, enableSNI, + artemisMessageClientFactory, bridgeMetricsService, + trace, sslHandshakeTimeout, + bridgeConnectionTTLSeconds) { + + companion object { + private val log = contextLogger() + } + + private val queueNamesToBridgesMap = ConcurrentBox(mutableMapOf>()) + private var artemis: ArtemisSessionProvider? = null + + /** + * Each LoopbackBridge is an independent consumer of messages from the Artemis local queue per designated endpoint. + * It attempts to loopback these messages via ArtemisClient to the local inbox. + */ + private class LoopbackBridge(val sourceX500Name: String, + val queueName: String, + val targets: List, + val legalNames: Set, + artemis: ArtemisSessionProvider, + private val bridgeMetricsService: BridgeMetricsService?) { + companion object { + private val log = contextLogger() + } + + // TODO: refactor MDC support, duplicated in AMQPBridgeManager. + private fun withMDC(block: () -> Unit) { + val oldMDC = MDC.getCopyOfContextMap() + try { + MDC.put("queueName", queueName) + MDC.put("source", sourceX500Name) + MDC.put("targets", targets.joinToString(separator = ";") { it.toString() }) + MDC.put("legalNames", legalNames.joinToString(separator = ";") { it.toString() }) + MDC.put("bridgeType", "loopback") + block() + } finally { + MDC.setContextMap(oldMDC) + } + } + + private fun logDebugWithMDC(msg: () -> String) { + if (log.isDebugEnabled) { + withMDC { log.debug(msg()) } + } + } + + private fun logInfoWithMDC(msg: String) = withMDC { log.info(msg) } + + private fun logWarnWithMDC(msg: String) = withMDC { log.warn(msg) } + + private val artemis = ConcurrentBox(artemis) + private var consumerSession: ClientSession? = null + private var producerSession: ClientSession? = null + private var consumer: ClientConsumer? = null + private var producer: ClientProducer? = null + + fun start() { + logInfoWithMDC("Create new Artemis loopback bridge") + artemis.exclusive { + logInfoWithMDC("Bridge Connected") + bridgeMetricsService?.bridgeConnected(targets, legalNames) + val sessionFactory = started!!.sessionFactory + this@LoopbackBridge.consumerSession = sessionFactory.createSession(NODE_P2P_USER, NODE_P2P_USER, false, true, true, false, DEFAULT_ACK_BATCH_SIZE) + this@LoopbackBridge.producerSession = sessionFactory.createSession(NODE_P2P_USER, NODE_P2P_USER, false, true, true, false, DEFAULT_ACK_BATCH_SIZE) + // Several producers (in the case of shared bridge) can put messages in the same outbound p2p queue. The consumers are created using the source x500 name as a filter + val consumer = consumerSession!!.createConsumer(queueName, "hyphenated_props:sender-subject-name = '$sourceX500Name'") + consumer.setMessageHandler(this@LoopbackBridge::clientArtemisMessageHandler) + this@LoopbackBridge.consumer = consumer + this@LoopbackBridge.producer = producerSession!!.createProducer() + consumerSession?.start() + producerSession?.start() + } + } + + fun stop() { + logInfoWithMDC("Stopping AMQP bridge") + artemis.exclusive { + bridgeMetricsService?.bridgeDisconnected(targets, legalNames) + consumer?.apply { if (!isClosed) close() } + consumer = null + producer?.apply { if (!isClosed) close() } + producer = null + consumerSession?.apply { if (stillOpen()) stop() } + consumerSession = null + producerSession?.apply { if (stillOpen()) stop()} + producerSession = null + } + } + + private fun clientArtemisMessageHandler(artemisMessage: ClientMessage) { + logDebugWithMDC { "Loopback Send to ${legalNames.first()} uuid: ${artemisMessage.getObjectProperty(MESSAGE_ID_KEY)}" } + val peerInbox = translateLocalQueueToInboxAddress(queueName) + producer?.send(SimpleString(peerInbox), artemisMessage) { artemisMessage.individualAcknowledge() } + bridgeMetricsService?.let { metricsService -> + val properties = ArtemisMessagingComponent.Companion.P2PMessagingHeaders.whitelistedHeaders.mapNotNull { key -> + if (artemisMessage.containsProperty(key)) { + key to artemisMessage.getObjectProperty(key).let { (it as? SimpleString)?.toString() ?: it } + } else { + null + } + }.toMap() + metricsService.packetAcceptedEvent(SendableMessageImpl(artemisMessage.payload(), peerInbox, legalNames.first().toString(), targets.first(), properties)) + } + } + } + + override fun deployBridge(sourceX500Name: String, queueName: String, targets: List, legalNames: Set) { + val inboxAddress = translateLocalQueueToInboxAddress(queueName) + if (isLocalInbox(inboxAddress)) { + log.info("Deploying loopback bridge for $queueName, source $sourceX500Name") + queueNamesToBridgesMap.exclusive { + val bridges = getOrPut(queueName) { mutableListOf() } + for (target in targets) { + if (bridges.any { it.targets.contains(target) && it.sourceX500Name == sourceX500Name }) { + return + } + } + val newBridge = LoopbackBridge(sourceX500Name, queueName, targets, legalNames, artemis!!, bridgeMetricsService) + bridges += newBridge + bridgeMetricsService?.bridgeCreated(targets, legalNames) + newBridge + }.start() + } else { + log.info("Deploying AMQP bridge for $queueName, source $sourceX500Name") + super.deployBridge(sourceX500Name, queueName, targets, legalNames) + } + } + + override fun destroyBridge(queueName: String, targets: List) { + super.destroyBridge(queueName, targets) + queueNamesToBridgesMap.exclusive { + val bridges = this[queueName] ?: mutableListOf() + for (target in targets) { + val bridge = bridges.firstOrNull { it.targets.contains(target) } + if (bridge != null) { + bridges -= bridge + if (bridges.isEmpty()) { + remove(queueName) + } + bridge.stop() + bridgeMetricsService?.bridgeDestroyed(bridge.targets, bridge.legalNames) + } + } + } + } + + /** + * Remove any AMQP bridge for the local inbox and create a loopback bridge for that queue. + */ + fun inboxesAdded(inboxes: List) { + for (inbox in inboxes) { + super.destroyAllBridges(translateInboxAddressToLocalQueue(inbox)).forEach { source, bridgeEntry -> + log.info("Destroyed AMQP Bridge '${bridgeEntry.queueName}', creating Loopback bridge for local inbox.") + deployBridge(source, bridgeEntry.queueName, bridgeEntry.targets, bridgeEntry.legalNames.toSet()) + } + } + } + + override fun start() { + super.start() + val artemis = artemisMessageClientFactory() + this.artemis = artemis + artemis.start() + } + + override fun stop() = close() + + override fun close() { + super.close() + queueNamesToBridgesMap.exclusive { + for (bridge in values.flatten()) { + bridge.stop() + } + clear() + artemis?.stop() + } + } +} diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/config/MessagingServerConnectionConfiguration.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/config/MessagingServerConnectionConfiguration.kt new file mode 100644 index 0000000000..2b1bf7829a --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/config/MessagingServerConnectionConfiguration.kt @@ -0,0 +1,63 @@ +package net.corda.nodeapi.internal.config + +import net.corda.core.utilities.minutes +import net.corda.core.utilities.seconds +import java.time.Duration + +/** + * Predefined connection configurations used by Artemis clients (currently used in the P2P messaging layer). + * The enum names represent the approximate total duration of the failover (with exponential back-off). The formula used to calculate + * this duration is as follows: + * + * totalFailoverDuration = SUM(k=0 to [reconnectAttempts]) of [retryInterval] * POW([retryIntervalMultiplier], k) + * + * Example calculation for [DEFAULT]: + * + * totalFailoverDuration = 5 + 5 * 1.5 + 5 * (1.5)^2 + 5 * (1.5)^3 + 5 * (1.5)^4 = ~66 seconds + * + * @param failoverOnInitialAttempt Determines whether failover is triggered if initial connection fails. + * @param initialConnectAttempts The number of reconnect attempts if failover is enabled for initial connection. A value + * of -1 represents infinite attempts. + * @param reconnectAttempts The number of reconnect attempts for failover after initial connection is done. A value + * of -1 represents infinite attempts. + * @param retryInterval Duration between reconnect attempts. + * @param retryIntervalMultiplier Value used in the reconnection back-off process. + * @param maxRetryInterval Determines the maximum duration between reconnection attempts. Useful when using infinite retries. + */ +enum class MessagingServerConnectionConfiguration { + + DEFAULT { + override fun failoverOnInitialAttempt(isHa: Boolean) = true + override fun initialConnectAttempts(isHa: Boolean) = 5 + override fun reconnectAttempts(isHa: Boolean) = 5 + override fun retryInterval() = 5.seconds + override fun retryIntervalMultiplier() = 1.5 + override fun maxRetryInterval(isHa: Boolean) = 3.minutes + }, + + FAIL_FAST { + override fun failoverOnInitialAttempt(isHa: Boolean) = isHa + override fun initialConnectAttempts(isHa: Boolean) = 0 + // Client die too fast during failover/failback, need a few reconnect attempts to allow new master to become active + override fun reconnectAttempts(isHa: Boolean) = if (isHa) 3 else 0 + override fun retryInterval() = 5.seconds + override fun retryIntervalMultiplier() = 1.5 + override fun maxRetryInterval(isHa: Boolean) = 3.minutes + }, + + CONTINUOUS_RETRY { + override fun failoverOnInitialAttempt(isHa: Boolean) = true + override fun initialConnectAttempts(isHa: Boolean) = if (isHa) 0 else -1 + override fun reconnectAttempts(isHa: Boolean) = -1 + override fun retryInterval() = 5.seconds + override fun retryIntervalMultiplier() = 1.5 + override fun maxRetryInterval(isHa: Boolean) = if (isHa) 3.minutes else 5.minutes + }; + + abstract fun failoverOnInitialAttempt(isHa: Boolean): Boolean + abstract fun initialConnectAttempts(isHa: Boolean): Int + abstract fun reconnectAttempts(isHa: Boolean): Int + abstract fun retryInterval(): Duration + abstract fun retryIntervalMultiplier(): Double + abstract fun maxRetryInterval(isHa: Boolean): Duration +} diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/engine/ConnectionStateMachine.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/engine/ConnectionStateMachine.kt index 1f7d9f398f..0e88c263e9 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/engine/ConnectionStateMachine.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/engine/ConnectionStateMachine.kt @@ -45,7 +45,11 @@ internal class ConnectionStateMachine(private val serverMode: Boolean, userName: String?, password: String?) : BaseHandler() { companion object { - private const val IDLE_TIMEOUT = 10000 + private const val CORDA_AMQP_FRAME_SIZE_PROP_NAME = "net.corda.nodeapi.connectionstatemachine.AmqpMaxFrameSize" + private const val CORDA_AMQP_IDLE_TIMEOUT_PROP_NAME = "net.corda.nodeapi.connectionstatemachine.AmqpIdleTimeout" + + private val MAX_FRAME_SIZE = Integer.getInteger(CORDA_AMQP_FRAME_SIZE_PROP_NAME, 128 * 1024) + private val IDLE_TIMEOUT = Integer.getInteger(CORDA_AMQP_IDLE_TIMEOUT_PROP_NAME, 10 * 1000) private val log = contextLogger() } @@ -102,6 +106,7 @@ internal class ConnectionStateMachine(private val serverMode: Boolean, transport.context = connection @Suppress("UsePropertyAccessSyntax") transport.setEmitFlowEventOnSend(true) + transport.maxFrameSize = MAX_FRAME_SIZE connection.collect(collector) val sasl = transport.sasl() if (userName != null) { @@ -488,7 +493,9 @@ internal class ConnectionStateMachine(private val serverMode: Boolean, } fun transportWriteMessage(msg: SendableMessageImpl) { - msg.buf = encodePayloadBytes(msg) + val encoded = encodePayloadBytes(msg) + msg.release() + msg.buf = encoded val messageQueue = messageQueues.getOrPut(msg.topic, { LinkedList() }) messageQueue.offer(msg) if (session != null) { diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/engine/EventProcessor.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/engine/EventProcessor.kt index b91642a840..8df0aa0f37 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/engine/EventProcessor.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/engine/EventProcessor.kt @@ -38,7 +38,9 @@ internal class EventProcessor(private val channel: Channel, userName: String?, password: String?) { companion object { - private const val FLOW_WINDOW_SIZE = 10 + private const val CORDA_AMQP_FLOW_WINDOW_SIZE_PROP_NAME = "net.corda.nodeapi.eventprocessor.FlowWindowSize" + + private val FLOW_WINDOW_SIZE = Integer.getInteger(CORDA_AMQP_FLOW_WINDOW_SIZE_PROP_NAME, 5) private val log = contextLogger() } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/messages/ApplicationMessage.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/messages/ApplicationMessage.kt index 03911bb50f..18c5f3b61e 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/messages/ApplicationMessage.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/messages/ApplicationMessage.kt @@ -11,4 +11,5 @@ interface ApplicationMessage { val destinationLegalName: String val destinationLink: NetworkHostAndPort val applicationProperties: Map + fun release() } \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/messages/impl/ReceivedMessageImpl.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/messages/impl/ReceivedMessageImpl.kt index 7bf83505c5..3e95b2fb8c 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/messages/impl/ReceivedMessageImpl.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/messages/impl/ReceivedMessageImpl.kt @@ -2,6 +2,7 @@ package net.corda.nodeapi.internal.protonwrapper.messages.impl import io.netty.channel.Channel import net.corda.core.utilities.NetworkHostAndPort +import net.corda.core.utilities.contextLogger import net.corda.nodeapi.internal.protonwrapper.messages.MessageStatus import net.corda.nodeapi.internal.protonwrapper.messages.ReceivedMessage import org.apache.qpid.proton.engine.Delivery @@ -10,7 +11,7 @@ import org.apache.qpid.proton.engine.Delivery * An internal packet management class that allows tracking of asynchronous acknowledgements * that in turn send Delivery messages back to the originator. */ -internal class ReceivedMessageImpl(override val payload: ByteArray, +internal class ReceivedMessageImpl(override var payload: ByteArray, override val topic: String, override val sourceLegalName: String, override val sourceLink: NetworkHostAndPort, @@ -19,11 +20,25 @@ internal class ReceivedMessageImpl(override val payload: ByteArray, override val applicationProperties: Map, private val channel: Channel, private val delivery: Delivery) : ReceivedMessage { + companion object { + private val emptyPayload = ByteArray(0) + private val logger = contextLogger() + } + data class MessageCompleter(val status: MessageStatus, val delivery: Delivery) + override fun release() { + payload = emptyPayload + } + override fun complete(accepted: Boolean) { + release() val status = if (accepted) MessageStatus.Acknowledged else MessageStatus.Rejected - channel.writeAndFlush(MessageCompleter(status, delivery)) + if (channel.isActive) { + channel.writeAndFlush(MessageCompleter(status, delivery)) + } else { + logger.info("Not writing $status as $channel is not active") + } } override fun toString(): String = "Received ${String(payload)} $topic" diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/messages/impl/SendableMessageImpl.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/messages/impl/SendableMessageImpl.kt index 6adc9b2bbc..addff653d1 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/messages/impl/SendableMessageImpl.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/messages/impl/SendableMessageImpl.kt @@ -11,11 +11,15 @@ import net.corda.nodeapi.internal.protonwrapper.messages.SendableMessage * An internal packet management class that allows handling of the encoded buffers and * allows registration of an acknowledgement handler when the remote receiver confirms durable storage. */ -internal class SendableMessageImpl(override val payload: ByteArray, +internal class SendableMessageImpl(override var payload: ByteArray, override val topic: String, override val destinationLegalName: String, override val destinationLink: NetworkHostAndPort, override val applicationProperties: Map) : SendableMessage { + companion object { + private val emptyPayload = ByteArray(0) + } + var buf: ByteBuf? = null @Volatile var status: MessageStatus = MessageStatus.Unsent @@ -23,12 +27,14 @@ internal class SendableMessageImpl(override val payload: ByteArray, private val _onComplete = openFuture() override val onComplete: CordaFuture get() = _onComplete - fun release() { + override fun release() { + payload = emptyPayload buf?.release() buf = null } fun doComplete(status: MessageStatus) { + release() this.status = status _onComplete.set(status) } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPChannelHandler.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPChannelHandler.kt index 273731c891..904c2f9c4d 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPChannelHandler.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPChannelHandler.kt @@ -5,11 +5,16 @@ import io.netty.channel.ChannelDuplexHandler import io.netty.channel.ChannelHandlerContext import io.netty.channel.ChannelPromise import io.netty.channel.socket.SocketChannel +import io.netty.handler.proxy.ProxyConnectException +import io.netty.handler.proxy.ProxyConnectionEvent +import io.netty.handler.ssl.SniCompletionEvent import io.netty.handler.ssl.SslHandler import io.netty.handler.ssl.SslHandshakeCompletionEvent import io.netty.util.ReferenceCountUtil import net.corda.core.identity.CordaX500Name import net.corda.core.utilities.contextLogger +import net.corda.core.utilities.trace +import net.corda.nodeapi.internal.ArtemisConstants.MESSAGE_ID_KEY import net.corda.nodeapi.internal.crypto.x509 import net.corda.nodeapi.internal.protonwrapper.engine.EventProcessor import net.corda.nodeapi.internal.protonwrapper.messages.ReceivedMessage @@ -23,6 +28,8 @@ import org.slf4j.MDC import java.net.InetSocketAddress import java.nio.channels.ClosedChannelException import java.security.cert.X509Certificate +import javax.net.ssl.ExtendedSSLSession +import javax.net.ssl.SNIHostName import javax.net.ssl.SSLException /** @@ -30,23 +37,29 @@ import javax.net.ssl.SSLException * It also add some extra checks to the SSL handshake to support our non-standard certificate checks of legal identity. * When a valid SSL connections is made then it initialises a proton-j engine instance to handle the protocol layer. */ +@Suppress("TooManyFunctions") internal class AMQPChannelHandler(private val serverMode: Boolean, private val allowedRemoteLegalNames: Set?, + private val keyManagerFactoriesMap: Map, private val userName: String?, private val password: String?, private val trace: Boolean, - private val onOpen: (Pair) -> Unit, - private val onClose: (Pair) -> Unit, + private val suppressLogs: Boolean, + private val onOpen: (SocketChannel, ConnectionChange) -> Unit, + private val onClose: (SocketChannel, ConnectionChange) -> Unit, private val onReceive: (ReceivedMessage) -> Unit) : ChannelDuplexHandler() { companion object { private val log = contextLogger() + const val PROXY_LOGGER_NAME = "preProxyLogger" } private lateinit var remoteAddress: InetSocketAddress - private var localCert: X509Certificate? = null private var remoteCert: X509Certificate? = null private var eventProcessor: EventProcessor? = null + private var suppressClose: Boolean = false private var badCert: Boolean = false + private var localCert: X509Certificate? = null + private var requestedServerName: String? = null private fun withMDC(block: () -> Unit) { val oldMDC = MDC.getCopyOfContextMap() ?: emptyMap() @@ -62,39 +75,50 @@ internal class AMQPChannelHandler(private val serverMode: Boolean, } } - private fun logDebugWithMDC(msg: () -> String) { - if (log.isDebugEnabled) { - withMDC { log.debug(msg()) } + private fun logDebugWithMDC(msgFn: () -> String) { + if (!suppressLogs) { + if (log.isDebugEnabled) { + withMDC { log.debug(msgFn()) } + } + } else { + withMDC { log.trace(msgFn) } } } - private fun logInfoWithMDC(msg: String) = withMDC { log.info(msg) } + private fun logInfoWithMDC(msgFn: () -> String) { + if (!suppressLogs) { + if (log.isInfoEnabled) { + withMDC { log.info(msgFn()) } + } + } else { + withMDC { log.trace(msgFn) } + } + } - private fun logWarnWithMDC(msg: String) = withMDC { log.warn(msg) } - - private fun logErrorWithMDC(msg: String, ex: Throwable? = null) = withMDC { log.error(msg, ex) } + private fun logWarnWithMDC(msg: String) = withMDC { if (!suppressLogs) log.warn(msg) else log.trace { msg } } + private fun logErrorWithMDC(msg: String, ex: Throwable? = null) = withMDC { if (!suppressLogs) log.error(msg, ex) else log.trace(msg, ex) } override fun channelActive(ctx: ChannelHandlerContext) { val ch = ctx.channel() remoteAddress = ch.remoteAddress() as InetSocketAddress val localAddress = ch.localAddress() as InetSocketAddress - logInfoWithMDC("New client connection ${ch.id()} from $remoteAddress to $localAddress") + logInfoWithMDC { "New client connection ${ch.id()} from $remoteAddress to $localAddress" } } private fun createAMQPEngine(ctx: ChannelHandlerContext) { val ch = ctx.channel() eventProcessor = EventProcessor(ch, serverMode, localCert!!.subjectX500Principal.toString(), remoteCert!!.subjectX500Principal.toString(), userName, password) - val connection = eventProcessor!!.connection - val transport = connection.transport as ProtonJTransport if (trace) { + val connection = eventProcessor!!.connection + val transport = connection.transport as ProtonJTransport transport.protocolTracer = object : ProtocolTracer { override fun sentFrame(transportFrame: TransportFrame) { - logInfoWithMDC("${transportFrame.body}") + logInfoWithMDC { "${transportFrame.body}" } } override fun receivedFrame(transportFrame: TransportFrame) { - logInfoWithMDC("${transportFrame.body}") + logInfoWithMDC { "${transportFrame.body}" } } } } @@ -104,51 +128,60 @@ internal class AMQPChannelHandler(private val serverMode: Boolean, override fun channelInactive(ctx: ChannelHandlerContext) { val ch = ctx.channel() - logInfoWithMDC("Closed client connection ${ch.id()} from $remoteAddress to ${ch.localAddress()}") - onClose(Pair(ch as SocketChannel, ConnectionChange(remoteAddress, remoteCert, false, badCert))) + logInfoWithMDC { "Closed client connection ${ch.id()} from $remoteAddress to ${ch.localAddress()}" } + if (!suppressClose) { + onClose(ch as SocketChannel, ConnectionChange(remoteAddress, remoteCert, false, badCert)) + } eventProcessor?.close() ctx.fireChannelInactive() } override fun userEventTriggered(ctx: ChannelHandlerContext, evt: Any) { - if (evt is SslHandshakeCompletionEvent) { - if (evt.isSuccess) { - val sslHandler = ctx.pipeline().get(SslHandler::class.java) - localCert = sslHandler.engine().session.localCertificates[0].x509 - remoteCert = sslHandler.engine().session.peerCertificates[0].x509 - val remoteX500Name = try { - CordaX500Name.build(remoteCert!!.subjectX500Principal) - } catch (ex: IllegalArgumentException) { - badCert = true - logErrorWithMDC("Certificate subject not a valid CordaX500Name", ex) - ctx.close() - return + when (evt) { + is ProxyConnectionEvent -> { + if (trace) { + log.info("ProxyConnectionEvent received: $evt") + try { + ctx.pipeline().remove(PROXY_LOGGER_NAME) + } catch (ex: NoSuchElementException) { + // ignore + } } - if (allowedRemoteLegalNames != null && remoteX500Name !in allowedRemoteLegalNames) { - badCert = true - logErrorWithMDC("Provided certificate subject $remoteX500Name not in expected set $allowedRemoteLegalNames") - ctx.close() - return - } - logInfoWithMDC("Handshake completed with subject: $remoteX500Name") - createAMQPEngine(ctx) - onOpen(Pair(ctx.channel() as SocketChannel, ConnectionChange(remoteAddress, remoteCert, true, false))) - } else { - val cause = evt.cause() - // This happens when the peer node is closed during SSL establishment. - if (cause is ClosedChannelException) { - logWarnWithMDC("SSL Handshake closed early.") - } else if (cause is SSLException && cause.message == "handshake timed out") { // Sadly the exception thrown by Netty wrapper requires that we check the message. - logWarnWithMDC("SSL Handshake timed out") - } else { - badCert = true - } - logErrorWithMDC("Handshake failure ${evt.cause().message}") - if (log.isTraceEnabled) { - withMDC { log.trace("Handshake failure", evt.cause()) } - } - ctx.close() + // update address to the real target address + remoteAddress = evt.destinationAddress() } + is SniCompletionEvent -> { + if (evt.isSuccess) { + // The SniCompletionEvent is fired up before context is switched (after SslHandshakeCompletionEvent) + // so we save the requested server name now to be able log it once the handshake is completed successfully + // Note: this event is only triggered when using OpenSSL. + requestedServerName = evt.hostname() + logInfoWithMDC { "SNI completion success." } + } else { + logErrorWithMDC("SNI completion failure: ${evt.cause().message}") + } + } + is SslHandshakeCompletionEvent -> { + if (evt.isSuccess) { + handleSuccessfulHandshake(ctx) + } else { + handleFailedHandshake(ctx, evt) + } + } + } + } + + private fun SslHandler.getRequestedServerName(): String? { + return if (serverMode) { + val session = engine().session + when (session) { + // Server name can be obtained from SSL session when using JavaSSL. + is ExtendedSSLSession -> (session.requestedServerNames.firstOrNull() as? SNIHostName)?.asciiName + // For Open SSL server name is obtained from SniCompletionEvent + else -> requestedServerName + } + } else { + (engine().sslParameters?.serverNames?.firstOrNull() as? SNIHostName)?.asciiName } } @@ -158,6 +191,10 @@ internal class AMQPChannelHandler(private val serverMode: Boolean, if (log.isTraceEnabled) { withMDC { log.trace("Pipeline uncaught exception", cause) } } + if (cause is ProxyConnectException) { + log.warn("Proxy connection failed ${cause.message}") + suppressClose = true // The pipeline gets marked as active on connection to the proxy rather than to the target, which causes excess close events + } ctx.close() } @@ -176,27 +213,27 @@ internal class AMQPChannelHandler(private val serverMode: Boolean, try { try { when (msg) { - // Transfers application packet into the AMQP engine. + // Transfers application packet into the AMQP engine. is SendableMessageImpl -> { val inetAddress = InetSocketAddress(msg.destinationLink.host, msg.destinationLink.port) - logDebugWithMDC { "Message for endpoint $inetAddress , expected $remoteAddress "} + logDebugWithMDC { "Message for endpoint $inetAddress , expected $remoteAddress " } require(CordaX500Name.parse(msg.destinationLegalName) == CordaX500Name.build(remoteCert!!.subjectX500Principal)) { "Message for incorrect legal identity ${msg.destinationLegalName} expected ${remoteCert!!.subjectX500Principal}" } - logDebugWithMDC { "channel write ${msg.applicationProperties["_AMQ_DUPL_ID"]}" } + logDebugWithMDC { "channel write ${msg.applicationProperties[MESSAGE_ID_KEY]}" } eventProcessor!!.transportWriteMessage(msg) } - // A received AMQP packet has been completed and this self-posted packet will be signalled out to the - // external application. + // A received AMQP packet has been completed and this self-posted packet will be signalled out to the + // external application. is ReceivedMessage -> { onReceive(msg) } - // A general self-posted event that triggers creation of AMQP frames when required. + // A general self-posted event that triggers creation of AMQP frames when required. is Transport -> { eventProcessor!!.transportProcessOutput(ctx) } - // A self-posted event that forwards status updates for delivered packets to the application. + // A self-posted event that forwards status updates for delivered packets to the application. is ReceivedMessageImpl.MessageCompleter -> { eventProcessor!!.complete(msg) } @@ -210,4 +247,65 @@ internal class AMQPChannelHandler(private val serverMode: Boolean, } eventProcessor!!.processEventsAsync() } + + private fun handleSuccessfulHandshake(ctx: ChannelHandlerContext) { + val sslHandler = ctx.pipeline().get(SslHandler::class.java) + val sslSession = sslHandler.engine().session + // Depending on what matching method is used, getting the local certificate is done by selecting the + // appropriate keyManagerFactory + val keyManagerFactory = requestedServerName?.let { + keyManagerFactoriesMap[it] + } ?: keyManagerFactoriesMap.values.single() + + localCert = keyManagerFactory.getCurrentCertChain()?.first() + + if (localCert == null) { + log.error("SSL KeyManagerFactory failed to provide a local cert") + ctx.close() + return + } + if (sslSession.peerCertificates == null || sslSession.peerCertificates.isEmpty()) { + log.error("No peer certificates") + ctx.close() + return + } + remoteCert = sslHandler.engine().session.peerCertificates.first().x509 + val remoteX500Name = try { + CordaX500Name.build(remoteCert!!.subjectX500Principal) + } catch (ex: IllegalArgumentException) { + badCert = true + logErrorWithMDC("Certificate subject not a valid CordaX500Name", ex) + ctx.close() + return + } + if (allowedRemoteLegalNames != null && remoteX500Name !in allowedRemoteLegalNames) { + badCert = true + logErrorWithMDC("Provided certificate subject $remoteX500Name not in expected set $allowedRemoteLegalNames") + ctx.close() + return + } + + logInfoWithMDC { "Handshake completed with subject: $remoteX500Name, requested server name: ${sslHandler.getRequestedServerName()}." } + createAMQPEngine(ctx) + onOpen(ctx.channel() as SocketChannel, ConnectionChange(remoteAddress, remoteCert, connected = true, badCert = false)) + } + + private fun handleFailedHandshake(ctx: ChannelHandlerContext, evt: SslHandshakeCompletionEvent) { + val cause = evt.cause() + // This happens when the peer node is closed during SSL establishment. + when { + cause is ClosedChannelException -> logWarnWithMDC("SSL Handshake closed early.") + // Sadly the exception thrown by Netty wrapper requires that we check the message. + cause is SSLException && cause.message == "handshake timed out" -> logWarnWithMDC("SSL Handshake timed out") + cause is SSLException && (cause.message?.contains("close_notify") == true) + -> logWarnWithMDC("Received close_notify during handshake") + + else -> badCert = true + } + logWarnWithMDC("Handshake failure: ${evt.cause().message}") + if (log.isTraceEnabled) { + withMDC { log.trace("Handshake failure", evt.cause()) } + } + ctx.close() + } } \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPClient.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPClient.kt index a9f16b8e77..561119b4f5 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPClient.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPClient.kt @@ -7,24 +7,45 @@ import io.netty.channel.socket.SocketChannel import io.netty.channel.socket.nio.NioSocketChannel import io.netty.handler.logging.LogLevel import io.netty.handler.logging.LoggingHandler +import io.netty.handler.proxy.HttpProxyHandler +import io.netty.handler.proxy.Socks4ProxyHandler +import io.netty.handler.proxy.Socks5ProxyHandler +import io.netty.resolver.NoopAddressResolverGroup import io.netty.util.internal.logging.InternalLoggerFactory import io.netty.util.internal.logging.Slf4JLoggerFactory import net.corda.core.identity.CordaX500Name import net.corda.core.utilities.NetworkHostAndPort import net.corda.core.utilities.contextLogger +import net.corda.core.utilities.debug import net.corda.nodeapi.internal.protonwrapper.messages.ReceivedMessage import net.corda.nodeapi.internal.protonwrapper.messages.SendableMessage import net.corda.nodeapi.internal.protonwrapper.messages.impl.SendableMessageImpl +import net.corda.nodeapi.internal.protonwrapper.netty.AMQPChannelHandler.Companion.PROXY_LOGGER_NAME import net.corda.nodeapi.internal.requireMessageSize import rx.Observable import rx.subjects.PublishSubject import java.lang.Long.min +import java.net.InetSocketAddress import java.util.concurrent.TimeUnit import java.util.concurrent.locks.ReentrantLock import javax.net.ssl.KeyManagerFactory import javax.net.ssl.TrustManagerFactory import kotlin.concurrent.withLock +enum class ProxyVersion { + SOCKS4, + SOCKS5, + HTTP +} + +data class ProxyConfig(val version: ProxyVersion, val proxyAddress: NetworkHostAndPort, val userName: String? = null, val password: String? = null, val proxyTimeoutMS: Long? = null) { + init { + if (version == ProxyVersion.SOCKS4) { + require(password == null) { "SOCKS4 does not support a password" } + } + } +} + /** * The AMQPClient creates a connection initiator that will try to connect in a round-robin fashion * to the first open SSL socket. It will keep retrying until it is stopped. @@ -42,15 +63,18 @@ class AMQPClient(val targets: List, } val log = contextLogger() - const val MIN_RETRY_INTERVAL = 1000L - const val MAX_RETRY_INTERVAL = 60000L - const val BACKOFF_MULTIPLIER = 2L - const val NUM_CLIENT_THREADS = 2 + + private const val CORDA_AMQP_NUM_CLIENT_THREAD_PROP_NAME = "net.corda.nodeapi.amqpclient.NumClientThread" + + private const val MIN_RETRY_INTERVAL = 1000L + private const val MAX_RETRY_INTERVAL = 60000L + private const val BACKOFF_MULTIPLIER = 2L + private val NUM_CLIENT_THREADS = Integer.getInteger(CORDA_AMQP_NUM_CLIENT_THREAD_PROP_NAME, 2) } private val lock = ReentrantLock() @Volatile - private var stopping: Boolean = false + private var started: Boolean = false private var workerGroup: EventLoopGroup? = null @Volatile private var clientChannel: Channel? = null @@ -59,6 +83,13 @@ class AMQPClient(val targets: List, private var currentTarget: NetworkHostAndPort = targets.first() private var retryInterval = MIN_RETRY_INTERVAL private val badCertTargets = mutableSetOf() + @Volatile + private var amqpActive = false + @Volatile + private var amqpChannelHandler: ChannelHandler? = null + + val localAddressString: String + get() = clientChannel?.localAddress()?.toString() ?: "" private fun nextTarget() { val origIndex = targetIndex @@ -80,29 +111,31 @@ class AMQPClient(val targets: List, private val connectListener = object : ChannelFutureListener { override fun operationComplete(future: ChannelFuture) { + amqpActive = false if (!future.isSuccess) { log.info("Failed to connect to $currentTarget") - if (!stopping) { + if (started) { workerGroup?.schedule({ nextTarget() restart() }, retryInterval, TimeUnit.MILLISECONDS) } } else { - log.info("Connected to $currentTarget") // Connection established successfully clientChannel = future.channel() clientChannel?.closeFuture()?.addListener(closeListener) + log.info("Connected to $currentTarget, Local address: $localAddressString") } } } private val closeListener = ChannelFutureListener { future -> - log.info("Disconnected from $currentTarget") + log.info("Disconnected from $currentTarget, Local address: $localAddressString") future.channel()?.disconnect() clientChannel = null - if (!stopping) { + if (started && !amqpActive) { + log.debug { "Scheduling restart of $currentTarget (AMQP inactive)" } workerGroup?.schedule({ nextTarget() restart() @@ -114,42 +147,110 @@ class AMQPClient(val targets: List, private val keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()) private val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()) private val conf = parent.configuration + @Volatile + private lateinit var amqpChannelHandler: AMQPChannelHandler init { keyManagerFactory.init(conf.keyStore) - trustManagerFactory.init(initialiseTrustStoreAndEnableCrlChecking(conf.trustStore, conf.crlCheckSoftFail)) + trustManagerFactory.init(initialiseTrustStoreAndEnableCrlChecking(conf.trustStore, conf.revocationConfig)) } + @Suppress("ComplexMethod") override fun initChannel(ch: SocketChannel) { val pipeline = ch.pipeline() + val proxyConfig = conf.proxyConfig + if (proxyConfig != null) { + if (conf.trace) pipeline.addLast(PROXY_LOGGER_NAME, LoggingHandler(LogLevel.INFO)) + val proxyAddress = InetSocketAddress(proxyConfig.proxyAddress.host, proxyConfig.proxyAddress.port) + val proxy = when (conf.proxyConfig!!.version) { + ProxyVersion.SOCKS4 -> { + Socks4ProxyHandler(proxyAddress, proxyConfig.userName) + } + ProxyVersion.SOCKS5 -> { + Socks5ProxyHandler(proxyAddress, proxyConfig.userName, proxyConfig.password) + } + ProxyVersion.HTTP -> { + val httpProxyHandler = if(proxyConfig.userName == null || proxyConfig.password == null) { + HttpProxyHandler(proxyAddress) + } else { + HttpProxyHandler(proxyAddress, proxyConfig.userName, proxyConfig.password) + } + //httpProxyHandler.setConnectTimeoutMillis(3600000) // 1hr for debugging purposes + httpProxyHandler + } + } + val proxyTimeout = proxyConfig.proxyTimeoutMS + if (proxyTimeout != null) { + proxy.setConnectTimeoutMillis(proxyTimeout) + } + pipeline.addLast("Proxy", proxy) + proxy.connectFuture().addListener { + if (!it.isSuccess) { + ch.disconnect() + } + } + } + + val wrappedKeyManagerFactory = CertHoldingKeyManagerFactoryWrapper(keyManagerFactory, parent.configuration) val target = parent.currentTarget - val handler = createClientSslHelper(target, parent.allowedRemoteLegalNames, keyManagerFactory, trustManagerFactory) + val handler = if (parent.configuration.useOpenSsl) { + createClientOpenSslHandler(target, parent.allowedRemoteLegalNames, wrappedKeyManagerFactory, trustManagerFactory, ch.alloc()) + } else { + createClientSslHelper(target, parent.allowedRemoteLegalNames, wrappedKeyManagerFactory, trustManagerFactory) + } + handler.handshakeTimeoutMillis = conf.sslHandshakeTimeout pipeline.addLast("sslHandler", handler) if (conf.trace) pipeline.addLast("logger", LoggingHandler(LogLevel.INFO)) - pipeline.addLast(AMQPChannelHandler(false, + amqpChannelHandler = AMQPChannelHandler(false, parent.allowedRemoteLegalNames, + // Single entry, key can be anything. + mapOf(DEFAULT to wrappedKeyManagerFactory), conf.userName, conf.password, conf.trace, - { - parent.retryInterval = MIN_RETRY_INTERVAL // reset to fast reconnect if we connect properly - parent._onConnection.onNext(it.second) - }, - { - parent._onConnection.onNext(it.second) - if (it.second.badCert) { - log.error("Blocking future connection attempts to $target due to bad certificate on endpoint") - parent.badCertTargets += target + false, + onOpen = { _, change -> + parent.run { + amqpActive = true + retryInterval = MIN_RETRY_INTERVAL // reset to fast reconnect if we connect properly + _onConnection.onNext(change) } }, - { rcv -> parent._onReceive.onNext(rcv) })) + onClose = { _, change -> + if (parent.amqpChannelHandler == amqpChannelHandler) { + parent.run { + _onConnection.onNext(change) + if (change.badCert) { + log.error("Blocking future connection attempts to $target due to bad certificate on endpoint") + badCertTargets += target + } + + if (started && amqpActive) { + log.debug { "Scheduling restart of $currentTarget (AMQP active)" } + workerGroup?.schedule({ + nextTarget() + restart() + }, retryInterval, TimeUnit.MILLISECONDS) + } + amqpActive = false + } + } + }, + onReceive = { rcv -> parent._onReceive.onNext(rcv) }) + parent.amqpChannelHandler = amqpChannelHandler + pipeline.addLast(amqpChannelHandler) } } fun start() { lock.withLock { - log.info("connect to: $currentTarget") + if (started) { + log.info("Already connected to: $currentTarget so returning") + return + } + log.info("Connect to: $currentTarget") workerGroup = sharedThreadPool ?: NioEventLoopGroup(NUM_CLIENT_THREADS) + started = true restart() } } @@ -161,6 +262,10 @@ class AMQPClient(val targets: List, val bootstrap = Bootstrap() // TODO Needs more configuration control when we profile. e.g. to use EPOLL on Linux bootstrap.group(workerGroup).channel(NioSocketChannel::class.java).handler(ClientChannelInitializer(this)) + // Delegate DNS Resolution to the proxy side, if we are using proxy. + if (configuration.proxyConfig != null) { + bootstrap.resolver(NoopAddressResolverGroup.INSTANCE) + } currentTarget = targets[targetIndex] val clientFuture = bootstrap.connect(currentTarget.host, currentTarget.port) clientFuture.addListener(connectListener) @@ -168,21 +273,17 @@ class AMQPClient(val targets: List, fun stop() { lock.withLock { - log.info("disconnect from: $currentTarget") - stopping = true - try { - if (sharedThreadPool == null) { - workerGroup?.shutdownGracefully() - workerGroup?.terminationFuture()?.sync() - } else { - clientChannel?.close()?.sync() - } - clientChannel = null - workerGroup = null - } finally { - stopping = false + log.info("Stopping connection to: $currentTarget, Local address: $localAddressString") + started = false + if (sharedThreadPool == null) { + workerGroup?.shutdownGracefully() + workerGroup?.terminationFuture()?.sync() + } else { + clientChannel?.close()?.sync() } - log.info("stopped connection to $currentTarget") + clientChannel = null + workerGroup = null + log.info("Stopped connection to $currentTarget") } } @@ -191,7 +292,7 @@ class AMQPClient(val targets: List, val connected: Boolean get() { val channel = lock.withLock { clientChannel } - return channel?.isActive ?: false + return isChannelWritable(channel) } fun createMessage(payload: ByteArray, @@ -204,13 +305,17 @@ class AMQPClient(val targets: List, fun write(msg: SendableMessage) { val channel = clientChannel - if (channel == null) { + if (channel == null || !isChannelWritable(channel)) { throw IllegalStateException("Connection to $targets not active") } else { channel.writeAndFlush(msg) } } + private fun isChannelWritable(channel: Channel?): Boolean { + return channel?.let { channel.isOpen && channel.isActive && amqpActive } ?: false + } + private val _onReceive = PublishSubject.create().toSerialized() val onReceive: Observable get() = _onReceive diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPConfiguration.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPConfiguration.kt index 3b7289a8c5..db0dd8023c 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPConfiguration.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPConfiguration.kt @@ -2,7 +2,7 @@ package net.corda.nodeapi.internal.protonwrapper.netty import net.corda.nodeapi.internal.ArtemisMessagingComponent import net.corda.nodeapi.internal.config.CertificateStore -import java.security.KeyStore +import net.corda.nodeapi.internal.config.DEFAULT_SSL_HANDSHAKE_TIMEOUT_MILLIS interface AMQPConfiguration { /** @@ -32,12 +32,11 @@ interface AMQPConfiguration { val trustStore: CertificateStore /** - * Setting crlCheckSoftFail to true allows certificate paths where some leaf certificates do not contain cRLDistributionPoints - * and also allows validation to continue if the CRL distribution server is not contactable. + * Control how CRL check will be performed. */ @JvmDefault - val crlCheckSoftFail: Boolean - get() = true + val revocationConfig: RevocationConfig + get() = RevocationConfigImpl(RevocationConfig.Mode.SOFT_FAIL) /** * Enables full debug tracing of all netty and AMQP level packets. This logs aat very high volume and is only for developers. @@ -51,5 +50,41 @@ interface AMQPConfiguration { * but currently that is deferred to Artemis and the bridge code. */ val maxMessageSize: Int + + @JvmDefault + val proxyConfig: ProxyConfig? + get() = null + + @JvmDefault + val sourceX500Name: String? + get() = null + + /** + * Whether to use the tcnative open/boring SSL provider or the default Java SSL provider + */ + @JvmDefault + val useOpenSsl: Boolean + get() = false + + @JvmDefault + val sslHandshakeTimeout: Long + get() = DEFAULT_SSL_HANDSHAKE_TIMEOUT_MILLIS // Aligned with sun.security.provider.certpath.URICertStore.DEFAULT_CRL_CONNECT_TIMEOUT + + /** + * An optional Health Check Phrase which if passed through the channel will cause AMQP Server to echo it back instead of doing normal pipeline processing + */ + val healthCheckPhrase: String? + get() = null + + /** + * An optional set of IPv4/IPv6 remote address strings which will be compared to the remote address of inbound connections and these will only log at TRACE level + */ + @JvmDefault + val silencedIPs: Set + get() = emptySet() + + @JvmDefault + val enableSNI: Boolean + get() = true } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPServer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPServer.kt index 56c8b8bfda..20834a2041 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPServer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPServer.kt @@ -2,6 +2,7 @@ package net.corda.nodeapi.internal.protonwrapper.netty import io.netty.bootstrap.ServerBootstrap import io.netty.channel.Channel +import io.netty.channel.ChannelHandler import io.netty.channel.ChannelInitializer import io.netty.channel.ChannelOption import io.netty.channel.EventLoopGroup @@ -14,6 +15,7 @@ import io.netty.util.internal.logging.InternalLoggerFactory import io.netty.util.internal.logging.Slf4JLoggerFactory import net.corda.core.utilities.NetworkHostAndPort import net.corda.core.utilities.contextLogger +import net.corda.core.utilities.debug import net.corda.nodeapi.internal.protonwrapper.messages.ReceivedMessage import net.corda.nodeapi.internal.protonwrapper.messages.SendableMessage import net.corda.nodeapi.internal.protonwrapper.messages.impl.SendableMessageImpl @@ -31,7 +33,6 @@ import kotlin.concurrent.withLock /** * This create a socket acceptor instance that can receive possibly multiple AMQP connections. - * As of now this is not used outside of testing, but in future it will be used for standalone bridging components. */ class AMQPServer(val hostName: String, val port: Int, @@ -42,8 +43,10 @@ class AMQPServer(val hostName: String, InternalLoggerFactory.setDefaultFactory(Slf4JLoggerFactory.INSTANCE) } + private const val CORDA_AMQP_NUM_SERVER_THREAD_PROP_NAME = "net.corda.nodeapi.amqpserver.NumServerThreads" + private val log = contextLogger() - const val NUM_SERVER_THREADS = 4 + private val NUM_SERVER_THREADS = Integer.getInteger(CORDA_AMQP_NUM_SERVER_THREAD_PROP_NAME, 4) } private val lock = ReentrantLock() @@ -60,29 +63,59 @@ class AMQPServer(val hostName: String, private val conf = parent.configuration init { - keyManagerFactory.init(conf.keyStore) - trustManagerFactory.init(initialiseTrustStoreAndEnableCrlChecking(conf.trustStore, conf.crlCheckSoftFail)) + keyManagerFactory.init(conf.keyStore.value.internal, conf.keyStore.entryPassword.toCharArray()) + trustManagerFactory.init(initialiseTrustStoreAndEnableCrlChecking(conf.trustStore, conf.revocationConfig)) } override fun initChannel(ch: SocketChannel) { + val amqpConfiguration = parent.configuration val pipeline = ch.pipeline() - val handler = createServerSslHelper(keyManagerFactory, trustManagerFactory) - pipeline.addLast("sslHandler", handler) + amqpConfiguration.healthCheckPhrase?.let { pipeline.addLast(ModeSelectingChannel.NAME, ModeSelectingChannel(it)) } + val (sslHandler, keyManagerFactoriesMap) = createSSLHandler(amqpConfiguration, ch) + pipeline.addLast("sslHandler", sslHandler) if (conf.trace) pipeline.addLast("logger", LoggingHandler(LogLevel.INFO)) + val suppressLogs = ch.remoteAddress()?.hostString in amqpConfiguration.silencedIPs pipeline.addLast(AMQPChannelHandler(true, null, + // Passing a mapping of legal names to key managers to be able to pick the correct one after + // SNI completion event is fired up. + keyManagerFactoriesMap, conf.userName, conf.password, conf.trace, - { - parent.clientChannels[it.first.remoteAddress()] = it.first - parent._onConnection.onNext(it.second) + suppressLogs, + onOpen = { channel, change -> + parent.run { + clientChannels[channel.remoteAddress()] = channel + _onConnection.onNext(change) + } }, - { - parent.clientChannels.remove(it.first.remoteAddress()) - parent._onConnection.onNext(it.second) + onClose = { channel, change -> + parent.run { + val remoteAddress = channel.remoteAddress() + clientChannels.remove(remoteAddress) + _onConnection.onNext(change) + } }, - { rcv -> parent._onReceive.onNext(rcv) })) + onReceive = { rcv -> parent._onReceive.onNext(rcv) })) + } + + private fun createSSLHandler(amqpConfig: AMQPConfiguration, ch: SocketChannel): Pair> { + return if (amqpConfig.useOpenSsl && amqpConfig.enableSNI && amqpConfig.keyStore.aliases().size > 1) { + val keyManagerFactoriesMap = splitKeystore(amqpConfig) + // SNI matching needed only when multiple nodes exist behind the server. + Pair(createServerSNIOpenSslHandler(keyManagerFactoriesMap, trustManagerFactory), keyManagerFactoriesMap) + } else { + val keyManagerFactory = CertHoldingKeyManagerFactoryWrapper(keyManagerFactory, amqpConfig) + val handler = if (amqpConfig.useOpenSsl) { + createServerOpenSslHandler(keyManagerFactory, trustManagerFactory, ch.alloc()) + } else { + // For javaSSL, SNI matching is handled at key manager level. + createServerSslHandler(amqpConfig.keyStore, keyManagerFactory, trustManagerFactory) + } + handler.handshakeTimeoutMillis = amqpConfig.sslHandshakeTimeout + Pair(handler, mapOf(DEFAULT to keyManagerFactory)) + } } } @@ -95,7 +128,10 @@ class AMQPServer(val hostName: String, val server = ServerBootstrap() // TODO Needs more configuration control when we profile. e.g. to use EPOLL on Linux - server.group(bossGroup, workerGroup).channel(NioServerSocketChannel::class.java).option(ChannelOption.SO_BACKLOG, 100).handler(LoggingHandler(LogLevel.INFO)).childHandler(ServerChannelInitializer(this)) + server.group(bossGroup, workerGroup).channel(NioServerSocketChannel::class.java) + .option(ChannelOption.SO_BACKLOG, 100) + .handler(NettyServerEventLogger(LogLevel.INFO, configuration.silencedIPs)) + .childHandler(ServerChannelInitializer(this)) log.info("Try to bind $port") val channelFuture = server.bind(hostName, port).sync() // block/throw here as better to know we failed to claim port than carry on @@ -144,7 +180,7 @@ class AMQPServer(val hostName: String, requireMessageSize(payload.size, configuration.maxMessageSize) val dest = InetSocketAddress(destinationLink.host, destinationLink.port) require(dest in clientChannels.keys) { - "Destination not available" + "Destination $dest is not available" } return SendableMessageImpl(payload, topic, destinationLegalName, destinationLink, properties) } @@ -155,21 +191,22 @@ class AMQPServer(val hostName: String, if (channel == null) { throw IllegalStateException("Connection to ${msg.destinationLink} not active") } else { + log.debug { "Writing message with payload of size ${msg.payload.size} into channel $channel" } channel.writeAndFlush(msg) + log.debug { "Done writing message with payload of size ${msg.payload.size} into channel $channel" } } } fun dropConnection(connectionRemoteHost: InetSocketAddress) { - val channel = clientChannels[connectionRemoteHost] - if (channel != null) { - channel.close() - } + clientChannels[connectionRemoteHost]?.close() } fun complete(delivery: Delivery, target: InetSocketAddress) { val channel = clientChannels[target] channel?.apply { + log.debug { "Writing delivery $delivery into channel $channel" } writeAndFlush(delivery) + log.debug { "Done writing delivery $delivery into channel $channel" } } } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AliasProvidingKeyMangerWrapper.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AliasProvidingKeyMangerWrapper.kt new file mode 100644 index 0000000000..4f44869212 --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AliasProvidingKeyMangerWrapper.kt @@ -0,0 +1,60 @@ +package net.corda.nodeapi.internal.protonwrapper.netty + +import java.net.Socket +import java.security.Principal +import javax.net.ssl.SSLEngine +import javax.net.ssl.X509ExtendedKeyManager +import javax.net.ssl.X509KeyManager + +interface AliasProvidingKeyMangerWrapper : X509KeyManager { + var lastAlias: String? +} + + +class AliasProvidingKeyMangerWrapperImpl(private val keyManager: X509KeyManager) : AliasProvidingKeyMangerWrapper, X509KeyManager by keyManager { + override var lastAlias: String? = null + + override fun chooseServerAlias(keyType: String?, issuers: Array?, socket: Socket?): String? { + return storeIfNotNull { keyManager.chooseServerAlias(keyType, issuers, socket) } + } + + override fun chooseClientAlias(keyType: Array?, issuers: Array?, socket: Socket?): String? { + return storeIfNotNull { keyManager.chooseClientAlias(keyType, issuers, socket) } + } + + private fun storeIfNotNull(func: () -> String?): String? { + val alias = func() + if (alias != null) { + lastAlias = alias + } + return alias + } +} + +class AliasProvidingExtendedKeyMangerWrapper(private val keyManager: X509ExtendedKeyManager) : X509ExtendedKeyManager(), X509KeyManager by keyManager, AliasProvidingKeyMangerWrapper { + override var lastAlias: String? = null + + override fun chooseServerAlias(keyType: String?, issuers: Array?, socket: Socket?): String? { + return storeIfNotNull { keyManager.chooseServerAlias(keyType, issuers, socket) } + } + + override fun chooseClientAlias(keyType: Array?, issuers: Array?, socket: Socket?): String? { + return storeIfNotNull { keyManager.chooseClientAlias(keyType, issuers, socket) } + } + + override fun chooseEngineClientAlias(keyType: Array?, issuers: Array?, engine: SSLEngine?): String? { + return storeIfNotNull { keyManager.chooseEngineClientAlias(keyType, issuers, engine) } + } + + override fun chooseEngineServerAlias(keyType: String?, issuers: Array?, engine: SSLEngine?): String? { + return storeIfNotNull { keyManager.chooseEngineServerAlias(keyType, issuers, engine) } + } + + private fun storeIfNotNull(func: () -> String?): String? { + val alias = func() + if (alias != null) { + lastAlias = alias + } + return alias + } +} \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AllowAllRevocationChecker.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AllowAllRevocationChecker.kt new file mode 100644 index 0000000000..30e0445689 --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AllowAllRevocationChecker.kt @@ -0,0 +1,34 @@ +package net.corda.nodeapi.internal.protonwrapper.netty + +import net.corda.core.utilities.debug +import org.slf4j.LoggerFactory +import java.security.cert.CertPathValidatorException +import java.security.cert.Certificate +import java.security.cert.PKIXRevocationChecker +import java.util.* + +object AllowAllRevocationChecker : PKIXRevocationChecker() { + + private val logger = LoggerFactory.getLogger(AllowAllRevocationChecker::class.java) + + override fun check(cert: Certificate?, unresolvedCritExts: MutableCollection?) { + logger.debug {"Passing certificate check for: $cert"} + // Nothing to do + } + + override fun isForwardCheckingSupported(): Boolean { + return true + } + + override fun getSupportedExtensions(): MutableSet? { + return null + } + + override fun init(forward: Boolean) { + // Nothing to do + } + + override fun getSoftFailExceptions(): MutableList { + return LinkedList() + } +} \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/CertHoldingKeyManagerFactoryWrapper.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/CertHoldingKeyManagerFactoryWrapper.kt new file mode 100644 index 0000000000..752b249a71 --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/CertHoldingKeyManagerFactoryWrapper.kt @@ -0,0 +1,81 @@ +package net.corda.nodeapi.internal.protonwrapper.netty + +import java.security.KeyStore +import java.security.cert.X509Certificate +import javax.net.ssl.KeyManager +import javax.net.ssl.KeyManagerFactory +import javax.net.ssl.KeyManagerFactorySpi +import javax.net.ssl.ManagerFactoryParameters +import javax.net.ssl.X509ExtendedKeyManager +import javax.net.ssl.X509KeyManager + +class CertHoldingKeyManagerFactorySpiWrapper(private val factorySpi: KeyManagerFactorySpi, private val amqpConfig: AMQPConfiguration) : KeyManagerFactorySpi() { + override fun engineInit(keyStore: KeyStore?, password: CharArray?) { + val engineInitMethod = KeyManagerFactorySpi::class.java.getDeclaredMethod("engineInit", KeyStore::class.java, CharArray::class.java) + engineInitMethod.isAccessible = true + engineInitMethod.invoke(factorySpi, keyStore, password) + } + + override fun engineInit(spec: ManagerFactoryParameters?) { + val engineInitMethod = KeyManagerFactorySpi::class.java.getDeclaredMethod("engineInit", ManagerFactoryParameters::class.java) + engineInitMethod.isAccessible = true + engineInitMethod.invoke(factorySpi, spec) + } + + private fun getKeyManagersImpl(): Array { + val engineGetKeyManagersMethod = KeyManagerFactorySpi::class.java.getDeclaredMethod("engineGetKeyManagers") + engineGetKeyManagersMethod.isAccessible = true + @Suppress("UNCHECKED_CAST") + val keyManagers = engineGetKeyManagersMethod.invoke(factorySpi) as Array + return if (factorySpi is CertHoldingKeyManagerFactorySpiWrapper) keyManagers else keyManagers.map { + val aliasProvidingKeyManager = getDefaultKeyManager(it) + // Use the SNIKeyManager if keystore has several entries and only for clients and non-openSSL servers. + // Condition of using SNIKeyManager: if its client, or JDKSsl server. + val isClient = amqpConfig.sourceX500Name != null + val enableSNI = amqpConfig.enableSNI && amqpConfig.keyStore.aliases().size > 1 + if (enableSNI && (isClient || !amqpConfig.useOpenSsl)) { + SNIKeyManager(aliasProvidingKeyManager as X509ExtendedKeyManager, amqpConfig) + } else { + aliasProvidingKeyManager + } + }.toTypedArray() + } + + private fun getDefaultKeyManager(keyManager: KeyManager): KeyManager { + return when (keyManager) { + is X509ExtendedKeyManager -> AliasProvidingExtendedKeyMangerWrapper(keyManager) + is X509KeyManager -> AliasProvidingKeyMangerWrapperImpl(keyManager) + else -> throw UnsupportedOperationException("Supported key manager types are: X509ExtendedKeyManager, X509KeyManager. Provided ${keyManager::class.java.name}") + } + } + + private val keyManagers = lazy { getKeyManagersImpl() } + + override fun engineGetKeyManagers(): Array { + return keyManagers.value + } +} + +/** + * You can wrap a key manager factory in this class if you need to get the cert chain currently used to identify or + * verify. When using for TLS channels, make sure to wrap the (singleton) factory separately on each channel, as + * the wrapper is not thread safe as in it will return the last used alias/cert chain and has itself no notion + * of belonging to a certain channel. + */ +class CertHoldingKeyManagerFactoryWrapper(factory: KeyManagerFactory, amqpConfig: AMQPConfiguration) : KeyManagerFactory(getFactorySpi(factory, amqpConfig), factory.provider, factory.algorithm) { + companion object { + private fun getFactorySpi(factory: KeyManagerFactory, amqpConfig: AMQPConfiguration): KeyManagerFactorySpi { + val spiField = KeyManagerFactory::class.java.getDeclaredField("factorySpi") + spiField.isAccessible = true + return CertHoldingKeyManagerFactorySpiWrapper(spiField.get(factory) as KeyManagerFactorySpi, amqpConfig) + } + } + + fun getCurrentCertChain(): Array? { + val keyManager = keyManagers.firstOrNull() + val alias = if (keyManager is AliasProvidingKeyMangerWrapper) keyManager.lastAlias else null + return if (alias != null && keyManager is X509KeyManager) { + keyManager.getCertificateChain(alias) + } else null + } +} \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/ExternalCrlSource.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/ExternalCrlSource.kt new file mode 100644 index 0000000000..654ead24a0 --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/ExternalCrlSource.kt @@ -0,0 +1,12 @@ +package net.corda.nodeapi.internal.protonwrapper.netty + +import java.security.cert.X509CRL +import java.security.cert.X509Certificate + +interface ExternalCrlSource { + + /** + * Given certificate provides a set of CRLs, potentially performing remote communication. + */ + fun fetch(certificate: X509Certificate) : Set +} \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/ModeSelectingChannel.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/ModeSelectingChannel.kt new file mode 100644 index 0000000000..1966309238 --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/ModeSelectingChannel.kt @@ -0,0 +1,76 @@ +package net.corda.nodeapi.internal.protonwrapper.netty + +import io.netty.buffer.ByteBuf +import io.netty.buffer.Unpooled +import io.netty.channel.ChannelHandlerContext +import io.netty.handler.codec.ByteToMessageDecoder +import io.netty.handler.ssl.SslHandler +import net.corda.core.utilities.contextLogger + +/** + * Responsible for deciding whether we are likely to be processing health probe request + * or this is a normal SSL/AMQP processing pipeline + */ +internal class ModeSelectingChannel(healthCheckPhrase: String) : ByteToMessageDecoder() { + + companion object { + const val NAME = "modeSelector" + private val log = contextLogger() + } + + private enum class TriState { + UNDECIDED, + ECHO_MODE, + NORMAL_MODE + } + + private val healthCheckPhraseArray = healthCheckPhrase.toByteArray(Charsets.UTF_8) + + private var currentMode = TriState.UNDECIDED + + private var alreadyEchoedPos = 0 + + override fun decode(ctx: ChannelHandlerContext, inByteBuf: ByteBuf, out: MutableList?) { + + fun ChannelHandlerContext.echoBack(inByteBuf: ByteBuf) { + + // WriteAndFlush() will decrement count and will blow unless we retain first + // And we have to ensure we are not sending the same information multiple times + val toBeWritten = inByteBuf.retainedSlice(alreadyEchoedPos, inByteBuf.readableBytes() - alreadyEchoedPos) + + writeAndFlush(toBeWritten) + + alreadyEchoedPos = inByteBuf.readableBytes() + } + + if(currentMode == TriState.ECHO_MODE) { + ctx.echoBack(inByteBuf) + return + } + + // Wait until the length prefix is available. + if (inByteBuf.readableBytes() < healthCheckPhraseArray.size) { + return + } + + // Direct buffers do not allow calling `.array()` on them, see `io.netty.buffer.UnpooledDirectByteBuf.array` + val incomingArray = Unpooled.copiedBuffer(inByteBuf).array() + val zipped = healthCheckPhraseArray.zip(incomingArray) + if (zipped.all { it.first == it.second }) { + // Matched the healthCheckPhrase + currentMode = TriState.ECHO_MODE + log.info("Echo mode activated for connection ${ctx.channel().id()}") + // Cancel scheduled action to avoid SSL handshake timeout, which starts "ticking" upon connection is established, + // namely upon call to `io.netty.handler.ssl.SslHandler#handlerAdded` is made + ctx.pipeline().get(SslHandler::class.java)?.handshakeFuture()?.cancel(false) + ctx.echoBack(inByteBuf) + } else { + currentMode = TriState.NORMAL_MODE + // Remove self from pipeline and replay all the messages received down the pipeline + // It is important to bump-up reference count as pipeline removal decrements it by one. + inByteBuf.retain() + ctx.pipeline().remove(this) + ctx.fireChannelRead(inByteBuf) + } + } +} \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/NettyServerEventLogger.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/NettyServerEventLogger.kt new file mode 100644 index 0000000000..d90a2255e2 --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/NettyServerEventLogger.kt @@ -0,0 +1,73 @@ +package net.corda.nodeapi.internal.protonwrapper.netty + +import io.netty.channel.ChannelDuplexHandler +import io.netty.channel.ChannelHandler +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelPromise +import io.netty.handler.logging.LogLevel +import io.netty.util.internal.logging.InternalLogLevel +import io.netty.util.internal.logging.InternalLogger +import io.netty.util.internal.logging.InternalLoggerFactory +import java.net.SocketAddress + +@ChannelHandler.Sharable +class NettyServerEventLogger(level: LogLevel = DEFAULT_LEVEL, val silencedIPs: Set = emptySet()) : ChannelDuplexHandler() { + companion object { + val DEFAULT_LEVEL: LogLevel = LogLevel.DEBUG + } + + private val logger: InternalLogger = InternalLoggerFactory.getInstance(javaClass) + private val internalLevel: InternalLogLevel = level.toInternalLevel() + + @Throws(Exception::class) + override fun channelActive(ctx: ChannelHandlerContext) { + if (logger.isEnabled(internalLevel)) { + logger.log(internalLevel, "Server socket ${ctx.channel()} ACTIVE") + } + ctx.fireChannelActive() + } + + @Throws(Exception::class) + override fun channelInactive(ctx: ChannelHandlerContext) { + if (logger.isEnabled(internalLevel)) { + logger.log(internalLevel, "Server socket ${ctx.channel()} INACTIVE") + } + ctx.fireChannelInactive() + } + + @Suppress("OverridingDeprecatedMember") + @Throws(Exception::class) + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + if (logger.isEnabled(internalLevel)) { + logger.log(internalLevel, "Server socket ${ctx.channel()} EXCEPTION ${cause.message}", cause) + } + ctx.fireExceptionCaught(cause) + } + + @Throws(Exception::class) + override fun bind(ctx: ChannelHandlerContext, localAddress: SocketAddress, promise: ChannelPromise) { + if (logger.isEnabled(internalLevel)) { + logger.log(internalLevel, "Server socket ${ctx.channel()} BIND $localAddress") + } + ctx.bind(localAddress, promise) + } + + @Throws(Exception::class) + override fun close(ctx: ChannelHandlerContext, promise: ChannelPromise) { + if (logger.isEnabled(internalLevel)) { + logger.log(internalLevel, "Server socket ${ctx.channel()} CLOSE") + } + ctx.close(promise) + } + + @Throws(Exception::class) + override fun channelRead(ctx: ChannelHandlerContext, msg: Any) { + val level = if (msg is io.netty.channel.socket.SocketChannel) { // Should always be the case as this is a server socket, but be defensive + if (msg.remoteAddress()?.hostString !in silencedIPs) internalLevel else InternalLogLevel.TRACE + } else internalLevel + if (logger.isEnabled(level)) { + logger.log(level, "Server socket ${ctx.channel()} ACCEPTED $msg") + } + ctx.fireChannelRead(msg) + } +} \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/RevocationConfig.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/RevocationConfig.kt new file mode 100644 index 0000000000..87535f37aa --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/RevocationConfig.kt @@ -0,0 +1,83 @@ +package net.corda.nodeapi.internal.protonwrapper.netty + +import com.typesafe.config.Config +import net.corda.nodeapi.internal.config.ConfigParser +import net.corda.nodeapi.internal.config.CustomConfigParser + +/** + * Data structure for controlling the way how Certificate Revocation Lists are handled. + */ +@CustomConfigParser(RevocationConfigParser::class) +interface RevocationConfig { + + enum class Mode { + + /** + * @see java.security.cert.PKIXRevocationChecker.Option.SOFT_FAIL + */ + SOFT_FAIL, + + /** + * Opposite of SOFT_FAIL - i.e. most rigorous check. + * Among other things, this check requires that CRL checking URL is available on every level of certificate chain. + * This is also known as Strict mode. + */ + HARD_FAIL, + + /** + * CRLs are obtained from external source + * @see ExternalCrlSource + */ + EXTERNAL_SOURCE, + + /** + * Switch CRL check off. + */ + OFF + } + + val mode: Mode + + /** + * Optional `ExternalCrlSource` which only makes sense with `mode` = `EXTERNAL_SOURCE` + */ + val externalCrlSource: ExternalCrlSource? + + /** + * Creates a copy of `RevocationConfig` with ExternalCrlSource enriched + */ + fun enrichExternalCrlSource(sourceFunc: (() -> ExternalCrlSource)?): RevocationConfig +} + +/** + * Maintained for legacy purposes to convert old style `crlCheckSoftFail`. + */ +fun Boolean.toRevocationConfig() = if(this) RevocationConfigImpl(RevocationConfig.Mode.SOFT_FAIL) else RevocationConfigImpl(RevocationConfig.Mode.HARD_FAIL) + +data class RevocationConfigImpl(override val mode: RevocationConfig.Mode, override val externalCrlSource: ExternalCrlSource? = null) : RevocationConfig { + override fun enrichExternalCrlSource(sourceFunc: (() -> ExternalCrlSource)?): RevocationConfig { + return if(mode != RevocationConfig.Mode.EXTERNAL_SOURCE) { + this + } else { + assert(sourceFunc != null) { "There should be a way to obtain ExternalCrlSource" } + copy(externalCrlSource = sourceFunc!!()) + } + } +} + +class RevocationConfigParser : ConfigParser { + override fun parse(config: Config): RevocationConfig { + val oneAndTheOnly = "mode" + val allKeys = config.entrySet().map { it.key } + require(allKeys.size == 1 && allKeys.contains(oneAndTheOnly)) {"For RevocationConfig, it is expected to have '$oneAndTheOnly' property only. " + + "Actual set of properties: $allKeys. Please check 'revocationConfig' section."} + val mode = config.getString(oneAndTheOnly) + return when (mode.toUpperCase()) { + "SOFT_FAIL" -> RevocationConfigImpl(RevocationConfig.Mode.SOFT_FAIL) + "HARD_FAIL" -> RevocationConfigImpl(RevocationConfig.Mode.HARD_FAIL) + "EXTERNAL_SOURCE" -> RevocationConfigImpl(RevocationConfig.Mode.EXTERNAL_SOURCE, null) // null for now till `enrichExternalCrlSource` is called + "OFF" -> RevocationConfigImpl(RevocationConfig.Mode.OFF) + else -> throw IllegalArgumentException("Unsupported mode : '$mode'") + } + } +} \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SNIKeyManager.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SNIKeyManager.kt new file mode 100644 index 0000000000..e28451fc44 --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SNIKeyManager.kt @@ -0,0 +1,112 @@ +package net.corda.nodeapi.internal.protonwrapper.netty + +import net.corda.core.identity.CordaX500Name +import net.corda.core.utilities.contextLogger +import net.corda.nodeapi.internal.config.CertificateStore +import net.corda.nodeapi.internal.crypto.x509 +import org.slf4j.MDC +import java.net.Socket +import java.security.Principal +import javax.net.ssl.SNIMatcher +import javax.net.ssl.SSLEngine +import javax.net.ssl.SSLSocket +import javax.net.ssl.X509ExtendedKeyManager +import javax.net.ssl.X509KeyManager + +internal class SNIKeyManager(private val keyManager: X509ExtendedKeyManager, private val amqpConfig: AMQPConfiguration) : X509ExtendedKeyManager(), X509KeyManager by keyManager, AliasProvidingKeyMangerWrapper { + + companion object { + private val log = contextLogger() + } + + override var lastAlias: String? = null + + private fun withMDC(block: () -> Unit) { + val oldMDC = MDC.getCopyOfContextMap() + try { + MDC.put("lastAlias", lastAlias) + MDC.put("isServer", amqpConfig.sourceX500Name.isNullOrEmpty().toString()) + MDC.put("sourceX500Name", amqpConfig.sourceX500Name) + MDC.put("useOpenSSL", amqpConfig.useOpenSsl.toString()) + block() + } finally { + MDC.setContextMap(oldMDC) + } + } + + private fun logDebugWithMDC(msg: () -> String) { + if (log.isDebugEnabled) { + withMDC { log.debug(msg()) } + } + } + + override fun chooseClientAlias(keyType: Array, issuers: Array, socket: Socket): String? { + return storeIfNotNull { chooseClientAlias(amqpConfig.keyStore, amqpConfig.sourceX500Name) } + } + + override fun chooseEngineClientAlias(keyType: Array, issuers: Array, engine: SSLEngine): String? { + return storeIfNotNull { chooseClientAlias(amqpConfig.keyStore, amqpConfig.sourceX500Name) } + } + + override fun chooseServerAlias(keyType: String?, issuers: Array?, socket: Socket): String? { + return storeIfNotNull { + val matcher = (socket as SSLSocket).sslParameters.sniMatchers.first() + chooseServerAlias(keyType, issuers, matcher) + } + } + + override fun chooseEngineServerAlias(keyType: String?, issuers: Array?, engine: SSLEngine?): String? { + return storeIfNotNull { + val matcher = engine?.sslParameters?.sniMatchers?.first() + chooseServerAlias(keyType, issuers, matcher) + } + } + + private fun chooseServerAlias(keyType: String?, issuers: Array?, matcher: SNIMatcher?): String? { + val aliases = keyManager.getServerAliases(keyType, issuers) + if (aliases == null || aliases.isEmpty()) { + logDebugWithMDC { "Keystore doesn't contain any aliases for key type $keyType and issuers $issuers." } + return null + } + + log.debug("Checking aliases: $aliases.") + matcher?.let { + val matchedAlias = (it as ServerSNIMatcher).matchedAlias + if (aliases.contains(matchedAlias)) { + logDebugWithMDC { "Found match for $matchedAlias." } + return matchedAlias + } + } + + logDebugWithMDC { "Unable to find a matching alias." } + return null + } + + private fun chooseClientAlias(keyStore: CertificateStore, clientLegalName: String?): String? { + clientLegalName?.let { + val aliases = keyStore.aliases() + if (aliases.isEmpty()) { + logDebugWithMDC { "Keystore doesn't contain any entries." } + } + aliases.forEach { alias -> + val x500Name = keyStore[alias].x509.subjectX500Principal + val aliasCordaX500Name = CordaX500Name.build(x500Name) + val clientCordaX500Name = CordaX500Name.parse(it) + if (clientCordaX500Name == aliasCordaX500Name) { + logDebugWithMDC { "Found alias $alias for $clientCordaX500Name." } + return alias + } + } + } + + return null + } + + private fun storeIfNotNull(func: () -> String?): String? { + val alias = func() + if (alias != null) { + lastAlias = alias + } + return alias + } +} diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelper.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelper.kt index 63128f3332..c6efef2e57 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelper.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelper.kt @@ -1,28 +1,91 @@ package net.corda.nodeapi.internal.protonwrapper.netty +import io.netty.buffer.ByteBufAllocator +import io.netty.handler.ssl.ClientAuth +import io.netty.handler.ssl.SniHandler +import io.netty.handler.ssl.SslContextBuilder import io.netty.handler.ssl.SslHandler +import io.netty.handler.ssl.SslProvider +import io.netty.util.DomainNameMappingBuilder import net.corda.core.crypto.SecureHash import net.corda.core.crypto.newSecureRandom import net.corda.core.identity.CordaX500Name +import net.corda.core.internal.VisibleForTesting import net.corda.core.utilities.NetworkHostAndPort import net.corda.core.utilities.contextLogger import net.corda.core.utilities.toHex import net.corda.nodeapi.internal.ArtemisTcpTransport import net.corda.nodeapi.internal.config.CertificateStore import net.corda.nodeapi.internal.crypto.toBc +import net.corda.nodeapi.internal.crypto.x509 +import net.corda.nodeapi.internal.protonwrapper.netty.revocation.ExternalSourceRevocationChecker +import org.bouncycastle.asn1.ASN1InputStream +import org.bouncycastle.asn1.DERIA5String +import org.bouncycastle.asn1.DEROctetString import org.bouncycastle.asn1.x509.AuthorityKeyIdentifier +import org.bouncycastle.asn1.x509.CRLDistPoint +import org.bouncycastle.asn1.x509.DistributionPointName import org.bouncycastle.asn1.x509.Extension +import org.bouncycastle.asn1.x509.GeneralName +import org.bouncycastle.asn1.x509.GeneralNames import org.bouncycastle.asn1.x509.SubjectKeyIdentifier +import org.slf4j.LoggerFactory +import java.io.ByteArrayInputStream import java.net.Socket +import java.security.KeyStore import java.security.cert.* import java.util.* +import java.util.concurrent.Executor import javax.net.ssl.* +import kotlin.system.measureTimeMillis private const val HOSTNAME_FORMAT = "%s.corda.net" -private const val SSL_HANDSHAKE_TIMEOUT_PROP_NAME = "corda.netty.sslHelper.handshakeTimeout" -private const val DEFAULT_SSL_TIMEOUT = 20000 // Aligned with sun.security.provider.certpath.URICertStore.DEFAULT_CRL_CONNECT_TIMEOUT +internal const val DEFAULT = "default" -internal class LoggingTrustManagerWrapper(val wrapped: X509ExtendedTrustManager) : X509ExtendedTrustManager() { +internal const val DP_DEFAULT_ANSWER = "NO CRLDP ext" + +internal val logger = LoggerFactory.getLogger("net.corda.nodeapi.internal.protonwrapper.netty.SSLHelper") + +fun X509Certificate.distributionPoints() : Set? { + logger.debug("Checking CRLDPs for $subjectX500Principal") + + val crldpExtBytes = getExtensionValue(Extension.cRLDistributionPoints.id) + if (crldpExtBytes == null) { + logger.debug(DP_DEFAULT_ANSWER) + return emptySet() + } + + val derObjCrlDP = ASN1InputStream(ByteArrayInputStream(crldpExtBytes)).readObject() + val dosCrlDP = derObjCrlDP as? DEROctetString + if (dosCrlDP == null) { + logger.error("Expected to have DEROctetString, actual type: ${derObjCrlDP.javaClass}") + return emptySet() + } + val crldpExtOctetsBytes = dosCrlDP.octets + val dpObj = ASN1InputStream(ByteArrayInputStream(crldpExtOctetsBytes)).readObject() + val distPoint = CRLDistPoint.getInstance(dpObj) + if (distPoint == null) { + logger.error("Could not instantiate CRLDistPoint, from: $dpObj") + return emptySet() + } + + val dpNames = distPoint.distributionPoints.mapNotNull { it.distributionPoint }.filter { it.type == DistributionPointName.FULL_NAME } + val generalNames = dpNames.flatMap { GeneralNames.getInstance(it.name).names.asList() } + return generalNames.filter { it.tagNo == GeneralName.uniformResourceIdentifier}.map { DERIA5String.getInstance(it.name).string }.toSet() +} + +fun X509Certificate.distributionPointsToString() : String { + return with(distributionPoints()) { + if(this == null || isEmpty()) { + DP_DEFAULT_ANSWER + } else { + sorted().joinToString() + } + } +} + +@VisibleForTesting +class LoggingTrustManagerWrapper(val wrapped: X509ExtendedTrustManager) : X509ExtendedTrustManager() { companion object { val log = contextLogger() } @@ -45,12 +108,11 @@ internal class LoggingTrustManagerWrapper(val wrapped: X509ExtendedTrustManager) } catch (ex: Exception) { "null" } - " $subject[$keyIdentifier] issued by $issuer[$authorityKeyIdentifier]" + " $subject[$keyIdentifier] issued by $issuer[$authorityKeyIdentifier] [${it.distributionPointsToString()}]" } return certs.joinToString("\r\n") } - private fun certPathToStringFull(chain: Array?): String { if (chain == null) { return "" @@ -107,6 +169,33 @@ internal class LoggingTrustManagerWrapper(val wrapped: X509ExtendedTrustManager) } +private object LoggingImmediateExecutor : Executor { + + override fun execute(command: Runnable?) { + val log = LoggerFactory.getLogger(javaClass) + + if (command == null) { + log.error("SSL handler executor called with a null command") + throw NullPointerException("command") + } + + @Suppress("TooGenericExceptionCaught", "MagicNumber") // log and rethrow all exceptions + try { + val commandName = command::class.qualifiedName?.let { "[$it]" } ?: "" + log.debug("Entering SSL command $commandName") + val elapsedTime = measureTimeMillis { command.run() } + log.debug("Exiting SSL command $elapsedTime millis") + if (elapsedTime > 100) { + log.info("Command: $commandName took $elapsedTime millis to execute") + } + } + catch (ex: Exception) { + log.error("Caught exception in SSL handler executor", ex) + throw ex + } + } +} + internal fun createClientSslHelper(target: NetworkHostAndPort, expectedRemoteLegalNames: Set, keyManagerFactory: KeyManagerFactory, @@ -125,13 +214,31 @@ internal fun createClientSslHelper(target: NetworkHostAndPort, sslParameters.serverNames = listOf(SNIHostName(x500toHostName(expectedRemoteLegalNames.single()))) sslEngine.sslParameters = sslParameters } - val sslHandler = SslHandler(sslEngine) - sslHandler.handshakeTimeoutMillis = Integer.getInteger(SSL_HANDSHAKE_TIMEOUT_PROP_NAME, DEFAULT_SSL_TIMEOUT).toLong() - return sslHandler + @Suppress("DEPRECATION") + return SslHandler(sslEngine, false, LoggingImmediateExecutor) } -internal fun createServerSslHelper(keyManagerFactory: KeyManagerFactory, - trustManagerFactory: TrustManagerFactory): SslHandler { +internal fun createClientOpenSslHandler(target: NetworkHostAndPort, + expectedRemoteLegalNames: Set, + keyManagerFactory: KeyManagerFactory, + trustManagerFactory: TrustManagerFactory, + alloc: ByteBufAllocator): SslHandler { + val sslContext = SslContextBuilder.forClient().sslProvider(SslProvider.OPENSSL).keyManager(keyManagerFactory).trustManager(LoggingTrustManagerFactoryWrapper(trustManagerFactory)).build() + val sslEngine = sslContext.newEngine(alloc, target.host, target.port) + sslEngine.enabledProtocols = ArtemisTcpTransport.TLS_VERSIONS.toTypedArray() + sslEngine.enabledCipherSuites = ArtemisTcpTransport.CIPHER_SUITES.toTypedArray() + if (expectedRemoteLegalNames.size == 1) { + val sslParameters = sslEngine.sslParameters + sslParameters.serverNames = listOf(SNIHostName(x500toHostName(expectedRemoteLegalNames.single()))) + sslEngine.sslParameters = sslParameters + } + @Suppress("DEPRECATION") + return SslHandler(sslEngine, false, LoggingImmediateExecutor) +} + +internal fun createServerSslHandler(keyStore: CertificateStore, + keyManagerFactory: KeyManagerFactory, + trustManagerFactory: TrustManagerFactory): SslHandler { val sslContext = SSLContext.getInstance("TLS") val keyManagers = keyManagerFactory.keyManagers val trustManagers = trustManagerFactory.trustManagers.filterIsInstance(X509ExtendedTrustManager::class.java).map { LoggingTrustManagerWrapper(it) }.toTypedArray() @@ -142,35 +249,106 @@ internal fun createServerSslHelper(keyManagerFactory: KeyManagerFactory, sslEngine.enabledProtocols = ArtemisTcpTransport.TLS_VERSIONS.toTypedArray() sslEngine.enabledCipherSuites = ArtemisTcpTransport.CIPHER_SUITES.toTypedArray() sslEngine.enableSessionCreation = true - val sslHandler = SslHandler(sslEngine) - sslHandler.handshakeTimeoutMillis = Integer.getInteger(SSL_HANDSHAKE_TIMEOUT_PROP_NAME, DEFAULT_SSL_TIMEOUT).toLong() - return sslHandler + val sslParameters = sslEngine.sslParameters + sslParameters.sniMatchers = listOf(ServerSNIMatcher(keyStore)) + sslEngine.sslParameters = sslParameters + @Suppress("DEPRECATION") + return SslHandler(sslEngine, false, LoggingImmediateExecutor) } -internal fun initialiseTrustStoreAndEnableCrlChecking(trustStore: CertificateStore, crlCheckSoftFail: Boolean): ManagerFactoryParameters { - val certPathBuilder = CertPathBuilder.getInstance("PKIX") - val revocationChecker = certPathBuilder.revocationChecker as PKIXRevocationChecker - revocationChecker.options = EnumSet.of( - // Prefer CRL over OCSP - PKIXRevocationChecker.Option.PREFER_CRLS, - // Don't fall back to OCSP checking - PKIXRevocationChecker.Option.NO_FALLBACK) - if (crlCheckSoftFail) { - // Allow revocation check to succeed if the revocation status cannot be determined for one of - // the following reasons: The CRL or OCSP response cannot be obtained because of a network error. - revocationChecker.options = revocationChecker.options + PKIXRevocationChecker.Option.SOFT_FAIL - } +@VisibleForTesting +fun initialiseTrustStoreAndEnableCrlChecking(trustStore: CertificateStore, revocationConfig: RevocationConfig): ManagerFactoryParameters { val pkixParams = PKIXBuilderParameters(trustStore.value.internal, X509CertSelector()) + val revocationChecker = when (revocationConfig.mode) { + RevocationConfig.Mode.OFF -> AllowAllRevocationChecker // Custom PKIXRevocationChecker skipping CRL check + RevocationConfig.Mode.EXTERNAL_SOURCE -> { + require(revocationConfig.externalCrlSource != null) { "externalCrlSource must not be null" } + ExternalSourceRevocationChecker(revocationConfig.externalCrlSource!!) { Date() } // Custom PKIXRevocationChecker which uses `externalCrlSource` + } + else -> { + val certPathBuilder = CertPathBuilder.getInstance("PKIX") + val pkixRevocationChecker = certPathBuilder.revocationChecker as PKIXRevocationChecker + pkixRevocationChecker.options = EnumSet.of( + // Prefer CRL over OCSP + PKIXRevocationChecker.Option.PREFER_CRLS, + // Don't fall back to OCSP checking + PKIXRevocationChecker.Option.NO_FALLBACK) + if (revocationConfig.mode == RevocationConfig.Mode.SOFT_FAIL) { + // Allow revocation check to succeed if the revocation status cannot be determined for one of + // the following reasons: The CRL or OCSP response cannot be obtained because of a network error. + pkixRevocationChecker.options = pkixRevocationChecker.options + PKIXRevocationChecker.Option.SOFT_FAIL + } + pkixRevocationChecker + } + } pkixParams.addCertPathChecker(revocationChecker) return CertPathTrustManagerParameters(pkixParams) } +internal fun createServerOpenSslHandler(keyManagerFactory: KeyManagerFactory, + trustManagerFactory: TrustManagerFactory, + alloc: ByteBufAllocator): SslHandler { + + val sslContext = getServerSslContextBuilder(keyManagerFactory, trustManagerFactory).build() + val sslEngine = sslContext.newEngine(alloc) + sslEngine.useClientMode = false + @Suppress("DEPRECATION") + return SslHandler(sslEngine, false, LoggingImmediateExecutor) +} + +/** + * Creates a special SNI handler used only when openSSL is used for AMQPServer + */ +internal fun createServerSNIOpenSslHandler(keyManagerFactoriesMap: Map, + trustManagerFactory: TrustManagerFactory): SniHandler { + + // Default value can be any in the map. + val sslCtxBuilder = getServerSslContextBuilder(keyManagerFactoriesMap.values.first(), trustManagerFactory) + val mapping = DomainNameMappingBuilder(sslCtxBuilder.build()) + keyManagerFactoriesMap.forEach { + mapping.add(it.key, sslCtxBuilder.keyManager(it.value).build()) + } + return SniHandler(mapping.build()) +} + +@Suppress("SpreadOperator") +private fun getServerSslContextBuilder(keyManagerFactory: KeyManagerFactory, trustManagerFactory: TrustManagerFactory): SslContextBuilder { + return SslContextBuilder.forServer(keyManagerFactory) + .sslProvider(SslProvider.OPENSSL) + .trustManager(LoggingTrustManagerFactoryWrapper(trustManagerFactory)) + .clientAuth(ClientAuth.REQUIRE) + .ciphers(ArtemisTcpTransport.CIPHER_SUITES) + .protocols(*ArtemisTcpTransport.TLS_VERSIONS.toTypedArray()) +} + +internal fun splitKeystore(config: AMQPConfiguration): Map { + val keyStore = config.keyStore.value.internal + val password = config.keyStore.entryPassword.toCharArray() + return keyStore.aliases().toList().map { alias -> + val key = keyStore.getKey(alias, password) + val certs = keyStore.getCertificateChain(alias) + val x500Name = keyStore.getCertificate(alias).x509.subjectX500Principal + val cordaX500Name = CordaX500Name.build(x500Name) + val newKeyStore = KeyStore.getInstance("JKS") + newKeyStore.load(null) + newKeyStore.setKeyEntry(alias, key, password, certs) + val newKeyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()) + newKeyManagerFactory.init(newKeyStore, password) + x500toHostName(cordaX500Name) to CertHoldingKeyManagerFactoryWrapper(newKeyManagerFactory, config) + }.toMap() +} + // As per Javadoc in: https://docs.oracle.com/javase/8/docs/api/javax/net/ssl/KeyManagerFactory.html `init` method // 2nd parameter `password` - the password for recovering keys in the KeyStore fun KeyManagerFactory.init(keyStore: CertificateStore) = init(keyStore.value.internal, keyStore.entryPassword.toCharArray()) fun TrustManagerFactory.init(trustStore: CertificateStore) = init(trustStore.value.internal) +/** + * Method that converts a [CordaX500Name] to a a valid hostname (RFC-1035). It's used for SNI to indicate the target + * when trying to communicate with nodes that reside behind the same firewall. This is a solution to TLS's extension not + * yet supporting x500 names as server names + */ internal fun x500toHostName(x500Name: CordaX500Name): String { val secureHash = SecureHash.sha256(x500Name.toString()) // RFC 1035 specifies a limit 255 bytes for hostnames with each label being 63 bytes or less. Due to this, the string diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/ServerSNIMatcher.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/ServerSNIMatcher.kt new file mode 100644 index 0000000000..7cff6bf55d --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/ServerSNIMatcher.kt @@ -0,0 +1,47 @@ +package net.corda.nodeapi.internal.protonwrapper.netty + +import net.corda.core.identity.CordaX500Name +import net.corda.core.utilities.contextLogger +import net.corda.nodeapi.internal.config.CertificateStore +import net.corda.nodeapi.internal.crypto.x509 +import javax.net.ssl.SNIHostName +import javax.net.ssl.SNIMatcher +import javax.net.ssl.SNIServerName +import javax.net.ssl.StandardConstants + +class ServerSNIMatcher(private val keyStore: CertificateStore) : SNIMatcher(0) { + + companion object { + val log = contextLogger() + } + + var matchedAlias: String? = null + private set + var matchedServerName: String? = null + private set + + override fun matches(serverName: SNIServerName): Boolean { + if (serverName.type == StandardConstants.SNI_HOST_NAME) { + keyStore.aliases().forEach { alias -> + val x500Name = keyStore[alias].x509.subjectX500Principal + val cordaX500Name = CordaX500Name.build(x500Name) + // Convert the CordaX500Name into the expected host name and compare + // E.g. O=Corda B, L=London, C=GB becomes 3c6dd991936308edb210555103ffc1bb.corda.net + if ((serverName as SNIHostName).asciiName == x500toHostName(cordaX500Name)) { + matchedAlias = alias + matchedServerName = serverName.asciiName + return true + } + } + } + + val knownSNIValues = keyStore.aliases().joinToString { + val x500Name = keyStore[it].x509.subjectX500Principal + val cordaX500Name = CordaX500Name.build(x500Name) + "hostname = ${x500toHostName(cordaX500Name)} alias = $it" + } + val requestedSNIValue = "hostname = ${(serverName as SNIHostName).asciiName}" + log.warn("The requested SNI value [$requestedSNIValue] does not match any of the following known SNI values [$knownSNIValues]") + return false + } +} diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/TrustManagerFactoryWrapper.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/TrustManagerFactoryWrapper.kt new file mode 100644 index 0000000000..7565b1cdc2 --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/TrustManagerFactoryWrapper.kt @@ -0,0 +1,40 @@ +package net.corda.nodeapi.internal.protonwrapper.netty + +import java.security.KeyStore +import javax.net.ssl.ManagerFactoryParameters +import javax.net.ssl.TrustManager +import javax.net.ssl.TrustManagerFactory +import javax.net.ssl.TrustManagerFactorySpi +import javax.net.ssl.X509ExtendedTrustManager + +class LoggingTrustManagerFactorySpiWrapper(private val factorySpi: TrustManagerFactorySpi) : TrustManagerFactorySpi() { + override fun engineGetTrustManagers(): Array { + val engineGetTrustManagersMethod = TrustManagerFactorySpi::class.java.getDeclaredMethod("engineGetTrustManagers") + engineGetTrustManagersMethod.isAccessible = true + @Suppress("UNCHECKED_CAST") + val trustManagers = engineGetTrustManagersMethod.invoke(factorySpi) as Array + return if (factorySpi is LoggingTrustManagerFactorySpiWrapper) trustManagers else trustManagers.filterIsInstance(X509ExtendedTrustManager::class.java).map { LoggingTrustManagerWrapper(it) }.toTypedArray() + } + + override fun engineInit(ks: KeyStore?) { + val engineInitMethod = TrustManagerFactorySpi::class.java.getDeclaredMethod("engineInit", KeyStore::class.java) + engineInitMethod.isAccessible = true + engineInitMethod.invoke(factorySpi, ks) + } + + override fun engineInit(spec: ManagerFactoryParameters?) { + val engineInitMethod = TrustManagerFactorySpi::class.java.getDeclaredMethod("engineInit", ManagerFactoryParameters::class.java) + engineInitMethod.isAccessible = true + engineInitMethod.invoke(factorySpi, spec) + } +} + +class LoggingTrustManagerFactoryWrapper(factory: TrustManagerFactory) : TrustManagerFactory(getFactorySpi(factory), factory.provider, factory.algorithm) { + companion object { + private fun getFactorySpi(factory: TrustManagerFactory): TrustManagerFactorySpi { + val spiField = TrustManagerFactory::class.java.getDeclaredField("factorySpi") + spiField.isAccessible = true + return LoggingTrustManagerFactorySpiWrapper(spiField.get(factory) as TrustManagerFactorySpi) + } + } +} \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/revocation/ExternalSourceRevocationChecker.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/revocation/ExternalSourceRevocationChecker.kt new file mode 100644 index 0000000000..23af94ca3d --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/revocation/ExternalSourceRevocationChecker.kt @@ -0,0 +1,88 @@ +package net.corda.nodeapi.internal.protonwrapper.netty.revocation + +import net.corda.core.utilities.contextLogger +import net.corda.nodeapi.internal.protonwrapper.netty.ExternalCrlSource +import org.bouncycastle.asn1.x509.Extension +import java.security.cert.CRLReason +import java.security.cert.CertPathValidatorException +import java.security.cert.Certificate +import java.security.cert.CertificateRevokedException +import java.security.cert.PKIXRevocationChecker +import java.security.cert.X509CRL +import java.security.cert.X509Certificate +import java.util.* + +/** + * Implementation of [PKIXRevocationChecker] which determines whether certificate is revoked using [externalCrlSource] which knows how to + * obtain a set of CRLs for a given certificate from an external source + */ +class ExternalSourceRevocationChecker(private val externalCrlSource: ExternalCrlSource, private val dateSource: () -> Date) : PKIXRevocationChecker() { + + companion object { + private val logger = contextLogger() + } + + override fun check(cert: Certificate, unresolvedCritExts: MutableCollection?) { + val x509Certificate = cert as X509Certificate + checkApprovedCRLs(x509Certificate, externalCrlSource.fetch(x509Certificate)) + } + + /** + * Borrowed from `RevocationChecker.checkApprovedCRLs()` + */ + @Suppress("NestedBlockDepth") + @Throws(CertPathValidatorException::class) + private fun checkApprovedCRLs(cert: X509Certificate, approvedCRLs: Set) { + // See if the cert is in the set of approved crls. + logger.debug("ExternalSourceRevocationChecker.checkApprovedCRLs() cert SN: ${cert.serialNumber}") + + for (crl in approvedCRLs) { + val entry = crl.getRevokedCertificate(cert) + if (entry != null) { + logger.debug("ExternalSourceRevocationChecker.checkApprovedCRLs() CRL entry: $entry") + + /* + * Abort CRL validation and throw exception if there are any + * unrecognized critical CRL entry extensions (see section + * 5.3 of RFC 5280). + */ + val unresCritExts = entry.criticalExtensionOIDs + if (unresCritExts != null && !unresCritExts.isEmpty()) { + /* remove any that we will process */ + unresCritExts.remove(Extension.cRLDistributionPoints.id) + unresCritExts.remove(Extension.certificateIssuer.id) + if (!unresCritExts.isEmpty()) { + throw CertPathValidatorException( + "Unrecognized critical extension(s) in revoked CRL entry: $unresCritExts") + } + } + + val reasonCode = entry.revocationReason ?: CRLReason.UNSPECIFIED + val revocationDate = entry.revocationDate + if (revocationDate.before(dateSource())) { + val t = CertificateRevokedException( + revocationDate, reasonCode, + crl.issuerX500Principal, mutableMapOf()) + throw CertPathValidatorException( + t.message, t, null, -1, CertPathValidatorException.BasicReason.REVOKED) + } + } + } + } + + override fun isForwardCheckingSupported(): Boolean { + return true + } + + override fun getSupportedExtensions(): MutableSet? { + return null + } + + override fun init(forward: Boolean) { + // Nothing to do + } + + override fun getSoftFailExceptions(): MutableList { + return LinkedList() + } +} \ No newline at end of file diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/config/ConfigParsingTest.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/config/ConfigParsingTest.kt index ef6bec0348..9c15c0f478 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/config/ConfigParsingTest.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/config/ConfigParsingTest.kt @@ -9,7 +9,6 @@ import net.corda.core.identity.CordaX500Name import net.corda.core.internal.div import net.corda.core.utilities.NetworkHostAndPort import org.assertj.core.api.Assertions.* -import org.hibernate.exception.DataException import org.junit.Test import java.net.URL import java.nio.file.Path diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelperTest.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelperTest.kt index 3802a357b0..782d8b8abe 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelperTest.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelperTest.kt @@ -4,7 +4,10 @@ import net.corda.core.crypto.SecureHash import net.corda.core.identity.CordaX500Name import net.corda.core.utilities.NetworkHostAndPort import net.corda.coretesting.internal.configureTestSSL +import net.corda.nodeapi.internal.DEV_CA_KEY_STORE_PASS +import net.corda.nodeapi.internal.DEV_CA_PRIVATE_KEY_PASS import net.corda.nodeapi.internal.config.CertificateStore +import net.corda.nodeapi.internal.crypto.X509Utilities.CORDA_CLIENT_TLS import org.junit.Test import javax.net.ssl.KeyManagerFactory import javax.net.ssl.SNIHostName @@ -23,7 +26,7 @@ class SSLHelperTest { val keyStore = sslConfig.keyStore keyManagerFactory.init(CertificateStore.fromFile(keyStore.path, keyStore.storePassword, keyStore.entryPassword, false)) val trustStore = sslConfig.trustStore - trustManagerFactory.init(initialiseTrustStoreAndEnableCrlChecking(CertificateStore.fromFile(trustStore.path, trustStore.storePassword, trustStore.entryPassword, false), false)) + trustManagerFactory.init(initialiseTrustStoreAndEnableCrlChecking(CertificateStore.fromFile(trustStore.path, trustStore.storePassword, trustStore.entryPassword, false), RevocationConfigImpl(RevocationConfig.Mode.HARD_FAIL))) val sslHandler = createClientSslHelper(NetworkHostAndPort("localhost", 1234), setOf(legalName), keyManagerFactory, trustManagerFactory) val legalNameHash = SecureHash.sha256(legalName.toString()).toString().take(32).toLowerCase() @@ -34,4 +37,14 @@ class SSLHelperTest { assertEquals(1, sslHandler.engine().sslParameters.serverNames.size) assertEquals("$legalNameHash.corda.net", (sslHandler.engine().sslParameters.serverNames.first() as SNIHostName).asciiName) } + + @Test(timeout=300_000) + fun `test distributionPointsToString`() { + val certStore = CertificateStore.fromResource( + "net/corda/nodeapi/internal/protonwrapper/netty/sslkeystore_Revoked.jks", + DEV_CA_KEY_STORE_PASS, DEV_CA_PRIVATE_KEY_PASS) + val distPoints = certStore.query { getCertificateChain(CORDA_CLIENT_TLS).map { it.distributionPointsToString() } } + assertEquals(listOf("NO CRLDP ext", "http://day-v3-doorman.cordaconnect.io/doorman", + "http://day3-doorman.cordaconnect.io/doorman", "http://day3-doorman.cordaconnect.io/subordinate", "NO CRLDP ext"), distPoints) + } } \ No newline at end of file diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/revocation/ExternalSourceRevocationCheckerTest.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/revocation/ExternalSourceRevocationCheckerTest.kt new file mode 100644 index 0000000000..7be350a525 --- /dev/null +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/revocation/ExternalSourceRevocationCheckerTest.kt @@ -0,0 +1,56 @@ +package net.corda.nodeapi.internal.protonwrapper.netty.revocation + +import net.corda.core.utilities.Try +import net.corda.nodeapi.internal.DEV_CA_KEY_STORE_PASS +import net.corda.nodeapi.internal.DEV_CA_PRIVATE_KEY_PASS +import net.corda.nodeapi.internal.config.CertificateStore +import net.corda.nodeapi.internal.crypto.X509Utilities +import net.corda.nodeapi.internal.protonwrapper.netty.ExternalCrlSource +import org.bouncycastle.jcajce.provider.asymmetric.x509.CertificateFactory +import org.junit.Test +import java.math.BigInteger + +import java.security.cert.X509CRL +import java.security.cert.X509Certificate +import java.sql.Date +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class ExternalSourceRevocationCheckerTest { + + @Test(timeout=300_000) + fun checkRevoked() { + val checkResult = performCheckOnDate(Date.valueOf("2019-09-27")) + val failedChecks = checkResult.filterNot { it.second.isSuccess } + assertEquals(1, failedChecks.size) + assertEquals(BigInteger.valueOf(8310484079152632582), failedChecks.first().first.serialNumber) + } + + @Test(timeout=300_000) + fun checkTooEarly() { + val checkResult = performCheckOnDate(Date.valueOf("2019-08-27")) + assertTrue(checkResult.all { it.second.isSuccess }) + } + + private fun performCheckOnDate(date: Date): List>> { + val certStore = CertificateStore.fromResource( + "net/corda/nodeapi/internal/protonwrapper/netty/sslkeystore_Revoked.jks", + DEV_CA_KEY_STORE_PASS, DEV_CA_PRIVATE_KEY_PASS) + + val resourceAsStream = javaClass.getResourceAsStream("/net/corda/nodeapi/internal/protonwrapper/netty/doorman.crl") + val crl = CertificateFactory().engineGenerateCRL(resourceAsStream) as X509CRL + + //val crlHolder = X509CRLHolder(resourceAsStream) + //crlHolder.revokedCertificates as X509CRLEntryHolder + + val instance = ExternalSourceRevocationChecker(object : ExternalCrlSource { + override fun fetch(certificate: X509Certificate): Set = setOf(crl) + }) { date } + + return certStore.query { + getCertificateChain(X509Utilities.CORDA_CLIENT_TLS).map { + Pair(it, Try.on { instance.check(it, mutableListOf()) }) + } + } + } +} \ No newline at end of file diff --git a/node-api/src/test/resources/net/corda/nodeapi/internal/protonwrapper/netty/Readme.txt b/node-api/src/test/resources/net/corda/nodeapi/internal/protonwrapper/netty/Readme.txt new file mode 100644 index 0000000000..e312c76d10 --- /dev/null +++ b/node-api/src/test/resources/net/corda/nodeapi/internal/protonwrapper/netty/Readme.txt @@ -0,0 +1,3 @@ +Represents some test data which contains real certificates produced by DayWatch Doorman as well as CRL list file. + +For all the keystores the password is "cordacadevpass". \ No newline at end of file diff --git a/node-api/src/test/resources/net/corda/nodeapi/internal/protonwrapper/netty/doorman.crl b/node-api/src/test/resources/net/corda/nodeapi/internal/protonwrapper/netty/doorman.crl new file mode 100644 index 0000000000000000000000000000000000000000..469901e9aa88897242fedba42004e35b1af45254 GIT binary patch literal 576 zcmXqLVzM!4V!Xk`$Y>zK#-Y{ban6>7nF-2bW@I-=GvqelWMd9xVH0Kw4K@@s;0JNo zg*p6E%M~K?i?R*HfI5LfT*ACT#tI(!IVsNh3O+v0hI|G*AaQPCR_FYplte=r11S)n zSyRi)WT4K(;dgb_ ztkD1aSRs-IW`@QF#>R$G20R8_K!asPS(v#PIYA1GLyp>7=1pdUDzr2-2P!m1Rmi%a z@sUB}eS^l?Aic^gjgt);Cp3s?NXl6`iX8CwTm0o^weE_Ap}sZl3-gxb7-Uf`925-X zAmJd691g{$Nx)Fb%u6guZQ@Wj@V$0<*5Vlk_8`Z}Ff#sUF*Pt=V7NfPO*f;Yq`*pF zKP9nJx6D`<8hm=mpde1p&&x|qF44=(*N2K@k5pz41_M_nh4AL&sI{Ay+UM8LU;l#V z{957jhc1;ZowfKvpZRkK`HUw_iVUxd|CGOdTYEm$FM+W?CP09TW$VYohLhs<2HEL0 Id_O1$06rO?9smFU literal 0 HcmV?d00001 diff --git a/node-api/src/test/resources/net/corda/nodeapi/internal/protonwrapper/netty/sslkeystore_Revoked.jks b/node-api/src/test/resources/net/corda/nodeapi/internal/protonwrapper/netty/sslkeystore_Revoked.jks new file mode 100644 index 0000000000000000000000000000000000000000..67e111462433d10035b4a7bc402f9275ba70efbc GIT binary patch literal 3594 zcmd7T3sh2h9tUvl<${u-F*s;V8s->22K*!WARnP(Scx?mrRFpPYU-e1h&idd3LMGG zvCLUVn=%U~9c_A-r!=+PqIM^(jBI*nZjaI9SZ4Ozw7n=DX|mawbKK5y_#f_{%YFU+ zpWm0^zTrL$1_Pr58X1&pKduF@uG7lwW8h zYX}=#@kfg|K9ZM>r`GxH;E5RCyAKBFKCb1}H4%xk7c#CisaP$F&t~pUZ;grH81(d4 zhqFgBZQ?@iD?FFn=x%hu;7`~0WJ32=kb1792=fjz%(G8g9{OdmZ~8wrELfMneslXp zi#&nnsui|fE}J;b&YSq`oUQ}&_4}T6v~GKPT}yY>J~?_~pnN}BhmF}=k3JU;gNtN% z0ybI&H=u-RKnXQhU||S`&AFX@-uH@@Uk28Ab60g}i$8*ejayjA3I{1SpDu9r~zt zmIwv0hFXdumOk)k>bYGYTcf?bz1426JD65W*$uC zC{i*PmwxwZfrb{LI{##gS6b}q4{F8E0a`qk9yf4fOwG@z8a+C)jcU@Cp7v);it%95 zm>RIk)Jg=C1gs1=%*f{9@fp-8sZ7kIpm%KWLMAR@QA4H4aluk*SXeOAa-@2Q1sNA? zXicV}Qt0onWHMK{cDYa1fu`foVXI5Y{UNBf|R@9 zIjEh+2VNnEc#WD9y3cA0=J6l-O|hu13!l%UK3tHyj(orh?Bg`lgcPg1LlWVx!ARnM zjS})i8<1%NYTWfURBsd%4q#DNCZ-_uA2StYIEB5zOP0Kx}9{$?;tnbsT*zoY))##r%O3lh3-;k8* z(F-^sh5&jUye1L+HdY7(8qPpP<^Uhy_4*hX0NN3{b~ zdj(zXHs<}$X*Wjt*7H^QhaLYeDlAVN3fr5QY%9))@~FdlJZ7xSzoylaVCqU52Xcf~ zH=c7Y;74_QzfxYApskrjdbxPXV8JBw|FQL?1&>=T+5Adq5YiG|s_6GOZ3FAH$ z*o<7!9;pyrJr}HzzC^y9V9Yy+| zwv;tA?Yh(-9Cef{Bc`?gu+`CHsZ>E{dLDbAJ{j+l6kK*L4ac-dR(7x ziT-Ukn_@8julL5xn4WV}R~XGaez#vYt1Q&4hx0RT*5&faA`+>&5~fn_cdZ|2n48KN v(C$$LM0dcF9IbgN{%ejM_Yv)zwxTK#_d@Db+V#*6cO5Eam7J;f>E8Pv6ki8X literal 0 HcmV?d00001 diff --git a/node-api/src/test/resources/net/corda/nodeapi/internal/serialization/amqp/networkParamsWrite b/node-api/src/test/resources/net/corda/nodeapi/internal/serialization/amqp/networkParamsWrite new file mode 100644 index 0000000000000000000000000000000000000000..dcdbaa7b5f0d92f240af49f1068fcaee9c5b77f2 GIT binary patch literal 3066 zcmcImTWl0n7@ob%P7#CHh$xCF4XB|yy_YJbOt(99mtH7zw`{j2=Ire0_O!ESmYK8N zZh0|E@bYAg#>51}u@Al?FG5Vr7}RJ&q{f&iQ4+(8C{bffG=Zr9Y;`-kI)#StGM(=E z=D&Ua_y0#T%nb4gg7C~oX?O~PZ%Gi=Uxt@}u-Hg3lQjqNfa7^fqH= zSKK_1-LnVK(se-ltI#vFYdWv0X5T_vegMyQV5`u*|KKvZ-f?mYK3i9|TwkQQ zbDb0OXl&k^=uh^yCJr4?2ZoxLefwg74#lfujppZKLlZND(PYQ%2-aYjV1+yQDj~hLEE04I>6aW~pHen{!0NJd`S!M4MK)zdtH0Zd)n! z44Ol-g*7;4C3&=v)`>QVOD7*|+1Vn70!=s{*e0C+>yJa%qd%Wl>pC{E_tIZZUYaa) z`zNn$FzPOTcjgNzd;}Q$zWfjtAE+_TNDj*}YB5BaE&3BeT!{NB9pDE}#it#XoR@KU z5i4gR{N#Jb`H6Q_jLOu=V3a2^p$wZ8=`tJ@4Rcl=S^hhUOq9cLW+`gLRKR}N@9r=! zmKDckuS0X?)1bXW;`inVCAmUwR7Y&eFmvNzo*0w|APSZHT9m%9>w~z&>!?Ej!4~)J zv=txUQ!aFY5nUc47VEj0e99EY=#5c%V4D!A+GT8CabR&3(0L>aW3p#?|=QwFl)TUVUJ7{c~p;<1a4MWF>s$85xx`o}XAljdcH?Y1+Dqf>Tv5&e_X_t7Af~5EkuQd9d zsh#X>X&C`nZ8}hibw-gwDlk_ZHEi+m%^qW?s=dnckCf$qbDmd8)oWTR_GP!yw?OGN zBbPI%lL-%5%!Ifhqp(}v4HDmIm{~-L1r!-HK?(E1J_w?6V_mqak#QgNi*9H@JM_zn z-zv-|P#cU<<{8*#iA$PMpv?5JvF-!dfbKb%jx{G^bpPl^;&^4$q!_y>wyk z@6VyDXDWF6Lej}HM?U~lGYd@7^aTXV0 zu+C&Bi!pqpfT@OiD>bj3l3sT*#co{L{XyI5=(JN79!e7m7na(jPK&n`-|b5zu2tMEkn9R#I~VKfKPqv5dQD^chAcWm=Wk|j B^OXPq literal 0 HcmV?d00001 diff --git a/node/src/integration-test/kotlin/net/corda/node/amqp/AMQPBridgeTest.kt b/node/src/integration-test/kotlin/net/corda/node/amqp/AMQPBridgeTest.kt index 221d11cd9a..be935b0856 100644 --- a/node/src/integration-test/kotlin/net/corda/node/amqp/AMQPBridgeTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/amqp/AMQPBridgeTest.kt @@ -22,6 +22,7 @@ import net.corda.testing.core.TestIdentity import net.corda.testing.driver.internal.incrementalPortAllocation import net.corda.coretesting.internal.rigorousMock import net.corda.coretesting.internal.stubs.CertificateStoreStubs +import net.corda.nodeapi.internal.protonwrapper.netty.toRevocationConfig import org.apache.activemq.artemis.api.core.Message.HDR_DUPLICATE_DETECTION_ID import org.apache.activemq.artemis.api.core.RoutingType import org.apache.activemq.artemis.api.core.SimpleString @@ -206,13 +207,22 @@ class AMQPBridgeTest { val artemisClient = ArtemisMessagingClient(artemisConfig.p2pSslOptions, artemisAddress, MAX_MESSAGE_SIZE) artemisServer.start() artemisClient.start() - val bridgeManager = AMQPBridgeManager(artemisConfig.p2pSslOptions, artemisAddress, MAX_MESSAGE_SIZE, artemisConfig.crlCheckSoftFail) + val bridgeManager = AMQPBridgeManager( + artemisConfig.p2pSslOptions.keyStore.get(), + artemisConfig.p2pSslOptions.trustStore.get(), + false, + null, + MAX_MESSAGE_SIZE, + artemisConfig.crlCheckSoftFail.toRevocationConfig(), + false, { ArtemisMessagingClient(artemisConfig.p2pSslOptions, artemisAddress, MAX_MESSAGE_SIZE) }, trace = false, + sslHandshakeTimeout = null, + bridgeConnectionTTLSeconds = 0) bridgeManager.start() val artemis = artemisClient.started!! if (sourceQueueName != null) { // Local queue for outgoing messages artemis.session.createQueue(sourceQueueName, RoutingType.ANYCAST, sourceQueueName, true) - bridgeManager.deployBridge(sourceQueueName, listOf(amqpAddress), setOf(BOB.name)) + bridgeManager.deployBridge(ALICE_NAME.toString(), sourceQueueName, listOf(amqpAddress), setOf(BOB.name)) } return Triple(artemisServer, artemisClient, bridgeManager) } @@ -228,7 +238,6 @@ class AMQPBridgeTest { doReturn(certificatesDirectory).whenever(it).certificatesDirectory doReturn(signingCertificateStore).whenever(it).signingCertificateStore doReturn(p2pSslConfiguration).whenever(it).p2pSslOptions - doReturn(true).whenever(it).crlCheckSoftFail } serverConfig.configureWithDevSSLCertificate() @@ -238,7 +247,6 @@ class AMQPBridgeTest { override val trustStore = serverConfig.p2pSslOptions.trustStore.get() override val trace: Boolean = true override val maxMessageSize: Int = maxMessageSize - override val crlCheckSoftFail: Boolean = serverConfig.crlCheckSoftFail } return AMQPServer("0.0.0.0", amqpAddress.port, diff --git a/node/src/integration-test/kotlin/net/corda/node/amqp/CertificateRevocationListNodeTests.kt b/node/src/integration-test/kotlin/net/corda/node/amqp/CertificateRevocationListNodeTests.kt index 1ab13fc6d7..6b884ef300 100644 --- a/node/src/integration-test/kotlin/net/corda/node/amqp/CertificateRevocationListNodeTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/amqp/CertificateRevocationListNodeTests.kt @@ -29,6 +29,8 @@ import net.corda.coretesting.internal.DEV_INTERMEDIATE_CA import net.corda.coretesting.internal.DEV_ROOT_CA import net.corda.coretesting.internal.rigorousMock import net.corda.coretesting.internal.stubs.CertificateStoreStubs +import net.corda.nodeapi.internal.protonwrapper.netty.RevocationConfig +import net.corda.nodeapi.internal.protonwrapper.netty.toRevocationConfig import org.assertj.core.api.Assertions.assertThatIllegalArgumentException import org.bouncycastle.asn1.x500.X500Name import org.bouncycastle.asn1.x509.* @@ -305,7 +307,7 @@ class CertificateRevocationListNodeTests { } @Test(timeout=300_000) - fun `Revocation status chceck fails when the CRL distribution point is not set and soft fail is disabled`() { + fun `Revocation status check fails when the CRL distribution point is not set and soft fail is disabled`() { val crlCheckSoftFail = false val (amqpServer, _) = createServer( serverPort, @@ -380,7 +382,6 @@ class CertificateRevocationListNodeTests { val amqpConfig = object : AMQPConfiguration { override val keyStore = keyStore override val trustStore = clientConfig.p2pSslOptions.trustStore.get() - override val crlCheckSoftFail: Boolean = crlCheckSoftFail override val maxMessageSize: Int = maxMessageSize } return Pair(AMQPClient( @@ -404,7 +405,6 @@ class CertificateRevocationListNodeTests { doReturn(name).whenever(it).myLegalName doReturn(p2pSslConfiguration).whenever(it).p2pSslOptions doReturn(signingCertificateStore).whenever(it).signingCertificateStore - doReturn(crlCheckSoftFail).whenever(it).crlCheckSoftFail } serverConfig.configureWithDevSSLCertificate() val nodeCert = (signingCertificateStore to p2pSslConfiguration).recreateNodeCaAndTlsCertificates(nodeCrlDistPoint, tlsCrlDistPoint) @@ -412,7 +412,7 @@ class CertificateRevocationListNodeTests { val amqpConfig = object : AMQPConfiguration { override val keyStore = keyStore override val trustStore = serverConfig.p2pSslOptions.trustStore.get() - override val crlCheckSoftFail: Boolean = crlCheckSoftFail + override val revocationConfig = crlCheckSoftFail.toRevocationConfig() override val maxMessageSize: Int = maxMessageSize } return Pair(AMQPServer( diff --git a/node/src/integration-test/kotlin/net/corda/node/amqp/ProtonWrapperTests.kt b/node/src/integration-test/kotlin/net/corda/node/amqp/ProtonWrapperTests.kt index d3515e56bd..874d1feca4 100644 --- a/node/src/integration-test/kotlin/net/corda/node/amqp/ProtonWrapperTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/amqp/ProtonWrapperTests.kt @@ -32,6 +32,7 @@ import net.corda.testing.driver.internal.incrementalPortAllocation import net.corda.testing.internal.createDevIntermediateCaCertPath import net.corda.coretesting.internal.rigorousMock import net.corda.coretesting.internal.stubs.CertificateStoreStubs +import net.corda.nodeapi.internal.protonwrapper.netty.toRevocationConfig import org.apache.activemq.artemis.api.core.RoutingType import org.assertj.core.api.Assertions.assertThatThrownBy import org.junit.Assert.assertArrayEquals @@ -39,6 +40,7 @@ import org.junit.Rule import org.junit.Test import org.junit.rules.TemporaryFolder import java.security.cert.X509Certificate +import java.util.concurrent.TimeUnit import javax.net.ssl.* import kotlin.concurrent.thread import kotlin.test.assertEquals @@ -341,6 +343,7 @@ class ProtonWrapperTests { val connection1ID = CordaX500Name.build(connection1.remoteCert!!.subjectX500Principal) assertEquals("client 0", connection1ID.organisationUnit) val source1 = connection1.remoteAddress + val client2Connected = amqpClient2.onConnection.toFuture() amqpClient2.start() val connection2 = connectionEvents.next() assertEquals(true, connection2.connected) @@ -353,6 +356,7 @@ class ProtonWrapperTests { assertEquals(false, connection3.connected) assertEquals(source1, connection3.remoteAddress) assertEquals(false, amqpClient1.connected) + client2Connected.get(60, TimeUnit.SECONDS) assertEquals(true, amqpClient2.connected) // Now shutdown both amqpClient2.stop() @@ -362,11 +366,13 @@ class ProtonWrapperTests { assertEquals(false, amqpClient1.connected) assertEquals(false, amqpClient2.connected) // Now restarting one should work + val client1Connected = amqpClient1.onConnection.toFuture() amqpClient1.start() val connection5 = connectionEvents.next() assertEquals(true, connection5.connected) val connection5ID = CordaX500Name.build(connection5.remoteCert!!.subjectX500Principal) assertEquals("client 0", connection5ID.organisationUnit) + client1Connected.get(60, TimeUnit.SECONDS) assertEquals(true, amqpClient1.connected) assertEquals(false, amqpClient2.connected) // Cleanup @@ -447,7 +453,6 @@ class ProtonWrapperTests { override val trustStore = clientTruststore override val trace: Boolean = true override val maxMessageSize: Int = maxMessageSize - override val crlCheckSoftFail: Boolean = clientConfig.crlCheckSoftFail } return AMQPClient( listOf(NetworkHostAndPort("localhost", serverPort), @@ -479,7 +484,6 @@ class ProtonWrapperTests { override val trustStore = clientTruststore override val trace: Boolean = true override val maxMessageSize: Int = maxMessageSize - override val crlCheckSoftFail: Boolean = clientConfig.crlCheckSoftFail } return AMQPClient( listOf(NetworkHostAndPort("localhost", serverPort)), @@ -502,7 +506,6 @@ class ProtonWrapperTests { doReturn(name).whenever(it).myLegalName doReturn(signingCertificateStore).whenever(it).signingCertificateStore doReturn(p2pSslConfiguration).whenever(it).p2pSslOptions - doReturn(crlCheckSoftFail).whenever(it).crlCheckSoftFail } serverConfig.configureWithDevSSLCertificate() @@ -512,8 +515,8 @@ class ProtonWrapperTests { override val keyStore = serverKeystore override val trustStore = serverTruststore override val trace: Boolean = true + override val revocationConfig = crlCheckSoftFail.toRevocationConfig() override val maxMessageSize: Int = maxMessageSize - override val crlCheckSoftFail: Boolean = serverConfig.crlCheckSoftFail } return AMQPServer( "0.0.0.0", diff --git a/node/src/main/kotlin/net/corda/node/internal/Node.kt b/node/src/main/kotlin/net/corda/node/internal/Node.kt index eee9e063f0..951fb25988 100644 --- a/node/src/main/kotlin/net/corda/node/internal/Node.kt +++ b/node/src/main/kotlin/net/corda/node/internal/Node.kt @@ -79,6 +79,7 @@ import net.corda.nodeapi.internal.bridging.BridgeControlListener import net.corda.nodeapi.internal.config.User import net.corda.nodeapi.internal.crypto.X509Utilities import net.corda.nodeapi.internal.persistence.CouldNotCreateDataSourceException +import net.corda.nodeapi.internal.protonwrapper.netty.toRevocationConfig import net.corda.serialization.internal.AMQP_P2P_CONTEXT import net.corda.serialization.internal.AMQP_RPC_CLIENT_CONTEXT import net.corda.serialization.internal.AMQP_RPC_SERVER_CONTEXT @@ -417,7 +418,15 @@ open class Node(configuration: NodeConfiguration, failoverCallback = { errorAndTerminate("ArtemisMessagingClient failed. Shutting down.", null) } ) } - return BridgeControlListener(configuration.p2pSslOptions, networkParameters.maxMessageSize, configuration.crlCheckSoftFail, artemisMessagingClientFactory) + return BridgeControlListener( + configuration.p2pSslOptions.keyStore.get(), + configuration.p2pSslOptions.trustStore.get(), + false, + null, + networkParameters.maxMessageSize, + configuration.crlCheckSoftFail.toRevocationConfig(), + false, + artemisMessagingClientFactory) } private fun startLocalRpcBroker(securityManager: RPCSecurityManager): BrokerAddresses? { diff --git a/node/src/main/kotlin/net/corda/node/services/messaging/P2PMessagingClient.kt b/node/src/main/kotlin/net/corda/node/services/messaging/P2PMessagingClient.kt index b4b74bc505..8254c0138f 100644 --- a/node/src/main/kotlin/net/corda/node/services/messaging/P2PMessagingClient.kt +++ b/node/src/main/kotlin/net/corda/node/services/messaging/P2PMessagingClient.kt @@ -197,7 +197,7 @@ class P2PMessagingClient(val config: NodeConfiguration, inboxes += RemoteInboxAddress(it).queueName } - inboxes.forEach { createQueueIfAbsent(it, producerSession!!, exclusive = true) } + inboxes.forEach { createQueueIfAbsent(it, producerSession!!, exclusive = true, isServiceAddress = false) } p2pConsumer = P2PMessagingConsumer(inboxes, createNewSession, isDrainingModeOn, drainingModeWasChangedEvents, metricRegistry) @@ -267,7 +267,7 @@ class P2PMessagingClient(val config: NodeConfiguration, return state.locked { node.legalIdentitiesAndCerts.map { val messagingAddress = NodeAddress(it.party.owningKey) - BridgeEntry(messagingAddress.queueName, node.addresses, node.legalIdentities.map { it.name }) + BridgeEntry(messagingAddress.queueName, node.addresses, node.legalIdentities.map { it.name }, serviceAddress = false) }.filter { producerSession!!.queueQuery(SimpleString(it.queueName)).isExists }.asSequence() } } @@ -306,7 +306,7 @@ class P2PMessagingClient(val config: NodeConfiguration, val keyHash = queueName.substring(PEERS_PREFIX.length) val peers = networkMap.getNodesByOwningKeyIndex(keyHash) for (node in peers) { - val bridge = BridgeEntry(queueName.toString(), node.addresses, node.legalIdentities.map { it.name }) + val bridge = BridgeEntry(queueName.toString(), node.addresses, node.legalIdentities.map { it.name }, serviceAddress = false) requiredBridges += bridge knownQueues += queueName.toString() } @@ -527,19 +527,20 @@ class P2PMessagingClient(val config: NodeConfiguration, val internalTargetQueue = (address as? ArtemisAddress)?.queueName ?: throw IllegalArgumentException("Not an Artemis address") state.locked { - createQueueIfAbsent(internalTargetQueue, producerSession!!, exclusive = address !is ServiceAddress) + val serviceAddress = address is ServiceAddress + createQueueIfAbsent(internalTargetQueue, producerSession!!, exclusive = !serviceAddress, isServiceAddress = serviceAddress) } internalTargetQueue } } /** Attempts to create a durable queue on the broker which is bound to an address of the same name. */ - private fun createQueueIfAbsent(queueName: String, session: ClientSession, exclusive: Boolean) { + private fun createQueueIfAbsent(queueName: String, session: ClientSession, exclusive: Boolean, isServiceAddress: Boolean) { fun sendBridgeCreateMessage() { val keyHash = queueName.substring(PEERS_PREFIX.length) val peers = networkMap.getNodesByOwningKeyIndex(keyHash) for (node in peers) { - val bridge = BridgeEntry(queueName, node.addresses, node.legalIdentities.map { it.name }) + val bridge = BridgeEntry(queueName, node.addresses, node.legalIdentities.map { it.name }, isServiceAddress) val createBridgeMessage = BridgeControl.Create(config.myLegalName.toString(), bridge) sendBridgeControl(createBridgeMessage) }