ExternalVendorCode/corda
1
0
Fork 0
mirror of https://github.com/corda/corda.git synced 2025-04-06 19:07:08 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

ENT-4494 Harmonize network stack (#6059)

Browse Source 
* ENT-4494 harmonize proton wrapper with ENT

* Harmonise Artemis and Bridge implementation

* Move tests across

* detekt changes

* Fix AMQP tests in node
This commit is contained in:
Christian Sailer 2020-03-13 14:26:24 +00:00 committed by GitHub
parent 3b335ebb00
commit ea22a10b3e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
48 changed files with 2372 additions and 340 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
detekt-baseline.xml
View File

@ -91,7 +91,6 @@
<ID>ComplexCondition:WireTransaction.kt$WireTransaction$notary != null &amp;&amp; (inputs.isNotEmpty() || references.isNotEmpty() || timeWindow != null)</ID>
<ID>ComplexMethod:AMQPBridgeManager.kt$AMQPBridgeManager.AMQPBridge$private fun clientArtemisMessageHandler(artemisMessage: ClientMessage)</ID>
<ID>ComplexMethod:AMQPBridgeTest.kt$AMQPBridgeTest$@Test(timeout=300_000) fun `test acked and nacked messages`()</ID>
<ID>ComplexMethod:AMQPChannelHandler.kt$AMQPChannelHandler$override fun userEventTriggered(ctx: ChannelHandlerContext, evt: Any)</ID>
<ID>ComplexMethod:AMQPTypeIdentifierParser.kt$AMQPTypeIdentifierParser$// Make sure our inputs aren't designed to blow things up. private fun validate(typeString: String)</ID>
<ID>ComplexMethod:ANSIProgressRenderer.kt$ANSIProgressRenderer$// Returns number of lines rendered. private fun renderLevel(ansi: Ansi, error: Boolean): Int</ID>
<ID>ComplexMethod:ANSIProgressRenderer.kt$ANSIProgressRenderer$@Synchronized protected fun draw(moveUp: Boolean, error: Throwable? = null)</ID>
@ -124,13 +123,10 @@
<ID>ComplexMethod:ConfigUtilities.kt$// For Iterables figure out the type parameter and apply the same logic as above on the individual elements. private fun Iterable&lt;*&gt;.toConfigIterable(field: Field): Iterable&lt;Any?&gt;</ID>
<ID>ComplexMethod:ConfigUtilities.kt$// TODO Move this to KeyStoreConfigHelpers. fun MutualSslConfiguration.configureDevKeyAndTrustStores(myLegalName: CordaX500Name, signingCertificateStore: FileBasedCertificateStoreSupplier, certificatesDirectory: Path, cryptoService: CryptoService? = null)</ID>
<ID>ComplexMethod:ConfigUtilities.kt$@Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN") // Reflect over the fields of the receiver and generate a value Map that can use to create Config object. private fun Any.toConfigMap(): Map&lt;String, Any&gt;</ID>
<ID>ComplexMethod:ConfigUtilities.kt$private fun Config.getCollectionValue(path: String, type: KType, onUnknownKeys: (Set&lt;String&gt;, logger: Logger) -&gt; Unit, nestedPath: String?, baseDirectory: Path?): Collection&lt;Any&gt;</ID>
<ID>ComplexMethod:ConfigUtilities.kt$private fun Config.getSingleValue(path: String, type: KType, onUnknownKeys: (Set&lt;String&gt;, logger: Logger) -&gt; Unit, nestedPath: String?, baseDirectory: Path?): Any?</ID>
<ID>ComplexMethod:ConfigUtilities.kt$private fun convertValue(value: Any): Any</ID>
<ID>ComplexMethod:ConnectionStateMachine.kt$ConnectionStateMachine$override fun onConnectionFinal(event: Event)</ID>
<ID>ComplexMethod:ConnectionStateMachine.kt$ConnectionStateMachine$override fun onDelivery(event: Event)</ID>
<ID>ComplexMethod:ConstraintsUtils.kt$ fun AttachmentConstraint.canBeTransitionedFrom(input: AttachmentConstraint, attachment: ContractAttachment): Boolean</ID>
<ID>ComplexMethod:CordaCliWrapper.kt$fun CordaCliWrapper.start(args: Array&lt;String&gt;)</ID>
<ID>ComplexMethod:CordaPersistence.kt$CordaPersistence$private fun &lt;T&gt; inTopLevelTransaction(isolationLevel: TransactionIsolationLevel, recoverableFailureTolerance: Int, recoverAnyNestedSQLException: Boolean, statement: DatabaseTransaction.() -&gt; T): T</ID>
<ID>ComplexMethod:CordaRPCClient.kt$CordaRPCClientConfiguration$override fun equals(other: Any?): Boolean</ID>
<ID>ComplexMethod:CordaRPCClientTest.kt$CordaRPCClientTest$@Test(timeout=300_000) fun `shutdown command stops the node`()</ID>
@ -182,7 +178,6 @@
<ID>ComplexMethod:RPCClientProxyHandler.kt$RPCClientProxyHandler$// This is the general function that transforms a client side RPC to internal Artemis messages. override fun invoke(proxy: Any, method: Method, arguments: Array&lt;out Any?&gt;?): Any?</ID>
<ID>ComplexMethod:RPCClientProxyHandler.kt$RPCClientProxyHandler$private fun attemptReconnect()</ID>
<ID>ComplexMethod:RPCServer.kt$RPCServer$private fun clientArtemisMessageHandler(artemisMessage: ClientMessage)</ID>
<ID>ComplexMethod:ReconnectingCordaRPCOps.kt$ReconnectingCordaRPCOps.ErrorInterceptingHandler$ private fun doInvoke(method: Method, args: Array&lt;out Any&gt;?, maxNumberOfAttempts: Int): Any?</ID>
<ID>ComplexMethod:ReconnectingCordaRPCOps.kt$ReconnectingCordaRPCOps.ReconnectingRPCConnection$ private tailrec fun establishConnectionWithRetry( retryInterval: Duration, roundRobinIndex: Int = 0, retries: Int = -1 ): CordaRPCConnection?</ID>
<ID>ComplexMethod:RemoteTypeCarpenter.kt$SchemaBuildingRemoteTypeCarpenter$override fun carpent(typeInformation: RemoteTypeInformation): Type</ID>
<ID>ComplexMethod:RpcReconnectTests.kt$RpcReconnectTests$ @Test(timeout=300_000) fun `test that the RPC client is able to reconnect and proceed after node failure, restart, or connection reset`()</ID>
@ -215,7 +210,6 @@
<ID>EmptyDefaultConstructor:FlowRetryTest.kt$AsyncRetryFlow$()</ID>
<ID>EmptyDefaultConstructor:FlowRetryTest.kt$RetryFlow$()</ID>
<ID>EmptyDefaultConstructor:FlowRetryTest.kt$ThrowingFlow$()</ID>
<ID>EmptyElseBlock:CordaCliWrapper.kt${ }</ID>
<ID>EmptyIfBlock:ContentSignerBuilder.kt$ContentSignerBuilder.SignatureOutputStream$if (alreadySigned) throw IllegalStateException("Cannot write to already signed object")</ID>
<ID>EmptyIfBlock:InMemoryIdentityService.kt$InMemoryIdentityService${ }</ID>
<ID>EmptyKtFile:KryoHook.kt$.KryoHook.kt</ID>
@ -331,7 +325,6 @@
<ID>ForbiddenComment:DigitalSignatureWithCert.kt$// TODO: Rename this to DigitalSignature.WithCert once we're happy for it to be public API. The methods will need documentation</ID>
<ID>ForbiddenComment:DriverDSLImpl.kt$DriverDSLImpl$// TODO: Derive name from the full picked name, don't just wrap the common name</ID>
<ID>ForbiddenComment:DriverDSLImpl.kt$DriverDSLImpl$//TODO: remove this once we can bundle quasar properly.</ID>
<ID>ForbiddenComment:DriverDSLImpl.kt$DriverDSLImpl.Companion$// TODO: There is pending work to fix this issue without custom blacklisting. See: https://r3-cev.atlassian.net/browse/CORDA-2164.</ID>
<ID>ForbiddenComment:DriverDSLImpl.kt$DriverDSLImpl.LocalNetworkMap$// TODO: this object will copy NodeInfo files from started nodes to other nodes additional-node-infos/</ID>
<ID>ForbiddenComment:DummyFungibleContract.kt$DummyFungibleContract$// TODO: This doesn't work with the trader demo, so use the underlying key instead</ID>
<ID>ForbiddenComment:E2ETestKeyManagementService.kt$E2ETestKeyManagementService$// TODO: A full KeyManagementService implementation needs to record activity to the Audit Service and to limit</ID>
@ -388,6 +381,7 @@
<ID>ForbiddenComment:LegalNameValidator.kt$LegalNameValidator.Rule.Companion$// TODO: Implement confusable character detection if we add more scripts.</ID>
<ID>ForbiddenComment:LocalTypeInformationBuilder.kt$// TODO: Revisit this when Kotlin issue is fixed.</ID>
<ID>ForbiddenComment:LoggingBuyerFlow.kt$LoggingBuyerFlow$// TODO: This is potentially very expensive, and requires transaction details we may no longer have once</ID>
<ID>ForbiddenComment:LoopbackBridgeManager.kt$LoopbackBridgeManager.LoopbackBridge$// TODO: refactor MDC support, duplicated in AMQPBridgeManager.</ID>
<ID>ForbiddenComment:MockServices.kt$MockServices.Companion$// TODO: Can we use an X509 principal generator here?</ID>
<ID>ForbiddenComment:NetParams.kt$NetParamsSigner$// TODO: not supported</ID>
<ID>ForbiddenComment:NetworkBootstrapper.kt$NetworkBootstrapper$// TODO: pass a commandline parameter to the bootstrapper instead. Better yet, a notary config map</ID>
@ -630,8 +624,6 @@
<ID>FunctionNaming:VersionExtractorTest.kt$VersionExtractorTest$@Test(timeout=300_000) fun version_header_extraction_no_metadata()</ID>
<ID>FunctionNaming:VersionExtractorTest.kt$VersionExtractorTest$@Test(timeout=300_000) fun version_header_extraction_no_value()</ID>
<ID>FunctionNaming:VersionExtractorTest.kt$VersionExtractorTest$@Test(timeout=300_000) fun version_header_extraction_present()</ID>
<ID>FunctionNaming:VersionedParsingExampleTest.kt$VersionedParsingExampleTest$@Test(timeout=300_000) fun correct_parsing_function_is_used_for_present_version()</ID>
<ID>FunctionNaming:VersionedParsingExampleTest.kt$VersionedParsingExampleTest$@Test(timeout=300_000) fun default_value_is_used_for_absent_version()</ID>
<ID>LargeClass:AbstractNode.kt$AbstractNode&lt;S&gt; : SingletonSerializeAsToken</ID>
<ID>LargeClass:SingleThreadedStateMachineManager.kt$SingleThreadedStateMachineManager : StateMachineManagerStateMachineManagerInternal</ID>
<ID>LongMethod:FlowCookbook.kt$InitiatorFlow$@Suppress("RemoveExplicitTypeArguments") @Suspendable override fun call()</ID>
@ -655,7 +647,6 @@
<ID>LongParameterList:CertificateRevocationListNodeTests.kt$CertificateRevocationListNodeTests$(port: Int, name: CordaX500Name = ALICE_NAME, crlCheckSoftFail: Boolean, nodeCrlDistPoint: String = "http://${server.hostAndPort}/crl/node.crl", tlsCrlDistPoint: String? = "http://${server.hostAndPort}/crl/empty.crl", maxMessageSize: Int = MAX_MESSAGE_SIZE)</ID>
<ID>LongParameterList:CertificateRevocationListNodeTests.kt$CertificateRevocationListNodeTests.Companion$(clrServer: CrlServer, signatureAlgorithm: String, caCertificate: X509Certificate, caPrivateKey: PrivateKey, endpoint: String, indirect: Boolean, vararg serialNumbers: BigInteger)</ID>
<ID>LongParameterList:CertificateStoreStubs.kt$CertificateStoreStubs.P2P.Companion$(baseDirectory: Path, certificatesDirectoryName: String = DEFAULT_CERTIFICATES_DIRECTORY_NAME, keyStoreFileName: String = KeyStore.DEFAULT_STORE_FILE_NAME, keyStorePassword: String = KeyStore.DEFAULT_STORE_PASSWORD, keyPassword: String = keyStorePassword, trustStoreFileName: String = TrustStore.DEFAULT_STORE_FILE_NAME, trustStorePassword: String = TrustStore.DEFAULT_STORE_PASSWORD)</ID>
<ID>LongParameterList:CertificateStoreStubs.kt$CertificateStoreStubs.P2P.Companion$(certificatesDirectory: Path, keyStoreFileName: String = KeyStore.DEFAULT_STORE_FILE_NAME, keyStorePassword: String = KeyStore.DEFAULT_STORE_PASSWORD, keyPassword: String = keyStorePassword, trustStoreFileName: String = TrustStore.DEFAULT_STORE_FILE_NAME, trustStorePassword: String = TrustStore.DEFAULT_STORE_PASSWORD, trustStoreKeyPassword: String = TrustStore.DEFAULT_KEY_PASSWORD, @Suppress("UNUSED_PARAMETER") useOpenSsl: Boolean = false)</ID>
<ID>LongParameterList:ContractAttachment.kt$ContractAttachment.Companion$(attachment: Attachment, contract: ContractClassName, additionalContracts: Set&lt;ContractClassName&gt; = emptySet(), uploader: String? = null, signerKeys: List&lt;PublicKey&gt; = emptyList(), version: Int = DEFAULT_CORDAPP_VERSION)</ID>
<ID>LongParameterList:ContractFunctions.kt$(expiry: String, notional: BigDecimal, strike: BigDecimal, foreignCurrency: Currency, domesticCurrency: Currency, partyA: Party, partyB: Party)</ID>
<ID>LongParameterList:ContractFunctions.kt$(expiry: String, notional: Long, strike: Double, foreignCurrency: Currency, domesticCurrency: Currency, partyA: Party, partyB: Party)</ID>
@ -925,7 +916,6 @@
<ID>MagicNumber:IrsDemoWebApplication.kt$IrsDemoWebApplication$1000</ID>
<ID>MagicNumber:JarScanningCordappLoader.kt$CordappLoaderTemplate$36</ID>
<ID>MagicNumber:JarScanningCordappLoader.kt$CordappLoaderTemplate$64</ID>
<ID>MagicNumber:JarScanningCordappLoader.kt$JarScanningCordappLoader$1000</ID>
<ID>MagicNumber:JarSignatureCollector.kt$JarSignatureCollector$1024</ID>
<ID>MagicNumber:JarSignatureTestUtils.kt$JarSignatureTestUtils$14</ID>
<ID>MagicNumber:KMSUtils.kt$3650</ID>
@ -1229,8 +1219,6 @@
<ID>MatchingDeclarationName:Query.kt$net.corda.webserver.api.Query.kt</ID>
<ID>MatchingDeclarationName:ReceiveAllFlowTests.kt$net.corda.coretests.flows.ReceiveAllFlowTests.kt</ID>
<ID>MatchingDeclarationName:ReferenceInputStateTests.kt$net.corda.coretests.transactions.ReferenceInputStateTests.kt</ID>
<ID>MatchingDeclarationName:RigorousMock.kt$net.corda.testing.internal.RigorousMock.kt</ID>
<ID>MatchingDeclarationName:RpcServerCordaFutureSerialiser.kt$net.corda.node.serialization.amqp.RpcServerCordaFutureSerialiser.kt</ID>
<ID>MatchingDeclarationName:SSLHelper.kt$net.corda.nodeapi.internal.protonwrapper.netty.SSLHelper.kt</ID>
<ID>MatchingDeclarationName:SampleData.kt$net.corda.deterministic.verifier.SampleData.kt</ID>
<ID>MatchingDeclarationName:SerializationHelper.kt$net.corda.networkbuilder.serialization.SerializationHelper.kt</ID>
@ -1285,7 +1273,6 @@
<ID>NestedBlockDepth:RPCClientProxyHandler.kt$RPCClientProxyHandler$// The handler for Artemis messages. private fun artemisMessageHandler(message: ClientMessage)</ID>
<ID>NestedBlockDepth:ShutdownManager.kt$ShutdownManager$fun shutdown()</ID>
<ID>NestedBlockDepth:SpringDriver.kt$SpringBootDriverDSL$private fun queryWebserver(handle: NodeHandle, process: Process, checkUrl: String): WebserverHandle</ID>
<ID>NestedBlockDepth:StartedFlowTransition.kt$StartedFlowTransition$private fun TransitionBuilder.sendToSessionsTransition(sourceSessionIdToMessage: Map&lt;SessionId, SerializedBytes&lt;Any&gt;&gt;)</ID>
<ID>NestedBlockDepth:StatusTransitions.kt$StatusTransitions$ fun verify(tx: LedgerTransaction)</ID>
<ID>NestedBlockDepth:ThrowableSerializer.kt$ThrowableSerializer$override fun fromProxy(proxy: ThrowableProxy): Throwable</ID>
<ID>NestedBlockDepth:TransactionVerifierServiceInternal.kt$Verifier$ private fun verifyConstraintsValidity(contractAttachmentsByContract: Map&lt;ContractClassName, ContractAttachment&gt;)</ID>
@ -1316,7 +1303,6 @@
<ID>SpreadOperator:ConfigUtilities.kt$(*pairs)</ID>
<ID>SpreadOperator:Configuration.kt$Configuration.Validation.Error$(*(containingPath.toList() + this.containingPath).toTypedArray())</ID>
<ID>SpreadOperator:ContractJarTestUtils.kt$ContractJarTestUtils$(jarName, *contractNames.map{ "${it.replace(".", "/")}.class" }.toTypedArray())</ID>
<ID>SpreadOperator:CordaCliWrapper.kt$(RunLast().useOut(System.out).useAnsi(defaultAnsiMode), DefaultExceptionHandler&lt;List&lt;Any&gt;&gt;().useErr(System.err).useAnsi(defaultAnsiMode).andExit(ExitCodes.FAILURE), *args)</ID>
<ID>SpreadOperator:CordaRPCOpsImpl.kt$CordaRPCOpsImpl$(logicType, context(), *args)</ID>
<ID>SpreadOperator:CordaX500Name.kt$CordaX500Name.Companion$(*Locale.getISOCountries(), unspecifiedCountry)</ID>
<ID>SpreadOperator:CustomCordapp.kt$CustomCordapp$(*classes.map { it.name }.toTypedArray())</ID>
@ -1326,7 +1312,6 @@
<ID>SpreadOperator:DockerInstantiator.kt$DockerInstantiator$(*it.toTypedArray())</ID>
<ID>SpreadOperator:DummyContract.kt$DummyContract.Companion$( /* INPUTS */ *priors.toTypedArray(), /* COMMAND */ Command(cmd, priorState.owner.owningKey), /* OUTPUT */ StateAndContract(state, PROGRAM_ID) )</ID>
<ID>SpreadOperator:DummyContract.kt$DummyContract.Companion$(*items)</ID>
<ID>SpreadOperator:DummyContractV2.kt$DummyContractV2.Companion$( /* INPUTS */ *priors.toTypedArray(), /* COMMAND */ Command(cmd, priorState.owners.map { it.owningKey }), /* OUTPUT */ StateAndContract(state, DummyContractV2.PROGRAM_ID) )</ID>
<ID>SpreadOperator:ExceptionsErrorCodeFunctions.kt$(*fields)</ID>
<ID>SpreadOperator:ExceptionsErrorCodeFunctions.kt$(*fields, cause.staticLocationBasedHash(hashedFields, visited + cause))</ID>
<ID>SpreadOperator:ExceptionsErrorCodeFunctions.kt$(*hashedFields.invoke(this))</ID>
@ -1491,7 +1476,6 @@
<ID>TooGenericExceptionCaught:BankOfCordaWebApi.kt$BankOfCordaWebApi$e: Exception</ID>
<ID>TooGenericExceptionCaught:BlobInspector.kt$BlobInspector$e: Exception</ID>
<ID>TooGenericExceptionCaught:BootstrapperView.kt$BootstrapperView$e: Exception</ID>
<ID>TooGenericExceptionCaught:BridgeControlListener.kt$BridgeControlListener$ex: Exception</ID>
<ID>TooGenericExceptionCaught:BrokerJaasLoginModule.kt$BrokerJaasLoginModule$e: Exception</ID>
<ID>TooGenericExceptionCaught:CertRole.kt$CertRole.Companion$ex: ArrayIndexOutOfBoundsException</ID>
<ID>TooGenericExceptionCaught:CheckpointAgent.kt$CheckpointAgent.Companion$e: Exception</ID>
@ -1506,9 +1490,6 @@
<ID>TooGenericExceptionCaught:ContractUpgradeTransactions.kt$ContractUpgradeWireTransaction$e: Exception</ID>
<ID>TooGenericExceptionCaught:CordaAuthenticationPlugin.kt$CordaAuthenticationPlugin$e: Exception</ID>
<ID>TooGenericExceptionCaught:CordaClassResolver.kt$LoggingWhitelist.Companion$ioEx: Exception</ID>
<ID>TooGenericExceptionCaught:CordaFutureImpl.kt$CordaFutureImpl$e: Exception</ID>
<ID>TooGenericExceptionCaught:CordaFutureImpl.kt$ValueOrException$e: Exception</ID>
<ID>TooGenericExceptionCaught:CordaFutureImpl.kt$e: Exception</ID>
<ID>TooGenericExceptionCaught:CordaPersistence.kt$CordaPersistence$e: Exception</ID>
<ID>TooGenericExceptionCaught:CordaRPCClientTest.kt$CordaRPCClientTest$e: Exception</ID>
<ID>TooGenericExceptionCaught:CordaRPCOpsImpl.kt$CordaRPCOpsImpl$e: Exception</ID>
@ -1644,7 +1625,6 @@
<ID>TooGenericExceptionCaught:ValidatingNotaryFlow.kt$ValidatingNotaryFlow$e: Exception</ID>
<ID>TooGenericExceptionCaught:VaultStateMigration.kt$VaultStateIterator$e: Exception</ID>
<ID>TooGenericExceptionCaught:VaultStateMigration.kt$VaultStateMigration$e: Exception</ID>
<ID>TooGenericExceptionCaught:VersionedParsingExampleTest.kt$VersionedParsingExampleTest.RpcSettingsSpec$e: Exception</ID>
<ID>TooGenericExceptionCaught:WebServer.kt$WebServer$e: Exception</ID>
<ID>TooGenericExceptionCaught:WebServer.kt$e: Exception</ID>
<ID>TooGenericExceptionCaught:WebServer.kt$ex: Exception</ID>
@ -1699,7 +1679,6 @@
<ID>TooManyFunctions:CryptoUtils.kt$net.corda.core.crypto.CryptoUtils.kt</ID>
<ID>TooManyFunctions:Currencies.kt$net.corda.finance.Currencies.kt</ID>
<ID>TooManyFunctions:Driver.kt$DriverParameters</ID>
<ID>TooManyFunctions:DriverDSLImpl.kt$DriverDSLImpl : InternalDriverDSL</ID>
<ID>TooManyFunctions:EncodingUtils.kt$net.corda.core.utilities.EncodingUtils.kt</ID>
<ID>TooManyFunctions:FlowLogic.kt$FlowLogic&lt;out T&gt;</ID>
<ID>TooManyFunctions:FlowStateMachineImpl.kt$FlowStateMachineImpl&lt;R&gt; : FiberFlowStateMachineFlowFiber</ID>
@ -1885,8 +1864,6 @@
<ID>VariableNaming:VaultQueryTests.kt$VaultQueryTestsBase$// Beware: do not use `MyContractClass::class.qualifiedName` as this returns a fully qualified name using "dot" notation for enclosed class val MYCONTRACT_ID = "net.corda.node.services.vault.VaultQueryTestsBase\$MyContractClass"</ID>
<ID>VariableNaming:ZeroCouponBond.kt$ZeroCouponBond$val TEST_TX_TIME_1: Instant get() = Instant.parse("2017-09-02T12:00:00.00Z")</ID>
<ID>WildcardImport:AMQPClient.kt$import io.netty.channel.*</ID>
<ID>WildcardImport:AMQPClientSerializationScheme.kt$import net.corda.serialization.internal.*</ID>
<ID>WildcardImport:AMQPClientSerializationScheme.kt$import net.corda.serialization.internal.amqp.*</ID>
<ID>WildcardImport:AMQPRemoteTypeModel.kt$import net.corda.serialization.internal.model.*</ID>
<ID>WildcardImport:AMQPSerializationScheme.kt$import net.corda.core.serialization.*</ID>
<ID>WildcardImport:AMQPServerSerializationScheme.kt$import net.corda.serialization.internal.amqp.*</ID>
@ -2023,9 +2000,6 @@
<ID>WildcardImport:CordaModule.kt$import net.corda.core.crypto.*</ID>
<ID>WildcardImport:CordaModule.kt$import net.corda.core.identity.*</ID>
<ID>WildcardImport:CordaModule.kt$import net.corda.core.transactions.*</ID>
<ID>WildcardImport:CordaRPCClientTest.kt$import net.corda.core.context.*</ID>
<ID>WildcardImport:CordaRPCClientTest.kt$import net.corda.core.messaging.*</ID>
<ID>WildcardImport:CordaRPCClientTest.kt$import net.corda.testing.core.*</ID>
<ID>WildcardImport:CordaRPCOps.kt$import net.corda.core.node.services.vault.*</ID>
<ID>WildcardImport:CordaRPCOpsImplTest.kt$import net.corda.core.messaging.*</ID>
<ID>WildcardImport:CordaRPCOpsImplTest.kt$import org.assertj.core.api.Assertions.*</ID>
@ -2121,8 +2095,6 @@
<ID>WildcardImport:FlowStateMachineImpl.kt$import net.corda.core.flows.*</ID>
<ID>WildcardImport:FlowStateMachineImpl.kt$import net.corda.core.internal.*</ID>
<ID>WildcardImport:FlowsDrainingModeContentionTest.kt$import net.corda.core.flows.*</ID>
<ID>WildcardImport:FxTransactionBuildTutorial.kt$import net.corda.core.contracts.*</ID>
<ID>WildcardImport:FxTransactionBuildTutorial.kt$import net.corda.core.flows.*</ID>
<ID>WildcardImport:FxTransactionBuildTutorialTest.kt$import net.corda.finance.*</ID>
<ID>WildcardImport:GenericsTests.kt$import net.corda.serialization.internal.amqp.testutils.*</ID>
<ID>WildcardImport:Gui.kt$import tornadofx.*</ID>
@ -2168,10 +2140,7 @@
<ID>WildcardImport:InternalMockNetwork.kt$import net.corda.core.internal.*</ID>
<ID>WildcardImport:InternalMockNetwork.kt$import net.corda.node.services.config.*</ID>
<ID>WildcardImport:InternalMockNetwork.kt$import net.corda.testing.node.*</ID>
<ID>WildcardImport:InternalSerializationTestHelpers.kt$import net.corda.serialization.internal.*</ID>
<ID>WildcardImport:InternalTestUtils.kt$import net.corda.core.contracts.*</ID>
<ID>WildcardImport:InternalUtils.kt$import java.security.cert.*</ID>
<ID>WildcardImport:InternalUtils.kt$import net.corda.core.crypto.*</ID>
<ID>WildcardImport:IssuerModel.kt$import tornadofx.*</ID>
<ID>WildcardImport:JVMConfig.kt$import tornadofx.*</ID>
<ID>WildcardImport:JacksonSupport.kt$import com.fasterxml.jackson.core.*</ID>
@ -2249,7 +2218,6 @@
<ID>WildcardImport:NetworkBootstrapper.kt$import net.corda.nodeapi.internal.*</ID>
<ID>WildcardImport:NetworkBootstrapperRunnerTests.kt$import org.junit.*</ID>
<ID>WildcardImport:NetworkBootstrapperTest.kt$import net.corda.core.internal.*</ID>
<ID>WildcardImport:NetworkBootstrapperTest.kt$import net.corda.testing.core.*</ID>
<ID>WildcardImport:NetworkBuilder.kt$import net.corda.networkbuilder.nodes.*</ID>
<ID>WildcardImport:NetworkIdentityModel.kt$import net.corda.client.jfx.utils.*</ID>
<ID>WildcardImport:NetworkMapServer.kt$import javax.ws.rs.*</ID>
@ -2362,7 +2330,6 @@
<ID>WildcardImport:PathUtils.kt$import java.io.*</ID>
<ID>WildcardImport:PathUtils.kt$import java.nio.file.*</ID>
<ID>WildcardImport:PersistentIdentityMigrationNewTableTest.kt$import net.corda.testing.core.*</ID>
<ID>WildcardImport:PersistentIdentityServiceTests.kt$import net.corda.testing.core.*</ID>
<ID>WildcardImport:PersistentNetworkMapCacheTest.kt$import net.corda.testing.core.*</ID>
<ID>WildcardImport:PersistentStateServiceTests.kt$import net.corda.core.contracts.*</ID>
<ID>WildcardImport:Portfolio.kt$import net.corda.core.contracts.*</ID>
@ -2387,8 +2354,6 @@
<ID>WildcardImport:QueryCriteriaUtils.kt$import net.corda.core.node.services.vault.LikenessOperator.*</ID>
<ID>WildcardImport:RPCMultipleInterfacesTests.kt$import org.junit.Assert.*</ID>
<ID>WildcardImport:RPCSecurityManagerImpl.kt$import org.apache.shiro.authc.*</ID>
<ID>WildcardImport:RPCServer.kt$import net.corda.core.utilities.*</ID>
<ID>WildcardImport:RPCServer.kt$import org.apache.activemq.artemis.api.core.client.*</ID>
<ID>WildcardImport:ReceiveFinalityFlowTest.kt$import net.corda.node.services.statemachine.StaffedFlowHospital.*</ID>
<ID>WildcardImport:ReceiveFinalityFlowTest.kt$import net.corda.testing.node.internal.*</ID>
<ID>WildcardImport:ReceiveTransactionFlow.kt$import net.corda.core.contracts.*</ID>
@ -2427,7 +2392,6 @@
<ID>WildcardImport:SearchField.kt$import tornadofx.*</ID>
<ID>WildcardImport:SecureHashTest.kt$import org.junit.Assert.*</ID>
<ID>WildcardImport:SendTransactionFlow.kt$import net.corda.core.internal.*</ID>
<ID>WildcardImport:SerializationEnvironmentRule.kt$import net.corda.testing.internal.*</ID>
<ID>WildcardImport:SerializationHelper.kt$import java.lang.reflect.*</ID>
<ID>WildcardImport:SerializationHelper.kt$import net.corda.core.serialization.*</ID>
<ID>WildcardImport:SerializationOutputTests.kt$import java.time.*</ID>
@ -2572,7 +2536,6 @@
<ID>WildcardImport:VaultWithCashTest.kt$import net.corda.testing.core.*</ID>
<ID>WildcardImport:VaultWithCashTest.kt$import net.corda.testing.internal.vault.*</ID>
<ID>WildcardImport:VerifyTransactionTest.kt$import net.corda.finance.contracts.asset.Cash.Commands.*</ID>
<ID>WildcardImport:VersionedParsingExampleTest.kt$import net.corda.common.configuration.parsing.internal.*</ID>
<ID>WildcardImport:WebServerController.kt$import tornadofx.*</ID>
<ID>WildcardImport:WhitelistBasedTypeModelConfiguration.kt$import org.apache.qpid.proton.amqp.*</ID>
<ID>WildcardImport:WhitelistGenerator.kt$import net.corda.core.internal.*</ID>
@ -2581,8 +2544,6 @@
<ID>WildcardImport:WireTransaction.kt$import net.corda.core.internal.*</ID>
<ID>WildcardImport:WithFinality.kt$import net.corda.core.flows.*</ID>
<ID>WildcardImport:WithMockNet.kt$import com.natpryce.hamkrest.*</ID>
<ID>WildcardImport:WorkflowTransactionBuildTutorial.kt$import net.corda.core.contracts.*</ID>
<ID>WildcardImport:WorkflowTransactionBuildTutorial.kt$import net.corda.core.flows.*</ID>
<ID>WildcardImport:X509CRLSerializer.kt$import net.corda.serialization.internal.amqp.*</ID>
<ID>WildcardImport:X509CertificateSerializer.kt$import net.corda.serialization.internal.amqp.*</ID>
<ID>WildcardImport:X509EdDSAEngine.kt$import java.security.*</ID>

2
node-api/build.gradle
View File

@ -19,6 +19,8 @@ dependencies {
compile "org.apache.activemq:artemis-core-client:${artemis_version}"
compile "org.apache.activemq:artemis-commons:${artemis_version}"
compile "io.netty:netty-handler-proxy:$netty_version"
// TypeSafe Config: for simple and human friendly config files.
compile "com.typesafe:config:$typesafe_config_version"

47
node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisMessagingClient.kt
View File

@ -4,6 +4,9 @@ import net.corda.core.serialization.internal.nodeSerializationEnv
import net.corda.core.utilities.NetworkHostAndPort
import net.corda.core.utilities.loggerFor
import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.NODE_P2P_USER
import net.corda.nodeapi.internal.ArtemisTcpTransport.Companion.p2pConnectorTcpTransport
import net.corda.nodeapi.internal.ArtemisTcpTransport.Companion.p2pConnectorTcpTransportFromList
import net.corda.nodeapi.internal.config.MessagingServerConnectionConfiguration
import net.corda.nodeapi.internal.config.MutualSslConfiguration
import org.apache.activemq.artemis.api.core.client.*
import org.apache.activemq.artemis.api.core.client.ActiveMQClient.DEFAULT_ACK_BATCH_SIZE
@ -17,28 +20,55 @@ interface ArtemisSessionProvider {
class ArtemisMessagingClient(private val config: MutualSslConfiguration,
private val serverAddress: NetworkHostAndPort,
private val maxMessageSize: Int,
private val failoverCallback: ((FailoverEventType) -> Unit)? = null) : ArtemisSessionProvider {
private val autoCommitSends: Boolean = true,
private val autoCommitAcks: Boolean = true,
private val confirmationWindowSize: Int = -1,
private val messagingServerConnectionConfig: MessagingServerConnectionConfiguration? = null,
private val backupServerAddressPool: List<NetworkHostAndPort> = emptyList(),
private val failoverCallback: ((FailoverEventType) -> Unit)? = null
) : ArtemisSessionProvider {
companion object {
private val log = loggerFor<ArtemisMessagingClient>()
const val CORDA_ARTEMIS_CALL_TIMEOUT_PROP_NAME = "net.corda.nodeapi.artemismessagingclient.CallTimeout"
const val CORDA_ARTEMIS_CALL_TIMEOUT_DEFAULT = 5000L
}
class Started(val sessionFactory: ClientSessionFactory, val session: ClientSession, val producer: ClientProducer)
class Started(val serverLocator: ServerLocator, val sessionFactory: ClientSessionFactory, val session: ClientSession, val producer: ClientProducer)
override var started: Started? = null
private set
override fun start(): Started = synchronized(this) {
check(started == null) { "start can't be called twice" }
val tcpTransport = p2pConnectorTcpTransport(serverAddress, config)
val backupTransports = p2pConnectorTcpTransportFromList(backupServerAddressPool, config)
log.info("Connecting to message broker: $serverAddress")
// TODO Add broker CN to config for host verification in case the embedded broker isn't used
val tcpTransport = ArtemisTcpTransport.p2pConnectorTcpTransport(serverAddress, config)
val locator = ActiveMQClient.createServerLocatorWithoutHA(tcpTransport).apply {
if (backupTransports.isNotEmpty()) {
log.info("Back-up message broker addresses: $backupServerAddressPool")
}
// If back-up artemis addresses are configured, the locator will be created using HA mode.
@Suppress("SpreadOperator")
val locator = ActiveMQClient.createServerLocator(backupTransports.isNotEmpty(), *(listOf(tcpTransport) + backupTransports).toTypedArray()).apply {
// Never time out on our loopback Artemis connections. If we switch back to using the InVM transport this
// would be the default and the two lines below can be deleted.
connectionTTL = 60000
clientFailureCheckPeriod = 30000
callFailoverTimeout = java.lang.Long.getLong(CORDA_ARTEMIS_CALL_TIMEOUT_PROP_NAME, CORDA_ARTEMIS_CALL_TIMEOUT_DEFAULT)
callTimeout = java.lang.Long.getLong(CORDA_ARTEMIS_CALL_TIMEOUT_PROP_NAME, CORDA_ARTEMIS_CALL_TIMEOUT_DEFAULT)
minLargeMessageSize = maxMessageSize
isUseGlobalPools = nodeSerializationEnv != null
confirmationWindowSize = this@ArtemisMessagingClient.confirmationWindowSize
producerWindowSize = -1
messagingServerConnectionConfig?.let {
connectionLoadBalancingPolicyClassName = RoundRobinConnectionPolicy::class.java.canonicalName
reconnectAttempts = messagingServerConnectionConfig.reconnectAttempts(isHA)
retryInterval = messagingServerConnectionConfig.retryInterval().toMillis()
retryIntervalMultiplier = messagingServerConnectionConfig.retryIntervalMultiplier()
maxRetryInterval = messagingServerConnectionConfig.maxRetryInterval(isHA).toMillis()
isFailoverOnInitialConnection = messagingServerConnectionConfig.failoverOnInitialAttempt(isHA)
initialConnectAttempts = messagingServerConnectionConfig.initialConnectAttempts(isHA)
}
addIncomingInterceptor(ArtemisMessageSizeChecksInterceptor(maxMessageSize))
}
val sessionFactory = locator.createSessionFactory()
@ -50,23 +80,24 @@ class ArtemisMessagingClient(private val config: MutualSslConfiguration,
// using our TLS certificate.
// Note that the acknowledgement of messages is not flushed to the Artermis journal until the default buffer
// size of 1MB is acknowledged.
val session = sessionFactory!!.createSession(NODE_P2P_USER, NODE_P2P_USER, false, true, true, false, DEFAULT_ACK_BATCH_SIZE)
val session = sessionFactory!!.createSession(NODE_P2P_USER, NODE_P2P_USER, false, autoCommitSends, autoCommitAcks, false, DEFAULT_ACK_BATCH_SIZE)
session.start()
// Create a general purpose producer.
val producer = session.createProducer()
return Started(sessionFactory, session, producer).also { started = it }
return Started(locator, sessionFactory, session, producer).also { started = it }
}
override fun stop() = synchronized(this) {
started?.run {
producer.close()
// Since we are leaking the session outside of this class it may well be already closed.
if(!session.isClosed) {
if (session.stillOpen()) {
// Ensure any trailing messages are committed to the journal
session.commit()
}
// Closing the factory closes all the sessions it produced as well.
sessionFactory.close()
serverLocator.close()
}
started = null
}

6
node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisMessagingComponent.kt
View File

@ -34,6 +34,7 @@ class ArtemisMessagingComponent {
// This is a rough guess on the extra space needed on top of maxMessageSize to store the journal.
// TODO: we might want to make this value configurable.
const val JOURNAL_HEADER_SIZE = 1024
object P2PMessagingHeaders {
// This is a "property" attached to an Artemis MQ message object, which contains our own notion of "topic".
// We should probably try to unify our notion of "topic" (really, just a string that identifies an endpoint
@ -123,6 +124,11 @@ class ArtemisMessagingComponent {
require(address.startsWith(PEERS_PREFIX)) { "Failed to map address: $address to a remote topic as it is not in the $PEERS_PREFIX namespace" }
return P2P_PREFIX + address.substring(PEERS_PREFIX.length)
}
fun translateInboxAddressToLocalQueue(address: String): String {
require(address.startsWith(P2P_PREFIX)) { "Failed to map topic: $address to a local address as it is not in the $P2P_PREFIX namespace" }
return PEERS_PREFIX + address.substring(P2P_PREFIX.length)
}
}
override val queueName: String = "$P2P_PREFIX${identity.toStringShort()}"

31
node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisTcpTransport.kt
View File

@ -100,35 +100,43 @@ class ArtemisTcpTransport {
fun p2pAcceptorTcpTransport(hostAndPort: NetworkHostAndPort, config: MutualSslConfiguration?, enableSSL: Boolean = true): TransportConfiguration {
return p2pAcceptorTcpTransport(hostAndPort, config?.keyStore, config?.trustStore, enableSSL = enableSSL)
return p2pAcceptorTcpTransport(hostAndPort, config?.keyStore, config?.trustStore, enableSSL = enableSSL, useOpenSsl = config?.useOpenSsl ?: false)
}
fun p2pConnectorTcpTransport(hostAndPort: NetworkHostAndPort, config: MutualSslConfiguration?, enableSSL: Boolean = true): TransportConfiguration {
fun p2pConnectorTcpTransport(hostAndPort: NetworkHostAndPort, config: MutualSslConfiguration?, enableSSL: Boolean = true, keyStoreProvider: String? = null): TransportConfiguration {
return p2pConnectorTcpTransport(hostAndPort, config?.keyStore, config?.trustStore, enableSSL = enableSSL)
return p2pConnectorTcpTransport(hostAndPort, config?.keyStore, config?.trustStore, enableSSL = enableSSL, useOpenSsl = config?.useOpenSsl ?: false, keyStoreProvider = keyStoreProvider)
}
fun p2pAcceptorTcpTransport(hostAndPort: NetworkHostAndPort, keyStore: FileBasedCertificateStoreSupplier?, trustStore: FileBasedCertificateStoreSupplier?, enableSSL: Boolean = true): TransportConfiguration {
fun p2pAcceptorTcpTransport(hostAndPort: NetworkHostAndPort, keyStore: FileBasedCertificateStoreSupplier?, trustStore: FileBasedCertificateStoreSupplier?, enableSSL: Boolean = true, useOpenSsl: Boolean = false): TransportConfiguration {
val options = defaultArtemisOptions(hostAndPort).toMutableMap()
if (enableSSL) {
options.putAll(defaultSSLOptions)
(keyStore to trustStore).addToTransportOptions(options)
options[TransportConstants.SSL_PROVIDER] = if (useOpenSsl) TransportConstants.OPENSSL_PROVIDER else TransportConstants.DEFAULT_SSL_PROVIDER
}
options[TransportConstants.HANDSHAKE_TIMEOUT] = 0 // Suppress core.server.lambda$channelActive$0 - AMQ224088 error from load balancer type connections
return TransportConfiguration(acceptorFactoryClassName, options)
}
fun p2pConnectorTcpTransport(hostAndPort: NetworkHostAndPort, keyStore: FileBasedCertificateStoreSupplier?, trustStore: FileBasedCertificateStoreSupplier?, enableSSL: Boolean = true): TransportConfiguration {
@Suppress("LongParameterList")
fun p2pConnectorTcpTransport(hostAndPort: NetworkHostAndPort, keyStore: FileBasedCertificateStoreSupplier?, trustStore: FileBasedCertificateStoreSupplier?, enableSSL: Boolean = true, useOpenSsl: Boolean = false, keyStoreProvider: String? = null): TransportConfiguration {
val options = defaultArtemisOptions(hostAndPort).toMutableMap()
if (enableSSL) {
options.putAll(defaultSSLOptions)
(keyStore to trustStore).addToTransportOptions(options)
options[TransportConstants.SSL_PROVIDER] = if (useOpenSsl) TransportConstants.OPENSSL_PROVIDER else TransportConstants.DEFAULT_SSL_PROVIDER
keyStoreProvider?.let { options.put(TransportConstants.KEYSTORE_PROVIDER_PROP_NAME, keyStoreProvider) }
}
return TransportConfiguration(connectorFactoryClassName, options)
}
fun p2pConnectorTcpTransportFromList(hostAndPortList: List<NetworkHostAndPort>, config: MutualSslConfiguration?, enableSSL: Boolean = true, keyStoreProvider: String? = null): List<TransportConfiguration> = hostAndPortList.map {
p2pConnectorTcpTransport(it, config, enableSSL, keyStoreProvider)
}
fun rpcAcceptorTcpTransport(hostAndPort: NetworkHostAndPort, config: BrokerRpcSslOptions?, enableSSL: Boolean = true): TransportConfiguration {
val options = defaultArtemisOptions(hostAndPort).toMutableMap()
@ -156,12 +164,17 @@ class ArtemisTcpTransport {
rpcConnectorTcpTransport(it, config, enableSSL)
}
fun rpcInternalClientTcpTransport(hostAndPort: NetworkHostAndPort, config: SslConfiguration): TransportConfiguration {
return TransportConfiguration(connectorFactoryClassName, defaultArtemisOptions(hostAndPort) + defaultSSLOptions + config.toTransportOptions())
fun rpcInternalClientTcpTransport(hostAndPort: NetworkHostAndPort, config: SslConfiguration, keyStoreProvider: String? = null): TransportConfiguration {
return TransportConfiguration(connectorFactoryClassName, defaultArtemisOptions(hostAndPort) + defaultSSLOptions + config.toTransportOptions() + asMap(keyStoreProvider))
}
fun rpcInternalAcceptorTcpTransport(hostAndPort: NetworkHostAndPort, config: SslConfiguration): TransportConfiguration {
return TransportConfiguration(acceptorFactoryClassName, defaultArtemisOptions(hostAndPort) + defaultSSLOptions + config.toTransportOptions() + (TransportConstants.HANDSHAKE_TIMEOUT to 0))
fun rpcInternalAcceptorTcpTransport(hostAndPort: NetworkHostAndPort, config: SslConfiguration, keyStoreProvider: String? = null): TransportConfiguration {
return TransportConfiguration(acceptorFactoryClassName, defaultArtemisOptions(hostAndPort) + defaultSSLOptions +
config.toTransportOptions() + (TransportConstants.HANDSHAKE_TIMEOUT to 0) + asMap(keyStoreProvider))
}
private fun asMap(keyStoreProvider: String?): Map<String, String> {
return keyStoreProvider?.let {mutableMapOf(TransportConstants.KEYSTORE_PROVIDER_PROP_NAME to it)} ?: emptyMap()
}
}
}

2
node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisUtils.kt
View File

@ -1,5 +1,4 @@
@file:JvmName("ArtemisUtils")
package net.corda.nodeapi.internal
import java.nio.file.FileSystems
@ -16,3 +15,4 @@ fun Path.requireOnDefaultFileSystem() {
fun requireMessageSize(messageSize: Int, limit: Int) {
require(messageSize <= limit) { "Message exceeds maxMessageSize network parameter, maxMessageSize: [$limit], message size: [$messageSize]" }
}

8
node-api/src/main/kotlin/net/corda/nodeapi/internal/ClientSessionUtils.kt Normal file
View File

@ -0,0 +1,8 @@
package net.corda.nodeapi.internal
import org.apache.activemq.artemis.api.core.client.ClientSession
import org.apache.activemq.artemis.core.client.impl.ClientSessionInternal
fun ClientSession.stillOpen(): Boolean {
return (!isClosed && (this as? ClientSessionInternal)?.isClosing != false)
}

17
node-api/src/main/kotlin/net/corda/nodeapi/internal/ConcurrentBox.kt Normal file
View File

@ -0,0 +1,17 @@
package net.corda.nodeapi.internal
import java.util.concurrent.locks.ReentrantReadWriteLock
import kotlin.concurrent.read
import kotlin.concurrent.write
/**
* A [ConcurrentBox] allows the implementation of track() with reduced contention. [concurrent] may be run from several
* threads (which means it MUST be threadsafe!), while [exclusive] stops the world until the tracking has been set up.
* Internally [ConcurrentBox] is implemented simply as a read-write lock.
*/
class ConcurrentBox<out T>(val content: T) {
val lock = ReentrantReadWriteLock()
inline fun <R> concurrent(block: T.() -> R): R = lock.read { block(content) }
inline fun <R> exclusive(block: T.() -> R): R = lock.write { block(content) }
}

18
node-api/src/main/kotlin/net/corda/nodeapi/internal/RoundRobinConnectionPolicy.kt Normal file
View File

@ -0,0 +1,18 @@
package net.corda.nodeapi.internal
import org.apache.activemq.artemis.api.core.client.loadbalance.ConnectionLoadBalancingPolicy
/**
* Implementation of an Artemis load balancing policy. It does round-robin always starting from the first position, whereas
* the current [RoundRobinConnectionLoadBalancingPolicy] in Artemis picks the starting position randomly. This can lead to
* attempting to connect to an inactive broker on the first attempt, which can cause start-up delays depending on what connection
* settings are used.
*/
class RoundRobinConnectionPolicy : ConnectionLoadBalancingPolicy {
private var pos = 0
override fun select(max: Int): Int {
pos = if (pos >= max) 0 else pos
return pos++
}
}

395
node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/AMQPBridgeManager.kt
View File

@ -1,5 +1,8 @@
@file:Suppress("TooGenericExceptionCaught") // needs to catch and handle/rethrow *all* exceptions in many places
package net.corda.nodeapi.internal.bridging
import com.google.common.util.concurrent.ThreadFactoryBuilder
import io.netty.channel.EventLoop
import io.netty.channel.EventLoopGroup
import io.netty.channel.nio.NioEventLoopGroup
import net.corda.core.identity.CordaX500Name
@ -11,11 +14,14 @@ import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.NODE_P2P_U
import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.P2PMessagingHeaders
import net.corda.nodeapi.internal.ArtemisMessagingComponent.RemoteInboxAddress.Companion.translateLocalQueueToInboxAddress
import net.corda.nodeapi.internal.ArtemisSessionProvider
import net.corda.nodeapi.internal.ArtemisConstants.MESSAGE_ID_KEY
import net.corda.nodeapi.internal.config.CertificateStore
import net.corda.nodeapi.internal.config.MutualSslConfiguration
import net.corda.nodeapi.internal.protonwrapper.messages.MessageStatus
import net.corda.nodeapi.internal.protonwrapper.netty.AMQPClient
import net.corda.nodeapi.internal.protonwrapper.netty.AMQPConfiguration
import net.corda.nodeapi.internal.protonwrapper.netty.ProxyConfig
import net.corda.nodeapi.internal.protonwrapper.netty.RevocationConfig
import org.apache.activemq.artemis.api.core.ActiveMQObjectClosedException
import org.apache.activemq.artemis.api.core.SimpleString
import org.apache.activemq.artemis.api.core.client.ActiveMQClient.DEFAULT_ACK_BATCH_SIZE
import org.apache.activemq.artemis.api.core.client.ClientConsumer
@ -23,6 +29,10 @@ import org.apache.activemq.artemis.api.core.client.ClientMessage
import org.apache.activemq.artemis.api.core.client.ClientSession
import org.slf4j.MDC
import rx.Subscription
import java.util.concurrent.Executors
import java.util.concurrent.ScheduledExecutorService
import java.util.concurrent.ScheduledFuture
import java.util.concurrent.TimeUnit
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
@ -34,33 +44,46 @@ import kotlin.concurrent.withLock
* The Netty thread pool used by the AMQPBridges is also shared and managed by the AMQPBridgeManager.
*/
@VisibleForTesting
class AMQPBridgeManager(config: MutualSslConfiguration,
maxMessageSize: Int,
crlCheckSoftFail: Boolean,
private val artemisMessageClientFactory: () -> ArtemisSessionProvider,
private val bridgeMetricsService: BridgeMetricsService? = null) : BridgeManager {
open class AMQPBridgeManager(keyStore: CertificateStore,
trustStore: CertificateStore,
useOpenSSL: Boolean,
proxyConfig: ProxyConfig? = null,
maxMessageSize: Int,
revocationConfig: RevocationConfig,
enableSNI: Boolean,
private val artemisMessageClientFactory: () -> ArtemisSessionProvider,
private val bridgeMetricsService: BridgeMetricsService? = null,
trace: Boolean,
sslHandshakeTimeout: Long?,
private val bridgeConnectionTTLSeconds: Int) : BridgeManager {
private val lock = ReentrantLock()
private val queueNamesToBridgesMap = mutableMapOf<String, MutableList<AMQPBridge>>()
private class AMQPConfigurationImpl private constructor(override val keyStore: CertificateStore,
override val trustStore: CertificateStore,
override val maxMessageSize: Int,
override val crlCheckSoftFail: Boolean) : AMQPConfiguration {
constructor(config: MutualSslConfiguration, maxMessageSize: Int, crlCheckSoftFail: Boolean) : this(config.keyStore.get(), config.trustStore.get(), maxMessageSize, crlCheckSoftFail)
private class AMQPConfigurationImpl(override val keyStore: CertificateStore,
override val trustStore: CertificateStore,
override val proxyConfig: ProxyConfig?,
override val maxMessageSize: Int,
override val revocationConfig: RevocationConfig,
override val useOpenSsl: Boolean,
override val enableSNI: Boolean,
override val sourceX500Name: String? = null,
override val trace: Boolean,
private val _sslHandshakeTimeout: Long?) : AMQPConfiguration {
override val sslHandshakeTimeout: Long
get() = _sslHandshakeTimeout ?: super.sslHandshakeTimeout
}
private val amqpConfig: AMQPConfiguration = AMQPConfigurationImpl(config, maxMessageSize, crlCheckSoftFail)
private val amqpConfig: AMQPConfiguration = AMQPConfigurationImpl(keyStore, trustStore, proxyConfig, maxMessageSize, revocationConfig,useOpenSSL, enableSNI, trace = trace, _sslHandshakeTimeout = sslHandshakeTimeout)
private var sharedEventLoopGroup: EventLoopGroup? = null
private var artemis: ArtemisSessionProvider? = null
constructor(config: MutualSslConfiguration,
p2pAddress: NetworkHostAndPort,
maxMessageSize: Int,
crlCheckSoftFail: Boolean) : this(config, maxMessageSize, crlCheckSoftFail, { ArtemisMessagingClient(config, p2pAddress, maxMessageSize) })
companion object {
private const val NUM_BRIDGE_THREADS = 0 // Default sized pool
private const val CORDA_NUM_BRIDGE_THREADS_PROP_NAME = "net.corda.nodeapi.amqpbridgemanager.NumBridgeThreads"
private val NUM_BRIDGE_THREADS = Integer.getInteger(CORDA_NUM_BRIDGE_THREADS_PROP_NAME, 0) // Default 0 means Netty default sized pool
private const val ARTEMIS_RETRY_BACKOFF = 5000L
}
/**
@ -71,13 +94,16 @@ class AMQPBridgeManager(config: MutualSslConfiguration,
* If the delivery fails the session is rolled back to prevent loss of the message. This may cause duplicate delivery,
* however Artemis and the remote Corda instanced will deduplicate these messages.
*/
private class AMQPBridge(val queueName: String,
@Suppress("TooManyFunctions")
private class AMQPBridge(val sourceX500Name: String,
val queueName: String,
val targets: List<NetworkHostAndPort>,
val legalNames: Set<CordaX500Name>,
private val amqpConfig: AMQPConfiguration,
sharedEventGroup: EventLoopGroup,
private val artemis: ArtemisSessionProvider,
private val bridgeMetricsService: BridgeMetricsService?) {
private val bridgeMetricsService: BridgeMetricsService?,
private val bridgeConnectionTTLSeconds: Int) {
companion object {
private val log = contextLogger()
}
@ -86,6 +112,7 @@ class AMQPBridgeManager(config: MutualSslConfiguration,
val oldMDC = MDC.getCopyOfContextMap() ?: emptyMap<String, String>()
try {
MDC.put("queueName", queueName)
MDC.put("source", amqpConfig.sourceX500Name)
MDC.put("targets", targets.joinToString(separator = ";") { it.toString() })
MDC.put("legalNames", legalNames.joinToString(separator = ";") { it.toString() })
MDC.put("maxMessageSize", amqpConfig.maxMessageSize.toString())
@ -106,10 +133,80 @@ class AMQPBridgeManager(config: MutualSslConfiguration,
private fun logWarnWithMDC(msg: String) = withMDC { log.warn(msg) }
val amqpClient = AMQPClient(targets, legalNames, amqpConfig, sharedThreadPool = sharedEventGroup)
private val lock = ReentrantLock() // lock to serialise session level access
private var session: ClientSession? = null
private var consumer: ClientConsumer? = null
private var connectedSubscription: Subscription? = null
@Volatile
private var messagesReceived: Boolean = false
private val eventLoop: EventLoop = sharedEventGroup.next()
private var artemisState: ArtemisState = ArtemisState.STOPPED
set(value) {
logDebugWithMDC { "State change $field to $value" }
field = value
}
@Suppress("MagicNumber")
private var artemisHeartbeatPlusBackoff = TimeUnit.SECONDS.toMillis(90)
private var amqpRestartEvent: ScheduledFuture<Unit>? = null
private var scheduledExecutorService: ScheduledExecutorService
= Executors.newSingleThreadScheduledExecutor(ThreadFactoryBuilder().setNameFormat("bridge-connection-reset-%d").build())
@Suppress("ClassNaming")
private sealed class ArtemisState {
object STARTING : ArtemisState()
data class STARTED(override val pending: ScheduledFuture<Unit>) : ArtemisState()
object CHECKING : ArtemisState()
object RESTARTED : ArtemisState()
object RECEIVING : ArtemisState()
object AMQP_STOPPED : ArtemisState()
object AMQP_STARTING : ArtemisState()
object AMQP_STARTED : ArtemisState()
object AMQP_RESTARTED : ArtemisState()
object STOPPING : ArtemisState()
object STOPPED : ArtemisState()
data class STOPPED_AMQP_START_SCHEDULED(override val pending: ScheduledFuture<Unit>) : ArtemisState()
open val pending: ScheduledFuture<Unit>? = null
override fun toString(): String = javaClass.simpleName
}
private fun artemis(inProgress: ArtemisState, block: (precedingState: ArtemisState) -> ArtemisState) {
val runnable = {
synchronized(artemis) {
try {
val precedingState = artemisState
artemisState.pending?.cancel(false)
artemisState = inProgress
artemisState = block(precedingState)
} catch (ex: Exception) {
withMDC { log.error("Unexpected error in Artemis processing in state $artemisState.", ex) }
}
}
}
if (eventLoop.inEventLoop()) {
runnable()
} else {
eventLoop.execute(runnable)
}
}
private fun scheduledArtemis(delay: Long, unit: TimeUnit, inProgress: ArtemisState, block: (precedingState: ArtemisState) -> ArtemisState): ScheduledFuture<Unit> {
return eventLoop.schedule<Unit>({
artemis(inProgress, block)
}, delay, unit)
}
private fun scheduledArtemisInExecutor(delay: Long, unit: TimeUnit, inProgress: ArtemisState, nextState: ArtemisState, block: () -> Unit): ScheduledFuture<Unit> {
return scheduledExecutorService.schedule<Unit>({
artemis(inProgress) {
nextState
}
block()
}, delay, unit)
}
fun start() {
logInfoWithMDC("Create new AMQP bridge")
@ -119,55 +216,196 @@ class AMQPBridgeManager(config: MutualSslConfiguration,
fun stop() {
logInfoWithMDC("Stopping AMQP bridge")
lock.withLock {
synchronized(artemis) {
consumer?.close()
consumer = null
session?.stop()
session = null
}
}
amqpClient.stop()
connectedSubscription?.unsubscribe()
connectedSubscription = null
}
private fun onSocketConnected(connected: Boolean) {
lock.withLock {
synchronized(artemis) {
if (connected) {
logInfoWithMDC("Bridge Connected")
bridgeMetricsService?.bridgeConnected(targets, legalNames)
val sessionFactory = artemis.started!!.sessionFactory
val session = sessionFactory.createSession(NODE_P2P_USER, NODE_P2P_USER, false, true, true, false, DEFAULT_ACK_BATCH_SIZE)
this.session = session
val consumer = session.createConsumer(queueName)
this.consumer = consumer
consumer.setMessageHandler(this@AMQPBridge::clientArtemisMessageHandler)
session.start()
} else {
logInfoWithMDC("Bridge Disconnected")
bridgeMetricsService?.bridgeDisconnected(targets, legalNames)
consumer?.close()
consumer = null
session?.stop()
artemis(ArtemisState.STOPPING) {
logInfoWithMDC("Stopping Artemis because stopping AMQP bridge")
closeConsumer()
consumer = null
eventLoop.execute {
artemis(ArtemisState.STOPPING) {
stopSession()
session = null
ArtemisState.STOPPED
}
}
ArtemisState.STOPPING
}
bridgeMetricsService?.bridgeDisconnected(targets, legalNames)
connectedSubscription?.unsubscribe()
connectedSubscription = null
// Do this last because we already scheduled the Artemis stop, so it's okay to unsubscribe onConnected first.
amqpClient.stop()
}
@Suppress("ComplexMethod")
private fun onSocketConnected(connected: Boolean) {
if (connected) {
logInfoWithMDC("Bridge Connected")
bridgeMetricsService?.bridgeConnected(targets, legalNames)
if (bridgeConnectionTTLSeconds > 0) {
// AMQP outbound connection will be restarted periodically with bridgeConnectionTTLSeconds interval
amqpRestartEvent = scheduledArtemisInExecutor(bridgeConnectionTTLSeconds.toLong(), TimeUnit.SECONDS,
ArtemisState.AMQP_STOPPED, ArtemisState.AMQP_RESTARTED) {
logInfoWithMDC("Bridge connection time to live exceeded. Restarting AMQP connection")
stopAndStartOutbound(ArtemisState.AMQP_RESTARTED)
}
}
artemis(ArtemisState.STARTING) {
val startedArtemis = artemis.started
if (startedArtemis == null) {
logInfoWithMDC("Bridge Connected but Artemis is disconnected")
ArtemisState.STOPPED
} else {
logInfoWithMDC("Bridge Connected so starting Artemis")
artemisHeartbeatPlusBackoff = startedArtemis.serverLocator.connectionTTL + ARTEMIS_RETRY_BACKOFF
try {
createSessionAndConsumer(startedArtemis)
ArtemisState.STARTED(scheduledArtemis(artemisHeartbeatPlusBackoff, TimeUnit.MILLISECONDS, ArtemisState.CHECKING) {
if (!messagesReceived) {
logInfoWithMDC("No messages received on new bridge. Restarting Artemis session")
if (restartSession()) {
ArtemisState.RESTARTED
} else {
logInfoWithMDC("Artemis session restart failed. Aborting by restarting AMQP connection.")
stopAndStartOutbound()
}
} else {
ArtemisState.RECEIVING
}
})
} catch (ex: Exception) {
// Now, bounce the AMQP connection to restart the sequence of establishing the connectivity back from the beginning.
withMDC { log.warn("Create Artemis start session error. Restarting AMQP connection", ex) }
stopAndStartOutbound()
}
}
}
} else {
logInfoWithMDC("Bridge Disconnected")
amqpRestartEvent?.cancel(false)
if (artemisState != ArtemisState.AMQP_STARTING && artemisState != ArtemisState.STOPPED) {
bridgeMetricsService?.bridgeDisconnected(targets, legalNames)
}
artemis(ArtemisState.STOPPING) { precedingState: ArtemisState ->
logInfoWithMDC("Stopping Artemis because AMQP bridge disconnected")
closeConsumer()
consumer = null
eventLoop.execute {
artemis(ArtemisState.STOPPING) {
stopSession()
session = null
when (precedingState) {
ArtemisState.AMQP_STOPPED ->
ArtemisState.STOPPED_AMQP_START_SCHEDULED(scheduledArtemis(artemisHeartbeatPlusBackoff,
TimeUnit.MILLISECONDS, ArtemisState.AMQP_STARTING) { startOutbound() })
ArtemisState.AMQP_RESTARTED -> {
artemis(ArtemisState.AMQP_STARTING) { startOutbound() }
ArtemisState.AMQP_STARTING
}
else -> ArtemisState.STOPPED
}
}
}
ArtemisState.STOPPING
}
}
}
private fun startOutbound(): ArtemisState {
logInfoWithMDC("Starting AMQP client")
amqpClient.start()
return ArtemisState.AMQP_STARTED
}
private fun stopAndStartOutbound(state: ArtemisState = ArtemisState.AMQP_STOPPED): ArtemisState {
amqpClient.stop()
// Bridge disconnect will detect this state and schedule an AMQP start.
return state
}
private fun createSessionAndConsumer(startedArtemis: ArtemisMessagingClient.Started): ClientSession {
logInfoWithMDC("Creating session and consumer.")
val sessionFactory = startedArtemis.sessionFactory
val session = sessionFactory.createSession(NODE_P2P_USER, NODE_P2P_USER, false, true,
true, false, DEFAULT_ACK_BATCH_SIZE)
this.session = session
// Several producers (in the case of shared bridge) can put messages in the same outbound p2p queue.
// The consumers are created using the source x500 name as a filter
val consumer = if (amqpConfig.enableSNI) {
session.createConsumer(queueName, "hyphenated_props:sender-subject-name = '${amqpConfig.sourceX500Name}'")
} else {
session.createConsumer(queueName)
}
this.consumer = consumer
session.start()
consumer.setMessageHandler(this@AMQPBridge::clientArtemisMessageHandler)
return session
}
private fun closeConsumer(): Boolean {
var closed = false
try {
consumer?.apply {
if (!isClosed) {
close()
}
}
closed = true
} catch (ex: Exception) {
withMDC { log.warn("Close artemis consumer error", ex) }
} finally {
return closed
}
}
private fun stopSession(): Boolean {
var stopped = false
try {
session?.apply {
if (!isClosed) {
stop()
}
}
stopped = true
} catch (ex: Exception) {
withMDC { log.warn("Stop Artemis session error", ex) }
} finally {
return stopped
}
}
private fun restartSession(): Boolean {
if (!stopSession()) {
// Session timed out stopping. The request/responses can be out of sequence on the session now, so abandon it.
session = null
// The consumer is also dead now too as attached to the dead session.
consumer = null
return false
}
try {
// Does not wait for a response.
this.session?.start()
} catch (ex: Exception) {
withMDC { log.error("Start Artemis session error", ex) }
}
return true
}
private fun clientArtemisMessageHandler(artemisMessage: ClientMessage) {
messagesReceived = true
if (artemisMessage.bodySize > amqpConfig.maxMessageSize) {
val msg = "Message exceeds maxMessageSize network parameter, maxMessageSize: [${amqpConfig.maxMessageSize}], message size: [${artemisMessage.bodySize}], " +
"dropping message, uuid: ${artemisMessage.getObjectProperty("_AMQ_DUPL_ID")}"
"dropping message, uuid: ${artemisMessage.getObjectProperty(MESSAGE_ID_KEY)}"
logWarnWithMDC(msg)
bridgeMetricsService?.packetDropEvent(artemisMessage, msg)
// Ack the message to prevent same message being sent to us again.
artemisMessage.individualAcknowledge()
try {
artemisMessage.individualAcknowledge()
} catch (ex: ActiveMQObjectClosedException) {
log.warn("Artemis message was closed")
}
return
}
val data = ByteArray(artemisMessage.bodySize).apply { artemisMessage.bodyBuffer.readBytes(this) }
val properties = HashMap<String, Any?>()
for (key in P2PMessagingHeaders.whitelistedHeaders) {
if (artemisMessage.containsProperty(key)) {
@ -178,18 +416,22 @@ class AMQPBridgeManager(config: MutualSslConfiguration,
properties[key] = value
}
}
logDebugWithMDC { "Bridged Send to ${legalNames.first()} uuid: ${artemisMessage.getObjectProperty("_AMQ_DUPL_ID")}" }
logDebugWithMDC { "Bridged Send to ${legalNames.first()} uuid: ${artemisMessage.getObjectProperty(MESSAGE_ID_KEY)}" }
val peerInbox = translateLocalQueueToInboxAddress(queueName)
val sendableMessage = amqpClient.createMessage(data, peerInbox,
val sendableMessage = amqpClient.createMessage(artemisMessage.payload(), peerInbox,
legalNames.first().toString(),
properties)
sendableMessage.onComplete.then {
logDebugWithMDC { "Bridge ACK ${sendableMessage.onComplete.get()}" }
lock.withLock {
eventLoop.submit {
if (sendableMessage.onComplete.get() == MessageStatus.Acknowledged) {
artemisMessage.individualAcknowledge()
try {
artemisMessage.individualAcknowledge()
} catch (ex: ActiveMQObjectClosedException) {
log.warn("Artemis message was closed")
}
} else {
logInfoWithMDC("Rollback rejected message uuid: ${artemisMessage.getObjectProperty("_AMQ_DUPL_ID")}")
logInfoWithMDC("Rollback rejected message uuid: ${artemisMessage.getObjectProperty(MESSAGE_ID_KEY)}")
// We need to commit any acknowledged messages before rolling back the failed
// (unacknowledged) message.
session?.commit()
@ -202,9 +444,9 @@ class AMQPBridgeManager(config: MutualSslConfiguration,
} catch (ex: IllegalStateException) {
// Attempting to send a message while the AMQP client is disconnected may cause message loss.
// The failed message is rolled back after committing acknowledged messages.
lock.withLock {
ex.message?.let { logInfoWithMDC(it)}
logInfoWithMDC("Rollback rejected message uuid: ${artemisMessage.getObjectProperty("_AMQ_DUPL_ID")}")
eventLoop.submit {
ex.message?.let { logInfoWithMDC(it) }
logInfoWithMDC("Rollback rejected message uuid: ${artemisMessage.getObjectProperty(MESSAGE_ID_KEY)}")
session?.commit()
session?.rollback(false)
}
@ -213,20 +455,22 @@ class AMQPBridgeManager(config: MutualSslConfiguration,
}
}
override fun deployBridge(queueName: String, targets: List<NetworkHostAndPort>, legalNames: Set<CordaX500Name>) {
val newBridge = lock.withLock {
override fun deployBridge(sourceX500Name: String, queueName: String, targets: List<NetworkHostAndPort>, legalNames: Set<CordaX500Name>) {
lock.withLock {
val bridges = queueNamesToBridgesMap.getOrPut(queueName) { mutableListOf() }
for (target in targets) {
if (bridges.any { it.targets.contains(target) }) {
if (bridges.any { it.targets.contains(target) && it.sourceX500Name == sourceX500Name }) {
return
}
}
val newBridge = AMQPBridge(queueName, targets, legalNames, amqpConfig, sharedEventLoopGroup!!, artemis!!, bridgeMetricsService)
val newAMQPConfig = with(amqpConfig) { AMQPConfigurationImpl(keyStore, trustStore, proxyConfig, maxMessageSize,
revocationConfig, useOpenSsl, enableSNI, sourceX500Name, trace, sslHandshakeTimeout) }
val newBridge = AMQPBridge(sourceX500Name, queueName, targets, legalNames, newAMQPConfig, sharedEventLoopGroup!!, artemis!!,
bridgeMetricsService, bridgeConnectionTTLSeconds)
bridges += newBridge
bridgeMetricsService?.bridgeCreated(targets, legalNames)
newBridge
}
newBridge.start()
}.start()
}
override fun destroyBridge(queueName: String, targets: List<NetworkHostAndPort>) {
@ -246,6 +490,17 @@ class AMQPBridgeManager(config: MutualSslConfiguration,
}
}
fun destroyAllBridges(queueName: String): Map<String, BridgeEntry> {
return lock.withLock {
// queueNamesToBridgesMap returns a mutable list, .toList converts it to a immutable list so it won't be changed by the [destroyBridge] method.
val bridges = queueNamesToBridgesMap[queueName]?.toList()
destroyBridge(queueName, bridges?.flatMap { it.targets } ?: emptyList())
bridges?.map {
it.sourceX500Name to BridgeEntry(it.queueName, it.targets, it.legalNames.toList(), serviceAddress = false)
}?.toMap() ?: emptyMap()
}
}
override fun start() {
sharedEventLoopGroup = NioEventLoopGroup(NUM_BRIDGE_THREADS)
val artemis = artemisMessageClientFactory()

205
node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/BridgeControlListener.kt
View File

@ -1,51 +1,124 @@
@file:Suppress("TooGenericExceptionCaught") // needs to catch and handle/rethrow *all* exceptions
package net.corda.nodeapi.internal.bridging
import net.corda.core.identity.CordaX500Name
import net.corda.core.serialization.SerializationDefaults
import net.corda.core.serialization.deserialize
import net.corda.core.serialization.serialize
import net.corda.core.utilities.NetworkHostAndPort
import net.corda.core.utilities.contextLogger
import net.corda.nodeapi.internal.ArtemisMessagingClient
import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.BRIDGE_CONTROL
import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.BRIDGE_NOTIFY
import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.P2P_PREFIX
import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.PEERS_PREFIX
import net.corda.nodeapi.internal.ArtemisSessionProvider
import net.corda.nodeapi.internal.config.CertificateStore
import net.corda.nodeapi.internal.config.MutualSslConfiguration
import net.corda.nodeapi.internal.crypto.x509
import net.corda.nodeapi.internal.protonwrapper.netty.ProxyConfig
import net.corda.nodeapi.internal.protonwrapper.netty.RevocationConfig
import org.apache.activemq.artemis.api.core.ActiveMQNonExistentQueueException
import org.apache.activemq.artemis.api.core.ActiveMQQueueExistsException
import org.apache.activemq.artemis.api.core.RoutingType
import org.apache.activemq.artemis.api.core.SimpleString
import org.apache.activemq.artemis.api.core.client.ClientConsumer
import org.apache.activemq.artemis.api.core.client.ClientMessage
import org.apache.activemq.artemis.api.core.client.ClientSession
import rx.Observable
import rx.subjects.PublishSubject
import java.util.*
class BridgeControlListener(val config: MutualSslConfiguration,
class BridgeControlListener(private val keyStore: CertificateStore,
trustStore: CertificateStore,
useOpenSSL: Boolean,
proxyConfig: ProxyConfig? = null,
maxMessageSize: Int,
crlCheckSoftFail: Boolean,
revocationConfig: RevocationConfig,
enableSNI: Boolean,
private val artemisMessageClientFactory: () -> ArtemisSessionProvider,
bridgeMetricsService: BridgeMetricsService? = null) : AutoCloseable {
bridgeMetricsService: BridgeMetricsService? = null,
trace: Boolean = false,
sslHandshakeTimeout: Long? = null,
bridgeConnectionTTLSeconds: Int = 0) : AutoCloseable {
private val bridgeId: String = UUID.randomUUID().toString()
private val bridgeManager: BridgeManager = AMQPBridgeManager(
config,
maxMessageSize,
crlCheckSoftFail,
artemisMessageClientFactory,
bridgeMetricsService)
private var bridgeControlQueue = "$BRIDGE_CONTROL.$bridgeId"
private var bridgeNotifyQueue = "$BRIDGE_NOTIFY.$bridgeId"
private val validInboundQueues = mutableSetOf<String>()
private val bridgeManager = if (enableSNI) {
LoopbackBridgeManager(keyStore, trustStore, useOpenSSL, proxyConfig, maxMessageSize, revocationConfig, enableSNI,
artemisMessageClientFactory, bridgeMetricsService, this::validateReceiveTopic, trace, sslHandshakeTimeout,
bridgeConnectionTTLSeconds)
} else {
AMQPBridgeManager(keyStore, trustStore, useOpenSSL, proxyConfig, maxMessageSize, revocationConfig, enableSNI,
artemisMessageClientFactory, bridgeMetricsService, trace, sslHandshakeTimeout, bridgeConnectionTTLSeconds)
}
private var artemis: ArtemisSessionProvider? = null
private var controlConsumer: ClientConsumer? = null
private var notifyConsumer: ClientConsumer? = null
constructor(config: MutualSslConfiguration,
p2pAddress: NetworkHostAndPort,
maxMessageSize: Int,
revocationConfig: RevocationConfig,
enableSNI: Boolean,
proxy: ProxyConfig? = null) : this(config.keyStore.get(), config.trustStore.get(), config.useOpenSsl, proxy, maxMessageSize, revocationConfig, enableSNI, { ArtemisMessagingClient(config, p2pAddress, maxMessageSize) })
companion object {
private val log = contextLogger()
}
val active: Boolean
get() = validInboundQueues.isNotEmpty()
private val _activeChange = PublishSubject.create<Boolean>().toSerialized()
val activeChange: Observable<Boolean>
get() = _activeChange
private val _failure = PublishSubject.create<BridgeControlListener>().toSerialized()
val failure: Observable<BridgeControlListener>
get() = _failure
fun start() {
stop()
bridgeManager.start()
val artemis = artemisMessageClientFactory()
this.artemis = artemis
artemis.start()
val artemisClient = artemis.started!!
val artemisSession = artemisClient.session
val bridgeControlQueue = "$BRIDGE_CONTROL.$bridgeId"
artemisSession.createTemporaryQueue(BRIDGE_CONTROL, RoutingType.MULTICAST, bridgeControlQueue)
try {
stop()
val queueDisambiguityId = UUID.randomUUID().toString()
bridgeControlQueue = "$BRIDGE_CONTROL.$queueDisambiguityId"
bridgeNotifyQueue = "$BRIDGE_NOTIFY.$queueDisambiguityId"
bridgeManager.start()
val artemis = artemisMessageClientFactory()
this.artemis = artemis
artemis.start()
val artemisClient = artemis.started!!
val artemisSession = artemisClient.session
registerBridgeControlListener(artemisSession)
registerBridgeDuplicateChecker(artemisSession)
// Attempt to read available inboxes directly from Artemis before requesting updates from connected nodes
validInboundQueues.addAll(artemisSession.addressQuery(SimpleString("$P2P_PREFIX#")).queueNames.map { it.toString() })
log.info("Found inboxes: $validInboundQueues")
if (active) {
_activeChange.onNext(true)
}
val startupMessage = BridgeControl.BridgeToNodeSnapshotRequest(bridgeId).serialize(context = SerializationDefaults.P2P_CONTEXT)
.bytes
val bridgeRequest = artemisSession.createMessage(false)
bridgeRequest.writeBodyBufferBytes(startupMessage)
artemisClient.producer.send(BRIDGE_NOTIFY, bridgeRequest)
} catch (e: Exception) {
log.error("Failure to start BridgeControlListener", e)
_failure.onNext(this)
}
}
private fun registerBridgeControlListener(artemisSession: ClientSession) {
try {
artemisSession.createTemporaryQueue(BRIDGE_CONTROL, RoutingType.MULTICAST, bridgeControlQueue)
} catch (ex: ActiveMQQueueExistsException) {
// Ignore if there is a queue still not cleaned up
}
val control = artemisSession.createConsumer(bridgeControlQueue)
controlConsumer = control
control.setMessageHandler { msg ->
@ -53,22 +126,64 @@ class BridgeControlListener(val config: MutualSslConfiguration,
processControlMessage(msg)
} catch (ex: Exception) {
log.error("Unable to process bridge control message", ex)
_failure.onNext(this)
}
msg.acknowledge()
}
}
private fun registerBridgeDuplicateChecker(artemisSession: ClientSession) {
try {
artemisSession.createTemporaryQueue(BRIDGE_NOTIFY, RoutingType.MULTICAST, bridgeNotifyQueue)
} catch (ex: ActiveMQQueueExistsException) {
// Ignore if there is a queue still not cleaned up
}
val notify = artemisSession.createConsumer(bridgeNotifyQueue)
notifyConsumer = notify
notify.setMessageHandler { msg ->
try {
val data: ByteArray = ByteArray(msg.bodySize).apply { msg.bodyBuffer.readBytes(this) }
val notifyMessage = data.deserialize<BridgeControl.BridgeToNodeSnapshotRequest>(context = SerializationDefaults.P2P_CONTEXT)
if (notifyMessage.bridgeIdentity != bridgeId) {
log.error("Fatal Error! Two bridges have been configured simultaneously! Check the enterpriseConfiguration.externalBridge status")
System.exit(1)
}
} catch (ex: Exception) {
log.error("Unable to process bridge notification message", ex)
_failure.onNext(this)
}
msg.acknowledge()
}
val startupMessage = BridgeControl.BridgeToNodeSnapshotRequest(bridgeId).serialize(context = SerializationDefaults.P2P_CONTEXT).bytes
val bridgeRequest = artemisSession.createMessage(false)
bridgeRequest.writeBodyBufferBytes(startupMessage)
artemisClient.producer.send(BRIDGE_NOTIFY, bridgeRequest)
}
fun stop() {
validInboundQueues.clear()
controlConsumer?.close()
controlConsumer = null
artemis?.stop()
artemis = null
bridgeManager.stop()
try {
if (active) {
_activeChange.onNext(false)
}
validInboundQueues.clear()
controlConsumer?.close()
controlConsumer = null
notifyConsumer?.close()
notifyConsumer = null
artemis?.apply {
try {
started?.session?.deleteQueue(bridgeControlQueue)
} catch (e: ActiveMQNonExistentQueueException) {
log.warn("Queue $bridgeControlQueue does not exist and it can't be deleted")
}
try {
started?.session?.deleteQueue(bridgeNotifyQueue)
} catch (e: ActiveMQNonExistentQueueException) {
log.warn("Queue $bridgeNotifyQueue does not exist and it can't be deleted")
}
stop()
}
artemis = null
bridgeManager.stop()
} catch (e: Exception) {
log.error("Failure to stop BridgeControlListener", e)
}
}
override fun close() = stop()
@ -91,6 +206,10 @@ class BridgeControlListener(val config: MutualSslConfiguration,
log.info("Received bridge control message $controlMessage")
when (controlMessage) {
is BridgeControl.NodeToBridgeSnapshot -> {
if (!isConfigured(controlMessage.nodeIdentity)) {
log.error("Fatal error! Bridge not configured with keystore for node with legal name ${controlMessage.nodeIdentity}.")
System.exit(1)
}
if (!controlMessage.inboxQueues.all { validateInboxQueueName(it) }) {
log.error("Invalid queue names in control message $controlMessage")
return
@ -99,10 +218,20 @@ class BridgeControlListener(val config: MutualSslConfiguration,
log.error("Invalid queue names in control message $controlMessage")
return
}
for (outQueue in controlMessage.sendQueues) {
bridgeManager.deployBridge(outQueue.queueName, outQueue.targets, outQueue.legalNames.toSet())
}
val wasActive = active
validInboundQueues.addAll(controlMessage.inboxQueues)
for (outQueue in controlMessage.sendQueues) {
bridgeManager.deployBridge(controlMessage.nodeIdentity, outQueue.queueName, outQueue.targets, outQueue.legalNames.toSet())
}
log.info("Added inbox: ${controlMessage.inboxQueues}. Current inboxes: $validInboundQueues.")
if (bridgeManager is LoopbackBridgeManager) {
// Notify loopback bridge manager inboxes has changed.
bridgeManager.inboxesAdded(controlMessage.inboxQueues)
}
if (!wasActive && active) {
_activeChange.onNext(true)
}
}
is BridgeControl.BridgeToNodeSnapshotRequest -> {
log.error("Message from Bridge $controlMessage detected on wrong topic!")
@ -112,7 +241,7 @@ class BridgeControlListener(val config: MutualSslConfiguration,
log.error("Invalid queue names in control message $controlMessage")
return
}
bridgeManager.deployBridge(controlMessage.bridgeInfo.queueName, controlMessage.bridgeInfo.targets, controlMessage.bridgeInfo.legalNames.toSet())
bridgeManager.deployBridge(controlMessage.nodeIdentity, controlMessage.bridgeInfo.queueName, controlMessage.bridgeInfo.targets, controlMessage.bridgeInfo.legalNames.toSet())
}
is BridgeControl.Delete -> {
if (!controlMessage.bridgeInfo.queueName.startsWith(PEERS_PREFIX)) {
@ -121,7 +250,19 @@ class BridgeControlListener(val config: MutualSslConfiguration,
}
bridgeManager.destroyBridge(controlMessage.bridgeInfo.queueName, controlMessage.bridgeInfo.targets)
}
is BridgeControl.BridgeHealthCheck -> {
log.warn("Not currently doing anything on BridgeHealthCheck")
return
}
}
}
private fun isConfigured(sourceX500Name: String): Boolean {
val keyStore = keyStore.value.internal
return keyStore.aliases().toList().any { alias ->
val x500Name = keyStore.getCertificate(alias).x509.subjectX500Principal
val cordaX500Name = CordaX500Name.build(x500Name)
cordaX500Name.toString() == sourceX500Name
}
}
}

11
node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/BridgeControlMessages.kt
View File

@ -11,7 +11,7 @@ import net.corda.core.utilities.NetworkHostAndPort
* @property legalNames The list of acceptable [CordaX500Name] names that should be presented as subject of the validated peer TLS certificate.
*/
@CordaSerializable
data class BridgeEntry(val queueName: String, val targets: List<NetworkHostAndPort>, val legalNames: List<CordaX500Name>)
data class BridgeEntry(val queueName: String, val targets: List<NetworkHostAndPort>, val legalNames: List<CordaX500Name>, val serviceAddress: Boolean)
sealed class BridgeControl {
/**
@ -47,4 +47,13 @@ sealed class BridgeControl {
*/
@CordaSerializable
data class Delete(val nodeIdentity: String, val bridgeInfo: BridgeEntry) : BridgeControl()
/**
* This message is sent to Bridge to check the health of it.
* @property requestId The identifier for the health check request as health check is likely to be produced repeatedly.
* @property command Allows to specify the sort fo health check that needs to be performed.
* @property bridgeInfo The connection details of the new bridge (optional).
*/
@CordaSerializable
data class BridgeHealthCheck(val requestId: Long, val command: String, val bridgeInfo: BridgeEntry?) : BridgeControl()
}

7
node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/BridgeManager.kt
View File

@ -3,17 +3,20 @@ package net.corda.nodeapi.internal.bridging
import net.corda.core.identity.CordaX500Name
import net.corda.core.internal.VisibleForTesting
import net.corda.core.utilities.NetworkHostAndPort
import org.apache.activemq.artemis.api.core.client.ClientMessage
/**
* Provides an internal interface that the [BridgeControlListener] delegates to for Bridge activities.
*/
@VisibleForTesting
interface BridgeManager : AutoCloseable {
fun deployBridge(queueName: String, targets: List<NetworkHostAndPort>, legalNames: Set<CordaX500Name>)
fun deployBridge(sourceX500Name: String, queueName: String, targets: List<NetworkHostAndPort>, legalNames: Set<CordaX500Name>)
fun destroyBridge(queueName: String, targets: List<NetworkHostAndPort>)
fun start()
fun stop()
}
}
fun ClientMessage.payload() = ByteArray(bodySize).apply { bodyBuffer.readBytes(this) }

223
node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/LoopbackBridgeManager.kt Normal file
View File

@ -0,0 +1,223 @@
package net.corda.nodeapi.internal.bridging
import net.corda.nodeapi.internal.ConcurrentBox
import net.corda.core.identity.CordaX500Name
import net.corda.core.internal.VisibleForTesting
import net.corda.core.utilities.NetworkHostAndPort
import net.corda.core.utilities.contextLogger
import net.corda.nodeapi.internal.ArtemisMessagingComponent
import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.NODE_P2P_USER
import net.corda.nodeapi.internal.ArtemisMessagingComponent.RemoteInboxAddress.Companion.translateInboxAddressToLocalQueue
import net.corda.nodeapi.internal.ArtemisMessagingComponent.RemoteInboxAddress.Companion.translateLocalQueueToInboxAddress
import net.corda.nodeapi.internal.ArtemisSessionProvider
import net.corda.nodeapi.internal.ArtemisConstants.MESSAGE_ID_KEY
import net.corda.nodeapi.internal.config.CertificateStore
import net.corda.nodeapi.internal.protonwrapper.messages.impl.SendableMessageImpl
import net.corda.nodeapi.internal.protonwrapper.netty.ProxyConfig
import net.corda.nodeapi.internal.protonwrapper.netty.RevocationConfig
import net.corda.nodeapi.internal.stillOpen
import org.apache.activemq.artemis.api.core.SimpleString
import org.apache.activemq.artemis.api.core.client.ActiveMQClient.DEFAULT_ACK_BATCH_SIZE
import org.apache.activemq.artemis.api.core.client.ClientConsumer
import org.apache.activemq.artemis.api.core.client.ClientMessage
import org.apache.activemq.artemis.api.core.client.ClientProducer
import org.apache.activemq.artemis.api.core.client.ClientSession
import org.slf4j.MDC
/**
* The LoopbackBridgeManager holds the list of independent LoopbackBridge objects that actively loopback messages to local Artemis
* inboxes.
*/
@VisibleForTesting
class LoopbackBridgeManager(keyStore: CertificateStore,
trustStore: CertificateStore,
useOpenSSL: Boolean,
proxyConfig: ProxyConfig? = null,
maxMessageSize: Int,
revocationConfig: RevocationConfig,
enableSNI: Boolean,
private val artemisMessageClientFactory: () -> ArtemisSessionProvider,
private val bridgeMetricsService: BridgeMetricsService? = null,
private val isLocalInbox: (String) -> Boolean,
trace: Boolean,
sslHandshakeTimeout: Long? = null,
bridgeConnectionTTLSeconds: Int = 0) : AMQPBridgeManager(keyStore, trustStore, useOpenSSL, proxyConfig,
maxMessageSize, revocationConfig, enableSNI,
artemisMessageClientFactory, bridgeMetricsService,
trace, sslHandshakeTimeout,
bridgeConnectionTTLSeconds) {
companion object {
private val log = contextLogger()
}
private val queueNamesToBridgesMap = ConcurrentBox(mutableMapOf<String, MutableList<LoopbackBridge>>())
private var artemis: ArtemisSessionProvider? = null
/**
* Each LoopbackBridge is an independent consumer of messages from the Artemis local queue per designated endpoint.
* It attempts to loopback these messages via ArtemisClient to the local inbox.
*/
private class LoopbackBridge(val sourceX500Name: String,
val queueName: String,
val targets: List<NetworkHostAndPort>,
val legalNames: Set<CordaX500Name>,
artemis: ArtemisSessionProvider,
private val bridgeMetricsService: BridgeMetricsService?) {
companion object {
private val log = contextLogger()
}
// TODO: refactor MDC support, duplicated in AMQPBridgeManager.
private fun withMDC(block: () -> Unit) {
val oldMDC = MDC.getCopyOfContextMap()
try {
MDC.put("queueName", queueName)
MDC.put("source", sourceX500Name)
MDC.put("targets", targets.joinToString(separator = ";") { it.toString() })
MDC.put("legalNames", legalNames.joinToString(separator = ";") { it.toString() })
MDC.put("bridgeType", "loopback")
block()
} finally {
MDC.setContextMap(oldMDC)
}
}
private fun logDebugWithMDC(msg: () -> String) {
if (log.isDebugEnabled) {
withMDC { log.debug(msg()) }
}
}
private fun logInfoWithMDC(msg: String) = withMDC { log.info(msg) }
private fun logWarnWithMDC(msg: String) = withMDC { log.warn(msg) }
private val artemis = ConcurrentBox(artemis)
private var consumerSession: ClientSession? = null
private var producerSession: ClientSession? = null
private var consumer: ClientConsumer? = null
private var producer: ClientProducer? = null
fun start() {
logInfoWithMDC("Create new Artemis loopback bridge")
artemis.exclusive {
logInfoWithMDC("Bridge Connected")
bridgeMetricsService?.bridgeConnected(targets, legalNames)
val sessionFactory = started!!.sessionFactory
this@LoopbackBridge.consumerSession = sessionFactory.createSession(NODE_P2P_USER, NODE_P2P_USER, false, true, true, false, DEFAULT_ACK_BATCH_SIZE)
this@LoopbackBridge.producerSession = sessionFactory.createSession(NODE_P2P_USER, NODE_P2P_USER, false, true, true, false, DEFAULT_ACK_BATCH_SIZE)
// Several producers (in the case of shared bridge) can put messages in the same outbound p2p queue. The consumers are created using the source x500 name as a filter
val consumer = consumerSession!!.createConsumer(queueName, "hyphenated_props:sender-subject-name = '$sourceX500Name'")
consumer.setMessageHandler(this@LoopbackBridge::clientArtemisMessageHandler)
this@LoopbackBridge.consumer = consumer
this@LoopbackBridge.producer = producerSession!!.createProducer()
consumerSession?.start()
producerSession?.start()
}
}
fun stop() {
logInfoWithMDC("Stopping AMQP bridge")
artemis.exclusive {
bridgeMetricsService?.bridgeDisconnected(targets, legalNames)
consumer?.apply { if (!isClosed) close() }
consumer = null
producer?.apply { if (!isClosed) close() }
producer = null
consumerSession?.apply { if (stillOpen()) stop() }
consumerSession = null
producerSession?.apply { if (stillOpen()) stop()}
producerSession = null
}
}
private fun clientArtemisMessageHandler(artemisMessage: ClientMessage) {
logDebugWithMDC { "Loopback Send to ${legalNames.first()} uuid: ${artemisMessage.getObjectProperty(MESSAGE_ID_KEY)}" }
val peerInbox = translateLocalQueueToInboxAddress(queueName)
producer?.send(SimpleString(peerInbox), artemisMessage) { artemisMessage.individualAcknowledge() }
bridgeMetricsService?.let { metricsService ->
val properties = ArtemisMessagingComponent.Companion.P2PMessagingHeaders.whitelistedHeaders.mapNotNull { key ->
if (artemisMessage.containsProperty(key)) {
key to artemisMessage.getObjectProperty(key).let { (it as? SimpleString)?.toString() ?: it }
} else {
null
}
}.toMap()
metricsService.packetAcceptedEvent(SendableMessageImpl(artemisMessage.payload(), peerInbox, legalNames.first().toString(), targets.first(), properties))
}
}
}
override fun deployBridge(sourceX500Name: String, queueName: String, targets: List<NetworkHostAndPort>, legalNames: Set<CordaX500Name>) {
val inboxAddress = translateLocalQueueToInboxAddress(queueName)
if (isLocalInbox(inboxAddress)) {
log.info("Deploying loopback bridge for $queueName, source $sourceX500Name")
queueNamesToBridgesMap.exclusive {
val bridges = getOrPut(queueName) { mutableListOf() }
for (target in targets) {
if (bridges.any { it.targets.contains(target) && it.sourceX500Name == sourceX500Name }) {
return
}
}
val newBridge = LoopbackBridge(sourceX500Name, queueName, targets, legalNames, artemis!!, bridgeMetricsService)
bridges += newBridge
bridgeMetricsService?.bridgeCreated(targets, legalNames)
newBridge
}.start()
} else {
log.info("Deploying AMQP bridge for $queueName, source $sourceX500Name")
super.deployBridge(sourceX500Name, queueName, targets, legalNames)
}
}
override fun destroyBridge(queueName: String, targets: List<NetworkHostAndPort>) {
super.destroyBridge(queueName, targets)
queueNamesToBridgesMap.exclusive {
val bridges = this[queueName] ?: mutableListOf()
for (target in targets) {
val bridge = bridges.firstOrNull { it.targets.contains(target) }
if (bridge != null) {
bridges -= bridge
if (bridges.isEmpty()) {
remove(queueName)
}
bridge.stop()
bridgeMetricsService?.bridgeDestroyed(bridge.targets, bridge.legalNames)
}
}
}
}
/**
* Remove any AMQP bridge for the local inbox and create a loopback bridge for that queue.
*/
fun inboxesAdded(inboxes: List<String>) {
for (inbox in inboxes) {
super.destroyAllBridges(translateInboxAddressToLocalQueue(inbox)).forEach { source, bridgeEntry ->
log.info("Destroyed AMQP Bridge '${bridgeEntry.queueName}', creating Loopback bridge for local inbox.")
deployBridge(source, bridgeEntry.queueName, bridgeEntry.targets, bridgeEntry.legalNames.toSet())
}
}
}
override fun start() {
super.start()
val artemis = artemisMessageClientFactory()
this.artemis = artemis
artemis.start()
}
override fun stop() = close()
override fun close() {
super.close()
queueNamesToBridgesMap.exclusive {
for (bridge in values.flatten()) {
bridge.stop()
}
clear()
artemis?.stop()
}
}
}

63
node-api/src/main/kotlin/net/corda/nodeapi/internal/config/MessagingServerConnectionConfiguration.kt Normal file
View File

@ -0,0 +1,63 @@
package net.corda.nodeapi.internal.config
import net.corda.core.utilities.minutes
import net.corda.core.utilities.seconds
import java.time.Duration
/**
* Predefined connection configurations used by Artemis clients (currently used in the P2P messaging layer).
* The enum names represent the approximate total duration of the failover (with exponential back-off). The formula used to calculate
* this duration is as follows:
*
* totalFailoverDuration = SUM(k=0 to [reconnectAttempts]) of [retryInterval] * POW([retryIntervalMultiplier], k)
*
* Example calculation for [DEFAULT]:
*
* totalFailoverDuration = 5 + 5 * 1.5 + 5 * (1.5)^2 + 5 * (1.5)^3 + 5 * (1.5)^4 = ~66 seconds
*
* @param failoverOnInitialAttempt Determines whether failover is triggered if initial connection fails.
* @param initialConnectAttempts The number of reconnect attempts if failover is enabled for initial connection. A value
* of -1 represents infinite attempts.
* @param reconnectAttempts The number of reconnect attempts for failover after initial connection is done. A value
* of -1 represents infinite attempts.
* @param retryInterval Duration between reconnect attempts.
* @param retryIntervalMultiplier Value used in the reconnection back-off process.
* @param maxRetryInterval Determines the maximum duration between reconnection attempts. Useful when using infinite retries.
*/
enum class MessagingServerConnectionConfiguration {
DEFAULT {
override fun failoverOnInitialAttempt(isHa: Boolean) = true
override fun initialConnectAttempts(isHa: Boolean) = 5
override fun reconnectAttempts(isHa: Boolean) = 5
override fun retryInterval() = 5.seconds
override fun retryIntervalMultiplier() = 1.5
override fun maxRetryInterval(isHa: Boolean) = 3.minutes
},
FAIL_FAST {
override fun failoverOnInitialAttempt(isHa: Boolean) = isHa
override fun initialConnectAttempts(isHa: Boolean) = 0
// Client die too fast during failover/failback, need a few reconnect attempts to allow new master to become active
override fun reconnectAttempts(isHa: Boolean) = if (isHa) 3 else 0
override fun retryInterval() = 5.seconds
override fun retryIntervalMultiplier() = 1.5
override fun maxRetryInterval(isHa: Boolean) = 3.minutes
},
CONTINUOUS_RETRY {
override fun failoverOnInitialAttempt(isHa: Boolean) = true
override fun initialConnectAttempts(isHa: Boolean) = if (isHa) 0 else -1
override fun reconnectAttempts(isHa: Boolean) = -1
override fun retryInterval() = 5.seconds
override fun retryIntervalMultiplier() = 1.5
override fun maxRetryInterval(isHa: Boolean) = if (isHa) 3.minutes else 5.minutes
};
abstract fun failoverOnInitialAttempt(isHa: Boolean): Boolean
abstract fun initialConnectAttempts(isHa: Boolean): Int
abstract fun reconnectAttempts(isHa: Boolean): Int
abstract fun retryInterval(): Duration
abstract fun retryIntervalMultiplier(): Double
abstract fun maxRetryInterval(isHa: Boolean): Duration
}

11
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/engine/ConnectionStateMachine.kt
View File

@ -45,7 +45,11 @@ internal class ConnectionStateMachine(private val serverMode: Boolean,
userName: String?,
password: String?) : BaseHandler() {
companion object {
private const val IDLE_TIMEOUT = 10000
private const val CORDA_AMQP_FRAME_SIZE_PROP_NAME = "net.corda.nodeapi.connectionstatemachine.AmqpMaxFrameSize"
private const val CORDA_AMQP_IDLE_TIMEOUT_PROP_NAME = "net.corda.nodeapi.connectionstatemachine.AmqpIdleTimeout"
private val MAX_FRAME_SIZE = Integer.getInteger(CORDA_AMQP_FRAME_SIZE_PROP_NAME, 128 * 1024)
private val IDLE_TIMEOUT = Integer.getInteger(CORDA_AMQP_IDLE_TIMEOUT_PROP_NAME, 10 * 1000)
private val log = contextLogger()
}
@ -102,6 +106,7 @@ internal class ConnectionStateMachine(private val serverMode: Boolean,
transport.context = connection
@Suppress("UsePropertyAccessSyntax")
transport.setEmitFlowEventOnSend(true)
transport.maxFrameSize = MAX_FRAME_SIZE
connection.collect(collector)
val sasl = transport.sasl()
if (userName != null) {
@ -488,7 +493,9 @@ internal class ConnectionStateMachine(private val serverMode: Boolean,
}
fun transportWriteMessage(msg: SendableMessageImpl) {
msg.buf = encodePayloadBytes(msg)
val encoded = encodePayloadBytes(msg)
msg.release()
msg.buf = encoded
val messageQueue = messageQueues.getOrPut(msg.topic, { LinkedList() })
messageQueue.offer(msg)
if (session != null) {

4
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/engine/EventProcessor.kt
View File

@ -38,7 +38,9 @@ internal class EventProcessor(private val channel: Channel,
userName: String?,
password: String?) {
companion object {
private const val FLOW_WINDOW_SIZE = 10
private const val CORDA_AMQP_FLOW_WINDOW_SIZE_PROP_NAME = "net.corda.nodeapi.eventprocessor.FlowWindowSize"
private val FLOW_WINDOW_SIZE = Integer.getInteger(CORDA_AMQP_FLOW_WINDOW_SIZE_PROP_NAME, 5)
private val log = contextLogger()
}

1
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/messages/ApplicationMessage.kt
View File

@ -11,4 +11,5 @@ interface ApplicationMessage {
val destinationLegalName: String
val destinationLink: NetworkHostAndPort
val applicationProperties: Map<String, Any?>
fun release()
}

19
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/messages/impl/ReceivedMessageImpl.kt
View File

@ -2,6 +2,7 @@ package net.corda.nodeapi.internal.protonwrapper.messages.impl
import io.netty.channel.Channel
import net.corda.core.utilities.NetworkHostAndPort
import net.corda.core.utilities.contextLogger
import net.corda.nodeapi.internal.protonwrapper.messages.MessageStatus
import net.corda.nodeapi.internal.protonwrapper.messages.ReceivedMessage
import org.apache.qpid.proton.engine.Delivery
@ -10,7 +11,7 @@ import org.apache.qpid.proton.engine.Delivery
* An internal packet management class that allows tracking of asynchronous acknowledgements
* that in turn send Delivery messages back to the originator.
*/
internal class ReceivedMessageImpl(override val payload: ByteArray,
internal class ReceivedMessageImpl(override var payload: ByteArray,
override val topic: String,
override val sourceLegalName: String,
override val sourceLink: NetworkHostAndPort,
@ -19,11 +20,25 @@ internal class ReceivedMessageImpl(override val payload: ByteArray,
override val applicationProperties: Map<String, Any?>,
private val channel: Channel,
private val delivery: Delivery) : ReceivedMessage {
companion object {
private val emptyPayload = ByteArray(0)
private val logger = contextLogger()
}
data class MessageCompleter(val status: MessageStatus, val delivery: Delivery)
override fun release() {
payload = emptyPayload
}
override fun complete(accepted: Boolean) {
release()
val status = if (accepted) MessageStatus.Acknowledged else MessageStatus.Rejected
channel.writeAndFlush(MessageCompleter(status, delivery))
if (channel.isActive) {
channel.writeAndFlush(MessageCompleter(status, delivery))
} else {
logger.info("Not writing $status as $channel is not active")
}
}
override fun toString(): String = "Received ${String(payload)} $topic"

10
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/messages/impl/SendableMessageImpl.kt
View File

@ -11,11 +11,15 @@ import net.corda.nodeapi.internal.protonwrapper.messages.SendableMessage
* An internal packet management class that allows handling of the encoded buffers and
* allows registration of an acknowledgement handler when the remote receiver confirms durable storage.
*/
internal class SendableMessageImpl(override val payload: ByteArray,
internal class SendableMessageImpl(override var payload: ByteArray,
override val topic: String,
override val destinationLegalName: String,
override val destinationLink: NetworkHostAndPort,
override val applicationProperties: Map<String, Any?>) : SendableMessage {
companion object {
private val emptyPayload = ByteArray(0)
}
var buf: ByteBuf? = null
@Volatile
var status: MessageStatus = MessageStatus.Unsent
@ -23,12 +27,14 @@ internal class SendableMessageImpl(override val payload: ByteArray,
private val _onComplete = openFuture<MessageStatus>()
override val onComplete: CordaFuture<MessageStatus> get() = _onComplete
fun release() {
override fun release() {
payload = emptyPayload
buf?.release()
buf = null
}
fun doComplete(status: MessageStatus) {
release()
this.status = status
_onComplete.set(status)
}

218
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPChannelHandler.kt
View File

@ -5,11 +5,16 @@ import io.netty.channel.ChannelDuplexHandler
import io.netty.channel.ChannelHandlerContext
import io.netty.channel.ChannelPromise
import io.netty.channel.socket.SocketChannel
import io.netty.handler.proxy.ProxyConnectException
import io.netty.handler.proxy.ProxyConnectionEvent
import io.netty.handler.ssl.SniCompletionEvent
import io.netty.handler.ssl.SslHandler
import io.netty.handler.ssl.SslHandshakeCompletionEvent
import io.netty.util.ReferenceCountUtil
import net.corda.core.identity.CordaX500Name
import net.corda.core.utilities.contextLogger
import net.corda.core.utilities.trace
import net.corda.nodeapi.internal.ArtemisConstants.MESSAGE_ID_KEY
import net.corda.nodeapi.internal.crypto.x509
import net.corda.nodeapi.internal.protonwrapper.engine.EventProcessor
import net.corda.nodeapi.internal.protonwrapper.messages.ReceivedMessage
@ -23,6 +28,8 @@ import org.slf4j.MDC
import java.net.InetSocketAddress
import java.nio.channels.ClosedChannelException
import java.security.cert.X509Certificate
import javax.net.ssl.ExtendedSSLSession
import javax.net.ssl.SNIHostName
import javax.net.ssl.SSLException
/**
@ -30,23 +37,29 @@ import javax.net.ssl.SSLException
* It also add some extra checks to the SSL handshake to support our non-standard certificate checks of legal identity.
* When a valid SSL connections is made then it initialises a proton-j engine instance to handle the protocol layer.
*/
@Suppress("TooManyFunctions")
internal class AMQPChannelHandler(private val serverMode: Boolean,
private val allowedRemoteLegalNames: Set<CordaX500Name>?,
private val keyManagerFactoriesMap: Map<String, CertHoldingKeyManagerFactoryWrapper>,
private val userName: String?,
private val password: String?,
private val trace: Boolean,
private val onOpen: (Pair<SocketChannel, ConnectionChange>) -> Unit,
private val onClose: (Pair<SocketChannel, ConnectionChange>) -> Unit,
private val suppressLogs: Boolean,
private val onOpen: (SocketChannel, ConnectionChange) -> Unit,
private val onClose: (SocketChannel, ConnectionChange) -> Unit,
private val onReceive: (ReceivedMessage) -> Unit) : ChannelDuplexHandler() {
companion object {
private val log = contextLogger()
const val PROXY_LOGGER_NAME = "preProxyLogger"
}
private lateinit var remoteAddress: InetSocketAddress
private var localCert: X509Certificate? = null
private var remoteCert: X509Certificate? = null
private var eventProcessor: EventProcessor? = null
private var suppressClose: Boolean = false
private var badCert: Boolean = false
private var localCert: X509Certificate? = null
private var requestedServerName: String? = null
private fun withMDC(block: () -> Unit) {
val oldMDC = MDC.getCopyOfContextMap() ?: emptyMap<String, String>()
@ -62,39 +75,50 @@ internal class AMQPChannelHandler(private val serverMode: Boolean,
}
}
private fun logDebugWithMDC(msg: () -> String) {
if (log.isDebugEnabled) {
withMDC { log.debug(msg()) }
private fun logDebugWithMDC(msgFn: () -> String) {
if (!suppressLogs) {
if (log.isDebugEnabled) {
withMDC { log.debug(msgFn()) }
}
} else {
withMDC { log.trace(msgFn) }
}
}
private fun logInfoWithMDC(msg: String) = withMDC { log.info(msg) }
private fun logInfoWithMDC(msgFn: () -> String) {
if (!suppressLogs) {
if (log.isInfoEnabled) {
withMDC { log.info(msgFn()) }
}
} else {
withMDC { log.trace(msgFn) }
}
}
private fun logWarnWithMDC(msg: String) = withMDC { log.warn(msg) }
private fun logErrorWithMDC(msg: String, ex: Throwable? = null) = withMDC { log.error(msg, ex) }
private fun logWarnWithMDC(msg: String) = withMDC { if (!suppressLogs) log.warn(msg) else log.trace { msg } }
private fun logErrorWithMDC(msg: String, ex: Throwable? = null) = withMDC { if (!suppressLogs) log.error(msg, ex) else log.trace(msg, ex) }
override fun channelActive(ctx: ChannelHandlerContext) {
val ch = ctx.channel()
remoteAddress = ch.remoteAddress() as InetSocketAddress
val localAddress = ch.localAddress() as InetSocketAddress
logInfoWithMDC("New client connection ${ch.id()} from $remoteAddress to $localAddress")
logInfoWithMDC { "New client connection ${ch.id()} from $remoteAddress to $localAddress" }
}
private fun createAMQPEngine(ctx: ChannelHandlerContext) {
val ch = ctx.channel()
eventProcessor = EventProcessor(ch, serverMode, localCert!!.subjectX500Principal.toString(), remoteCert!!.subjectX500Principal.toString(), userName, password)
val connection = eventProcessor!!.connection
val transport = connection.transport as ProtonJTransport
if (trace) {
val connection = eventProcessor!!.connection
val transport = connection.transport as ProtonJTransport
transport.protocolTracer = object : ProtocolTracer {
override fun sentFrame(transportFrame: TransportFrame) {
logInfoWithMDC("${transportFrame.body}")
logInfoWithMDC { "${transportFrame.body}" }
}
override fun receivedFrame(transportFrame: TransportFrame) {
logInfoWithMDC("${transportFrame.body}")
logInfoWithMDC { "${transportFrame.body}" }
}
}
}
@ -104,51 +128,60 @@ internal class AMQPChannelHandler(private val serverMode: Boolean,
override fun channelInactive(ctx: ChannelHandlerContext) {
val ch = ctx.channel()
logInfoWithMDC("Closed client connection ${ch.id()} from $remoteAddress to ${ch.localAddress()}")
onClose(Pair(ch as SocketChannel, ConnectionChange(remoteAddress, remoteCert, false, badCert)))
logInfoWithMDC { "Closed client connection ${ch.id()} from $remoteAddress to ${ch.localAddress()}" }
if (!suppressClose) {
onClose(ch as SocketChannel, ConnectionChange(remoteAddress, remoteCert, false, badCert))
}
eventProcessor?.close()
ctx.fireChannelInactive()
}
override fun userEventTriggered(ctx: ChannelHandlerContext, evt: Any) {
if (evt is SslHandshakeCompletionEvent) {
if (evt.isSuccess) {
val sslHandler = ctx.pipeline().get(SslHandler::class.java)
localCert = sslHandler.engine().session.localCertificates[0].x509
remoteCert = sslHandler.engine().session.peerCertificates[0].x509
val remoteX500Name = try {
CordaX500Name.build(remoteCert!!.subjectX500Principal)
} catch (ex: IllegalArgumentException) {
badCert = true
logErrorWithMDC("Certificate subject not a valid CordaX500Name", ex)
ctx.close()
return
when (evt) {
is ProxyConnectionEvent -> {
if (trace) {
log.info("ProxyConnectionEvent received: $evt")
try {
ctx.pipeline().remove(PROXY_LOGGER_NAME)
} catch (ex: NoSuchElementException) {
// ignore
}
}
if (allowedRemoteLegalNames != null && remoteX500Name !in allowedRemoteLegalNames) {
badCert = true
logErrorWithMDC("Provided certificate subject $remoteX500Name not in expected set $allowedRemoteLegalNames")
ctx.close()
return
}
logInfoWithMDC("Handshake completed with subject: $remoteX500Name")
createAMQPEngine(ctx)
onOpen(Pair(ctx.channel() as SocketChannel, ConnectionChange(remoteAddress, remoteCert, true, false)))
} else {
val cause = evt.cause()
// This happens when the peer node is closed during SSL establishment.
if (cause is ClosedChannelException) {
logWarnWithMDC("SSL Handshake closed early.")
} else if (cause is SSLException && cause.message == "handshake timed out") { // Sadly the exception thrown by Netty wrapper requires that we check the message.
logWarnWithMDC("SSL Handshake timed out")
} else {
badCert = true
}
logErrorWithMDC("Handshake failure ${evt.cause().message}")
if (log.isTraceEnabled) {
withMDC { log.trace("Handshake failure", evt.cause()) }
}
ctx.close()
// update address to the real target address
remoteAddress = evt.destinationAddress()
}
is SniCompletionEvent -> {
if (evt.isSuccess) {
// The SniCompletionEvent is fired up before context is switched (after SslHandshakeCompletionEvent)
// so we save the requested server name now to be able log it once the handshake is completed successfully
// Note: this event is only triggered when using OpenSSL.
requestedServerName = evt.hostname()
logInfoWithMDC { "SNI completion success." }
} else {
logErrorWithMDC("SNI completion failure: ${evt.cause().message}")
}
}
is SslHandshakeCompletionEvent -> {
if (evt.isSuccess) {
handleSuccessfulHandshake(ctx)
} else {
handleFailedHandshake(ctx, evt)
}
}
}
}
private fun SslHandler.getRequestedServerName(): String? {
return if (serverMode) {
val session = engine().session
when (session) {
// Server name can be obtained from SSL session when using JavaSSL.
is ExtendedSSLSession -> (session.requestedServerNames.firstOrNull() as? SNIHostName)?.asciiName
// For Open SSL server name is obtained from SniCompletionEvent
else -> requestedServerName
}
} else {
(engine().sslParameters?.serverNames?.firstOrNull() as? SNIHostName)?.asciiName
}
}
@ -158,6 +191,10 @@ internal class AMQPChannelHandler(private val serverMode: Boolean,
if (log.isTraceEnabled) {
withMDC { log.trace("Pipeline uncaught exception", cause) }
}
if (cause is ProxyConnectException) {
log.warn("Proxy connection failed ${cause.message}")
suppressClose = true // The pipeline gets marked as active on connection to the proxy rather than to the target, which causes excess close events
}
ctx.close()
}
@ -176,27 +213,27 @@ internal class AMQPChannelHandler(private val serverMode: Boolean,
try {
try {
when (msg) {
// Transfers application packet into the AMQP engine.
// Transfers application packet into the AMQP engine.
is SendableMessageImpl -> {
val inetAddress = InetSocketAddress(msg.destinationLink.host, msg.destinationLink.port)
logDebugWithMDC { "Message for endpoint $inetAddress , expected $remoteAddress "}
logDebugWithMDC { "Message for endpoint $inetAddress , expected $remoteAddress " }
require(CordaX500Name.parse(msg.destinationLegalName) == CordaX500Name.build(remoteCert!!.subjectX500Principal)) {
"Message for incorrect legal identity ${msg.destinationLegalName} expected ${remoteCert!!.subjectX500Principal}"
}
logDebugWithMDC { "channel write ${msg.applicationProperties["_AMQ_DUPL_ID"]}" }
logDebugWithMDC { "channel write ${msg.applicationProperties[MESSAGE_ID_KEY]}" }
eventProcessor!!.transportWriteMessage(msg)
}
// A received AMQP packet has been completed and this self-posted packet will be signalled out to the
// external application.
// A received AMQP packet has been completed and this self-posted packet will be signalled out to the
// external application.
is ReceivedMessage -> {
onReceive(msg)
}
// A general self-posted event that triggers creation of AMQP frames when required.
// A general self-posted event that triggers creation of AMQP frames when required.
is Transport -> {
eventProcessor!!.transportProcessOutput(ctx)
}
// A self-posted event that forwards status updates for delivered packets to the application.
// A self-posted event that forwards status updates for delivered packets to the application.
is ReceivedMessageImpl.MessageCompleter -> {
eventProcessor!!.complete(msg)
}
@ -210,4 +247,65 @@ internal class AMQPChannelHandler(private val serverMode: Boolean,
}
eventProcessor!!.processEventsAsync()
}
private fun handleSuccessfulHandshake(ctx: ChannelHandlerContext) {
val sslHandler = ctx.pipeline().get(SslHandler::class.java)
val sslSession = sslHandler.engine().session
// Depending on what matching method is used, getting the local certificate is done by selecting the
// appropriate keyManagerFactory
val keyManagerFactory = requestedServerName?.let {
keyManagerFactoriesMap[it]
} ?: keyManagerFactoriesMap.values.single()
localCert = keyManagerFactory.getCurrentCertChain()?.first()
if (localCert == null) {
log.error("SSL KeyManagerFactory failed to provide a local cert")
ctx.close()
return
}
if (sslSession.peerCertificates == null || sslSession.peerCertificates.isEmpty()) {
log.error("No peer certificates")
ctx.close()
return
}
remoteCert = sslHandler.engine().session.peerCertificates.first().x509
val remoteX500Name = try {
CordaX500Name.build(remoteCert!!.subjectX500Principal)
} catch (ex: IllegalArgumentException) {
badCert = true
logErrorWithMDC("Certificate subject not a valid CordaX500Name", ex)
ctx.close()
return
}
if (allowedRemoteLegalNames != null && remoteX500Name !in allowedRemoteLegalNames) {
badCert = true
logErrorWithMDC("Provided certificate subject $remoteX500Name not in expected set $allowedRemoteLegalNames")
ctx.close()
return
}
logInfoWithMDC { "Handshake completed with subject: $remoteX500Name, requested server name: ${sslHandler.getRequestedServerName()}." }
createAMQPEngine(ctx)
onOpen(ctx.channel() as SocketChannel, ConnectionChange(remoteAddress, remoteCert, connected = true, badCert = false))
}
private fun handleFailedHandshake(ctx: ChannelHandlerContext, evt: SslHandshakeCompletionEvent) {
val cause = evt.cause()
// This happens when the peer node is closed during SSL establishment.
when {
cause is ClosedChannelException -> logWarnWithMDC("SSL Handshake closed early.")
// Sadly the exception thrown by Netty wrapper requires that we check the message.
cause is SSLException && cause.message == "handshake timed out" -> logWarnWithMDC("SSL Handshake timed out")
cause is SSLException && (cause.message?.contains("close_notify") == true)
-> logWarnWithMDC("Received close_notify during handshake")
else -> badCert = true
}
logWarnWithMDC("Handshake failure: ${evt.cause().message}")
if (log.isTraceEnabled) {
withMDC { log.trace("Handshake failure", evt.cause()) }
}
ctx.close()
}
}

183
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPClient.kt
View File

@ -7,24 +7,45 @@ import io.netty.channel.socket.SocketChannel
import io.netty.channel.socket.nio.NioSocketChannel
import io.netty.handler.logging.LogLevel
import io.netty.handler.logging.LoggingHandler
import io.netty.handler.proxy.HttpProxyHandler
import io.netty.handler.proxy.Socks4ProxyHandler
import io.netty.handler.proxy.Socks5ProxyHandler
import io.netty.resolver.NoopAddressResolverGroup
import io.netty.util.internal.logging.InternalLoggerFactory
import io.netty.util.internal.logging.Slf4JLoggerFactory
import net.corda.core.identity.CordaX500Name
import net.corda.core.utilities.NetworkHostAndPort
import net.corda.core.utilities.contextLogger
import net.corda.core.utilities.debug
import net.corda.nodeapi.internal.protonwrapper.messages.ReceivedMessage
import net.corda.nodeapi.internal.protonwrapper.messages.SendableMessage
import net.corda.nodeapi.internal.protonwrapper.messages.impl.SendableMessageImpl
import net.corda.nodeapi.internal.protonwrapper.netty.AMQPChannelHandler.Companion.PROXY_LOGGER_NAME
import net.corda.nodeapi.internal.requireMessageSize
import rx.Observable
import rx.subjects.PublishSubject
import java.lang.Long.min
import java.net.InetSocketAddress
import java.util.concurrent.TimeUnit
import java.util.concurrent.locks.ReentrantLock
import javax.net.ssl.KeyManagerFactory
import javax.net.ssl.TrustManagerFactory
import kotlin.concurrent.withLock
enum class ProxyVersion {
SOCKS4,
SOCKS5,
HTTP
}
data class ProxyConfig(val version: ProxyVersion, val proxyAddress: NetworkHostAndPort, val userName: String? = null, val password: String? = null, val proxyTimeoutMS: Long? = null) {
init {
if (version == ProxyVersion.SOCKS4) {
require(password == null) { "SOCKS4 does not support a password" }
}
}
}
/**
* The AMQPClient creates a connection initiator that will try to connect in a round-robin fashion
* to the first open SSL socket. It will keep retrying until it is stopped.
@ -42,15 +63,18 @@ class AMQPClient(val targets: List<NetworkHostAndPort>,
}
val log = contextLogger()
const val MIN_RETRY_INTERVAL = 1000L
const val MAX_RETRY_INTERVAL = 60000L
const val BACKOFF_MULTIPLIER = 2L
const val NUM_CLIENT_THREADS = 2
private const val CORDA_AMQP_NUM_CLIENT_THREAD_PROP_NAME = "net.corda.nodeapi.amqpclient.NumClientThread"
private const val MIN_RETRY_INTERVAL = 1000L
private const val MAX_RETRY_INTERVAL = 60000L
private const val BACKOFF_MULTIPLIER = 2L
private val NUM_CLIENT_THREADS = Integer.getInteger(CORDA_AMQP_NUM_CLIENT_THREAD_PROP_NAME, 2)
}
private val lock = ReentrantLock()
@Volatile
private var stopping: Boolean = false
private var started: Boolean = false
private var workerGroup: EventLoopGroup? = null
@Volatile
private var clientChannel: Channel? = null
@ -59,6 +83,13 @@ class AMQPClient(val targets: List<NetworkHostAndPort>,
private var currentTarget: NetworkHostAndPort = targets.first()
private var retryInterval = MIN_RETRY_INTERVAL
private val badCertTargets = mutableSetOf<NetworkHostAndPort>()
@Volatile
private var amqpActive = false
@Volatile
private var amqpChannelHandler: ChannelHandler? = null
val localAddressString: String
get() = clientChannel?.localAddress()?.toString() ?: "<unknownLocalAddress>"
private fun nextTarget() {
val origIndex = targetIndex
@ -80,29 +111,31 @@ class AMQPClient(val targets: List<NetworkHostAndPort>,
private val connectListener = object : ChannelFutureListener {
override fun operationComplete(future: ChannelFuture) {
amqpActive = false
if (!future.isSuccess) {
log.info("Failed to connect to $currentTarget")
if (!stopping) {
if (started) {
workerGroup?.schedule({
nextTarget()
restart()
}, retryInterval, TimeUnit.MILLISECONDS)
}
} else {
log.info("Connected to $currentTarget")
// Connection established successfully
clientChannel = future.channel()
clientChannel?.closeFuture()?.addListener(closeListener)
log.info("Connected to $currentTarget, Local address: $localAddressString")
}
}
}
private val closeListener = ChannelFutureListener { future ->
log.info("Disconnected from $currentTarget")
log.info("Disconnected from $currentTarget, Local address: $localAddressString")
future.channel()?.disconnect()
clientChannel = null
if (!stopping) {
if (started && !amqpActive) {
log.debug { "Scheduling restart of $currentTarget (AMQP inactive)" }
workerGroup?.schedule({
nextTarget()
restart()
@ -114,42 +147,110 @@ class AMQPClient(val targets: List<NetworkHostAndPort>,
private val keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm())
private val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm())
private val conf = parent.configuration
@Volatile
private lateinit var amqpChannelHandler: AMQPChannelHandler
init {
keyManagerFactory.init(conf.keyStore)
trustManagerFactory.init(initialiseTrustStoreAndEnableCrlChecking(conf.trustStore, conf.crlCheckSoftFail))
trustManagerFactory.init(initialiseTrustStoreAndEnableCrlChecking(conf.trustStore, conf.revocationConfig))
}
@Suppress("ComplexMethod")
override fun initChannel(ch: SocketChannel) {
val pipeline = ch.pipeline()
val proxyConfig = conf.proxyConfig
if (proxyConfig != null) {
if (conf.trace) pipeline.addLast(PROXY_LOGGER_NAME, LoggingHandler(LogLevel.INFO))
val proxyAddress = InetSocketAddress(proxyConfig.proxyAddress.host, proxyConfig.proxyAddress.port)
val proxy = when (conf.proxyConfig!!.version) {
ProxyVersion.SOCKS4 -> {
Socks4ProxyHandler(proxyAddress, proxyConfig.userName)
}
ProxyVersion.SOCKS5 -> {
Socks5ProxyHandler(proxyAddress, proxyConfig.userName, proxyConfig.password)
}
ProxyVersion.HTTP -> {
val httpProxyHandler = if(proxyConfig.userName == null || proxyConfig.password == null) {
HttpProxyHandler(proxyAddress)
} else {
HttpProxyHandler(proxyAddress, proxyConfig.userName, proxyConfig.password)
}
//httpProxyHandler.setConnectTimeoutMillis(3600000) // 1hr for debugging purposes
httpProxyHandler
}
}
val proxyTimeout = proxyConfig.proxyTimeoutMS
if (proxyTimeout != null) {
proxy.setConnectTimeoutMillis(proxyTimeout)
}
pipeline.addLast("Proxy", proxy)
proxy.connectFuture().addListener {
if (!it.isSuccess) {
ch.disconnect()
}
}
}
val wrappedKeyManagerFactory = CertHoldingKeyManagerFactoryWrapper(keyManagerFactory, parent.configuration)
val target = parent.currentTarget
val handler = createClientSslHelper(target, parent.allowedRemoteLegalNames, keyManagerFactory, trustManagerFactory)
val handler = if (parent.configuration.useOpenSsl) {
createClientOpenSslHandler(target, parent.allowedRemoteLegalNames, wrappedKeyManagerFactory, trustManagerFactory, ch.alloc())
} else {
createClientSslHelper(target, parent.allowedRemoteLegalNames, wrappedKeyManagerFactory, trustManagerFactory)
}
handler.handshakeTimeoutMillis = conf.sslHandshakeTimeout
pipeline.addLast("sslHandler", handler)
if (conf.trace) pipeline.addLast("logger", LoggingHandler(LogLevel.INFO))
pipeline.addLast(AMQPChannelHandler(false,
amqpChannelHandler = AMQPChannelHandler(false,
parent.allowedRemoteLegalNames,
// Single entry, key can be anything.
mapOf(DEFAULT to wrappedKeyManagerFactory),
conf.userName,
conf.password,
conf.trace,
{
parent.retryInterval = MIN_RETRY_INTERVAL // reset to fast reconnect if we connect properly
parent._onConnection.onNext(it.second)
},
{
parent._onConnection.onNext(it.second)
if (it.second.badCert) {
log.error("Blocking future connection attempts to $target due to bad certificate on endpoint")
parent.badCertTargets += target
false,
onOpen = { _, change ->
parent.run {
amqpActive = true
retryInterval = MIN_RETRY_INTERVAL // reset to fast reconnect if we connect properly
_onConnection.onNext(change)
}
},
{ rcv -> parent._onReceive.onNext(rcv) }))
onClose = { _, change ->
if (parent.amqpChannelHandler == amqpChannelHandler) {
parent.run {
_onConnection.onNext(change)
if (change.badCert) {
log.error("Blocking future connection attempts to $target due to bad certificate on endpoint")
badCertTargets += target
}
if (started && amqpActive) {
log.debug { "Scheduling restart of $currentTarget (AMQP active)" }
workerGroup?.schedule({
nextTarget()
restart()
}, retryInterval, TimeUnit.MILLISECONDS)
}
amqpActive = false
}
}
},
onReceive = { rcv -> parent._onReceive.onNext(rcv) })
parent.amqpChannelHandler = amqpChannelHandler
pipeline.addLast(amqpChannelHandler)
}
}
fun start() {
lock.withLock {
log.info("connect to: $currentTarget")
if (started) {
log.info("Already connected to: $currentTarget so returning")
return
}
log.info("Connect to: $currentTarget")
workerGroup = sharedThreadPool ?: NioEventLoopGroup(NUM_CLIENT_THREADS)
started = true
restart()
}
}
@ -161,6 +262,10 @@ class AMQPClient(val targets: List<NetworkHostAndPort>,
val bootstrap = Bootstrap()
// TODO Needs more configuration control when we profile. e.g. to use EPOLL on Linux
bootstrap.group(workerGroup).channel(NioSocketChannel::class.java).handler(ClientChannelInitializer(this))
// Delegate DNS Resolution to the proxy side, if we are using proxy.
if (configuration.proxyConfig != null) {
bootstrap.resolver(NoopAddressResolverGroup.INSTANCE)
}
currentTarget = targets[targetIndex]
val clientFuture = bootstrap.connect(currentTarget.host, currentTarget.port)
clientFuture.addListener(connectListener)
@ -168,21 +273,17 @@ class AMQPClient(val targets: List<NetworkHostAndPort>,
fun stop() {
lock.withLock {
log.info("disconnect from: $currentTarget")
stopping = true
try {
if (sharedThreadPool == null) {
workerGroup?.shutdownGracefully()
workerGroup?.terminationFuture()?.sync()
} else {
clientChannel?.close()?.sync()
}
clientChannel = null
workerGroup = null
} finally {
stopping = false
log.info("Stopping connection to: $currentTarget, Local address: $localAddressString")
started = false
if (sharedThreadPool == null) {
workerGroup?.shutdownGracefully()
workerGroup?.terminationFuture()?.sync()
} else {
clientChannel?.close()?.sync()
}
log.info("stopped connection to $currentTarget")
clientChannel = null
workerGroup = null
log.info("Stopped connection to $currentTarget")
}
}
@ -191,7 +292,7 @@ class AMQPClient(val targets: List<NetworkHostAndPort>,
val connected: Boolean
get() {
val channel = lock.withLock { clientChannel }
return channel?.isActive ?: false
return isChannelWritable(channel)
}
fun createMessage(payload: ByteArray,
@ -204,13 +305,17 @@ class AMQPClient(val targets: List<NetworkHostAndPort>,
fun write(msg: SendableMessage) {
val channel = clientChannel
if (channel == null) {
if (channel == null || !isChannelWritable(channel)) {
throw IllegalStateException("Connection to $targets not active")
} else {
channel.writeAndFlush(msg)
}
}
private fun isChannelWritable(channel: Channel?): Boolean {
return channel?.let { channel.isOpen && channel.isActive && amqpActive } ?: false
}
private val _onReceive = PublishSubject.create<ReceivedMessage>().toSerialized()
val onReceive: Observable<ReceivedMessage>
get() = _onReceive

45
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPConfiguration.kt
View File

@ -2,7 +2,7 @@ package net.corda.nodeapi.internal.protonwrapper.netty
import net.corda.nodeapi.internal.ArtemisMessagingComponent
import net.corda.nodeapi.internal.config.CertificateStore
import java.security.KeyStore
import net.corda.nodeapi.internal.config.DEFAULT_SSL_HANDSHAKE_TIMEOUT_MILLIS
interface AMQPConfiguration {
/**
@ -32,12 +32,11 @@ interface AMQPConfiguration {
val trustStore: CertificateStore
/**
* Setting crlCheckSoftFail to true allows certificate paths where some leaf certificates do not contain cRLDistributionPoints
* and also allows validation to continue if the CRL distribution server is not contactable.
* Control how CRL check will be performed.
*/
@JvmDefault
val crlCheckSoftFail: Boolean
get() = true
val revocationConfig: RevocationConfig
get() = RevocationConfigImpl(RevocationConfig.Mode.SOFT_FAIL)
/**
* Enables full debug tracing of all netty and AMQP level packets. This logs aat very high volume and is only for developers.
@ -51,5 +50,41 @@ interface AMQPConfiguration {
* but currently that is deferred to Artemis and the bridge code.
*/
val maxMessageSize: Int
@JvmDefault
val proxyConfig: ProxyConfig?
get() = null
@JvmDefault
val sourceX500Name: String?
get() = null
/**
* Whether to use the tcnative open/boring SSL provider or the default Java SSL provider
*/
@JvmDefault
val useOpenSsl: Boolean
get() = false
@JvmDefault
val sslHandshakeTimeout: Long
get() = DEFAULT_SSL_HANDSHAKE_TIMEOUT_MILLIS // Aligned with sun.security.provider.certpath.URICertStore.DEFAULT_CRL_CONNECT_TIMEOUT
/**
* An optional Health Check Phrase which if passed through the channel will cause AMQP Server to echo it back instead of doing normal pipeline processing
*/
val healthCheckPhrase: String?
get() = null
/**
* An optional set of IPv4/IPv6 remote address strings which will be compared to the remote address of inbound connections and these will only log at TRACE level
*/
@JvmDefault
val silencedIPs: Set<String>
get() = emptySet()
@JvmDefault
val enableSNI: Boolean
get() = true
}

75
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPServer.kt
View File

@ -2,6 +2,7 @@ package net.corda.nodeapi.internal.protonwrapper.netty
import io.netty.bootstrap.ServerBootstrap
import io.netty.channel.Channel
import io.netty.channel.ChannelHandler
import io.netty.channel.ChannelInitializer
import io.netty.channel.ChannelOption
import io.netty.channel.EventLoopGroup
@ -14,6 +15,7 @@ import io.netty.util.internal.logging.InternalLoggerFactory
import io.netty.util.internal.logging.Slf4JLoggerFactory
import net.corda.core.utilities.NetworkHostAndPort
import net.corda.core.utilities.contextLogger
import net.corda.core.utilities.debug
import net.corda.nodeapi.internal.protonwrapper.messages.ReceivedMessage
import net.corda.nodeapi.internal.protonwrapper.messages.SendableMessage
import net.corda.nodeapi.internal.protonwrapper.messages.impl.SendableMessageImpl
@ -31,7 +33,6 @@ import kotlin.concurrent.withLock
/**
* This create a socket acceptor instance that can receive possibly multiple AMQP connections.
* As of now this is not used outside of testing, but in future it will be used for standalone bridging components.
*/
class AMQPServer(val hostName: String,
val port: Int,
@ -42,8 +43,10 @@ class AMQPServer(val hostName: String,
InternalLoggerFactory.setDefaultFactory(Slf4JLoggerFactory.INSTANCE)
}
private const val CORDA_AMQP_NUM_SERVER_THREAD_PROP_NAME = "net.corda.nodeapi.amqpserver.NumServerThreads"
private val log = contextLogger()
const val NUM_SERVER_THREADS = 4
private val NUM_SERVER_THREADS = Integer.getInteger(CORDA_AMQP_NUM_SERVER_THREAD_PROP_NAME, 4)
}
private val lock = ReentrantLock()
@ -60,29 +63,59 @@ class AMQPServer(val hostName: String,
private val conf = parent.configuration
init {
keyManagerFactory.init(conf.keyStore)
trustManagerFactory.init(initialiseTrustStoreAndEnableCrlChecking(conf.trustStore, conf.crlCheckSoftFail))
keyManagerFactory.init(conf.keyStore.value.internal, conf.keyStore.entryPassword.toCharArray())
trustManagerFactory.init(initialiseTrustStoreAndEnableCrlChecking(conf.trustStore, conf.revocationConfig))
}
override fun initChannel(ch: SocketChannel) {
val amqpConfiguration = parent.configuration
val pipeline = ch.pipeline()
val handler = createServerSslHelper(keyManagerFactory, trustManagerFactory)
pipeline.addLast("sslHandler", handler)
amqpConfiguration.healthCheckPhrase?.let { pipeline.addLast(ModeSelectingChannel.NAME, ModeSelectingChannel(it)) }
val (sslHandler, keyManagerFactoriesMap) = createSSLHandler(amqpConfiguration, ch)
pipeline.addLast("sslHandler", sslHandler)
if (conf.trace) pipeline.addLast("logger", LoggingHandler(LogLevel.INFO))
val suppressLogs = ch.remoteAddress()?.hostString in amqpConfiguration.silencedIPs
pipeline.addLast(AMQPChannelHandler(true,
null,
// Passing a mapping of legal names to key managers to be able to pick the correct one after
// SNI completion event is fired up.
keyManagerFactoriesMap,
conf.userName,
conf.password,
conf.trace,
{
parent.clientChannels[it.first.remoteAddress()] = it.first
parent._onConnection.onNext(it.second)
suppressLogs,
onOpen = { channel, change ->
parent.run {
clientChannels[channel.remoteAddress()] = channel
_onConnection.onNext(change)
}
},
{
parent.clientChannels.remove(it.first.remoteAddress())
parent._onConnection.onNext(it.second)
onClose = { channel, change ->
parent.run {
val remoteAddress = channel.remoteAddress()
clientChannels.remove(remoteAddress)
_onConnection.onNext(change)
}
},
{ rcv -> parent._onReceive.onNext(rcv) }))
onReceive = { rcv -> parent._onReceive.onNext(rcv) }))
}
private fun createSSLHandler(amqpConfig: AMQPConfiguration, ch: SocketChannel): Pair<ChannelHandler, Map<String, CertHoldingKeyManagerFactoryWrapper>> {
return if (amqpConfig.useOpenSsl && amqpConfig.enableSNI && amqpConfig.keyStore.aliases().size > 1) {
val keyManagerFactoriesMap = splitKeystore(amqpConfig)
// SNI matching needed only when multiple nodes exist behind the server.
Pair(createServerSNIOpenSslHandler(keyManagerFactoriesMap, trustManagerFactory), keyManagerFactoriesMap)
} else {
val keyManagerFactory = CertHoldingKeyManagerFactoryWrapper(keyManagerFactory, amqpConfig)
val handler = if (amqpConfig.useOpenSsl) {
createServerOpenSslHandler(keyManagerFactory, trustManagerFactory, ch.alloc())
} else {
// For javaSSL, SNI matching is handled at key manager level.
createServerSslHandler(amqpConfig.keyStore, keyManagerFactory, trustManagerFactory)
}
handler.handshakeTimeoutMillis = amqpConfig.sslHandshakeTimeout
Pair(handler, mapOf(DEFAULT to keyManagerFactory))
}
}
}
@ -95,7 +128,10 @@ class AMQPServer(val hostName: String,
val server = ServerBootstrap()
// TODO Needs more configuration control when we profile. e.g. to use EPOLL on Linux
server.group(bossGroup, workerGroup).channel(NioServerSocketChannel::class.java).option(ChannelOption.SO_BACKLOG, 100).handler(LoggingHandler(LogLevel.INFO)).childHandler(ServerChannelInitializer(this))
server.group(bossGroup, workerGroup).channel(NioServerSocketChannel::class.java)
.option(ChannelOption.SO_BACKLOG, 100)
.handler(NettyServerEventLogger(LogLevel.INFO, configuration.silencedIPs))
.childHandler(ServerChannelInitializer(this))
log.info("Try to bind $port")
val channelFuture = server.bind(hostName, port).sync() // block/throw here as better to know we failed to claim port than carry on
@ -144,7 +180,7 @@ class AMQPServer(val hostName: String,
requireMessageSize(payload.size, configuration.maxMessageSize)
val dest = InetSocketAddress(destinationLink.host, destinationLink.port)
require(dest in clientChannels.keys) {
"Destination not available"
"Destination $dest is not available"
}
return SendableMessageImpl(payload, topic, destinationLegalName, destinationLink, properties)
}
@ -155,21 +191,22 @@ class AMQPServer(val hostName: String,
if (channel == null) {
throw IllegalStateException("Connection to ${msg.destinationLink} not active")
} else {
log.debug { "Writing message with payload of size ${msg.payload.size} into channel $channel" }
channel.writeAndFlush(msg)
log.debug { "Done writing message with payload of size ${msg.payload.size} into channel $channel" }
}
}
fun dropConnection(connectionRemoteHost: InetSocketAddress) {
val channel = clientChannels[connectionRemoteHost]
if (channel != null) {
channel.close()
}
clientChannels[connectionRemoteHost]?.close()
}
fun complete(delivery: Delivery, target: InetSocketAddress) {
val channel = clientChannels[target]
channel?.apply {
log.debug { "Writing delivery $delivery into channel $channel" }
writeAndFlush(delivery)
log.debug { "Done writing delivery $delivery into channel $channel" }
}
}

60
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AliasProvidingKeyMangerWrapper.kt Normal file
View File

@ -0,0 +1,60 @@
package net.corda.nodeapi.internal.protonwrapper.netty
import java.net.Socket
import java.security.Principal
import javax.net.ssl.SSLEngine
import javax.net.ssl.X509ExtendedKeyManager
import javax.net.ssl.X509KeyManager
interface AliasProvidingKeyMangerWrapper : X509KeyManager {
var lastAlias: String?
}
class AliasProvidingKeyMangerWrapperImpl(private val keyManager: X509KeyManager) : AliasProvidingKeyMangerWrapper, X509KeyManager by keyManager {
override var lastAlias: String? = null
override fun chooseServerAlias(keyType: String?, issuers: Array<out Principal>?, socket: Socket?): String? {
return storeIfNotNull { keyManager.chooseServerAlias(keyType, issuers, socket) }
}
override fun chooseClientAlias(keyType: Array<out String>?, issuers: Array<out Principal>?, socket: Socket?): String? {
return storeIfNotNull { keyManager.chooseClientAlias(keyType, issuers, socket) }
}
private fun storeIfNotNull(func: () -> String?): String? {
val alias = func()
if (alias != null) {
lastAlias = alias
}
return alias
}
}
class AliasProvidingExtendedKeyMangerWrapper(private val keyManager: X509ExtendedKeyManager) : X509ExtendedKeyManager(), X509KeyManager by keyManager, AliasProvidingKeyMangerWrapper {
override var lastAlias: String? = null
override fun chooseServerAlias(keyType: String?, issuers: Array<out Principal>?, socket: Socket?): String? {
return storeIfNotNull { keyManager.chooseServerAlias(keyType, issuers, socket) }
}
override fun chooseClientAlias(keyType: Array<out String>?, issuers: Array<out Principal>?, socket: Socket?): String? {
return storeIfNotNull { keyManager.chooseClientAlias(keyType, issuers, socket) }
}
override fun chooseEngineClientAlias(keyType: Array<out String>?, issuers: Array<out Principal>?, engine: SSLEngine?): String? {
return storeIfNotNull { keyManager.chooseEngineClientAlias(keyType, issuers, engine) }
}
override fun chooseEngineServerAlias(keyType: String?, issuers: Array<out Principal>?, engine: SSLEngine?): String? {
return storeIfNotNull { keyManager.chooseEngineServerAlias(keyType, issuers, engine) }
}
private fun storeIfNotNull(func: () -> String?): String? {
val alias = func()
if (alias != null) {
lastAlias = alias
}
return alias
}
}

34
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AllowAllRevocationChecker.kt Normal file
View File

@ -0,0 +1,34 @@
package net.corda.nodeapi.internal.protonwrapper.netty
import net.corda.core.utilities.debug
import org.slf4j.LoggerFactory
import java.security.cert.CertPathValidatorException
import java.security.cert.Certificate
import java.security.cert.PKIXRevocationChecker
import java.util.*
object AllowAllRevocationChecker : PKIXRevocationChecker() {
private val logger = LoggerFactory.getLogger(AllowAllRevocationChecker::class.java)
override fun check(cert: Certificate?, unresolvedCritExts: MutableCollection<String>?) {
logger.debug {"Passing certificate check for: $cert"}
// Nothing to do
}
override fun isForwardCheckingSupported(): Boolean {
return true
}
override fun getSupportedExtensions(): MutableSet<String>? {
return null
}
override fun init(forward: Boolean) {
// Nothing to do
}
override fun getSoftFailExceptions(): MutableList<CertPathValidatorException> {
return LinkedList()
}
}

81
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/CertHoldingKeyManagerFactoryWrapper.kt Normal file
View File

@ -0,0 +1,81 @@
package net.corda.nodeapi.internal.protonwrapper.netty
import java.security.KeyStore
import java.security.cert.X509Certificate
import javax.net.ssl.KeyManager
import javax.net.ssl.KeyManagerFactory
import javax.net.ssl.KeyManagerFactorySpi
import javax.net.ssl.ManagerFactoryParameters
import javax.net.ssl.X509ExtendedKeyManager
import javax.net.ssl.X509KeyManager
class CertHoldingKeyManagerFactorySpiWrapper(private val factorySpi: KeyManagerFactorySpi, private val amqpConfig: AMQPConfiguration) : KeyManagerFactorySpi() {
override fun engineInit(keyStore: KeyStore?, password: CharArray?) {
val engineInitMethod = KeyManagerFactorySpi::class.java.getDeclaredMethod("engineInit", KeyStore::class.java, CharArray::class.java)
engineInitMethod.isAccessible = true
engineInitMethod.invoke(factorySpi, keyStore, password)
}
override fun engineInit(spec: ManagerFactoryParameters?) {
val engineInitMethod = KeyManagerFactorySpi::class.java.getDeclaredMethod("engineInit", ManagerFactoryParameters::class.java)
engineInitMethod.isAccessible = true
engineInitMethod.invoke(factorySpi, spec)
}
private fun getKeyManagersImpl(): Array<KeyManager> {
val engineGetKeyManagersMethod = KeyManagerFactorySpi::class.java.getDeclaredMethod("engineGetKeyManagers")
engineGetKeyManagersMethod.isAccessible = true
@Suppress("UNCHECKED_CAST")
val keyManagers = engineGetKeyManagersMethod.invoke(factorySpi) as Array<KeyManager>
return if (factorySpi is CertHoldingKeyManagerFactorySpiWrapper) keyManagers else keyManagers.map {
val aliasProvidingKeyManager = getDefaultKeyManager(it)
// Use the SNIKeyManager if keystore has several entries and only for clients and non-openSSL servers.
// Condition of using SNIKeyManager: if its client, or JDKSsl server.
val isClient = amqpConfig.sourceX500Name != null
val enableSNI = amqpConfig.enableSNI && amqpConfig.keyStore.aliases().size > 1
if (enableSNI && (isClient || !amqpConfig.useOpenSsl)) {
SNIKeyManager(aliasProvidingKeyManager as X509ExtendedKeyManager, amqpConfig)
} else {
aliasProvidingKeyManager
}
}.toTypedArray()
}
private fun getDefaultKeyManager(keyManager: KeyManager): KeyManager {
return when (keyManager) {
is X509ExtendedKeyManager -> AliasProvidingExtendedKeyMangerWrapper(keyManager)
is X509KeyManager -> AliasProvidingKeyMangerWrapperImpl(keyManager)
else -> throw UnsupportedOperationException("Supported key manager types are: X509ExtendedKeyManager, X509KeyManager. Provided ${keyManager::class.java.name}")
}
}
private val keyManagers = lazy { getKeyManagersImpl() }
override fun engineGetKeyManagers(): Array<KeyManager> {
return keyManagers.value
}
}
/**
* You can wrap a key manager factory in this class if you need to get the cert chain currently used to identify or
* verify. When using for TLS channels, make sure to wrap the (singleton) factory separately on each channel, as
* the wrapper is not thread safe as in it will return the last used alias/cert chain and has itself no notion
* of belonging to a certain channel.
*/
class CertHoldingKeyManagerFactoryWrapper(factory: KeyManagerFactory, amqpConfig: AMQPConfiguration) : KeyManagerFactory(getFactorySpi(factory, amqpConfig), factory.provider, factory.algorithm) {
companion object {
private fun getFactorySpi(factory: KeyManagerFactory, amqpConfig: AMQPConfiguration): KeyManagerFactorySpi {
val spiField = KeyManagerFactory::class.java.getDeclaredField("factorySpi")
spiField.isAccessible = true
return CertHoldingKeyManagerFactorySpiWrapper(spiField.get(factory) as KeyManagerFactorySpi, amqpConfig)
}
}
fun getCurrentCertChain(): Array<out X509Certificate>? {
val keyManager = keyManagers.firstOrNull()
val alias = if (keyManager is AliasProvidingKeyMangerWrapper) keyManager.lastAlias else null
return if (alias != null && keyManager is X509KeyManager) {
keyManager.getCertificateChain(alias)
} else null
}
}

12
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/ExternalCrlSource.kt Normal file
View File

@ -0,0 +1,12 @@
package net.corda.nodeapi.internal.protonwrapper.netty
import java.security.cert.X509CRL
import java.security.cert.X509Certificate
interface ExternalCrlSource {
/**
* Given certificate provides a set of CRLs, potentially performing remote communication.
*/
fun fetch(certificate: X509Certificate) : Set<X509CRL>
}

76
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/ModeSelectingChannel.kt Normal file
View File

@ -0,0 +1,76 @@
package net.corda.nodeapi.internal.protonwrapper.netty
import io.netty.buffer.ByteBuf
import io.netty.buffer.Unpooled
import io.netty.channel.ChannelHandlerContext
import io.netty.handler.codec.ByteToMessageDecoder
import io.netty.handler.ssl.SslHandler
import net.corda.core.utilities.contextLogger
/**
* Responsible for deciding whether we are likely to be processing health probe request
* or this is a normal SSL/AMQP processing pipeline
*/
internal class ModeSelectingChannel(healthCheckPhrase: String) : ByteToMessageDecoder() {
companion object {
const val NAME = "modeSelector"
private val log = contextLogger()
}
private enum class TriState {
UNDECIDED,
ECHO_MODE,
NORMAL_MODE
}
private val healthCheckPhraseArray = healthCheckPhrase.toByteArray(Charsets.UTF_8)
private var currentMode = TriState.UNDECIDED
private var alreadyEchoedPos = 0
override fun decode(ctx: ChannelHandlerContext, inByteBuf: ByteBuf, out: MutableList<Any>?) {
fun ChannelHandlerContext.echoBack(inByteBuf: ByteBuf) {
// WriteAndFlush() will decrement count and will blow unless we retain first
// And we have to ensure we are not sending the same information multiple times
val toBeWritten = inByteBuf.retainedSlice(alreadyEchoedPos, inByteBuf.readableBytes() - alreadyEchoedPos)
writeAndFlush(toBeWritten)
alreadyEchoedPos = inByteBuf.readableBytes()
}
if(currentMode == TriState.ECHO_MODE) {
ctx.echoBack(inByteBuf)
return
}
// Wait until the length prefix is available.
if (inByteBuf.readableBytes() < healthCheckPhraseArray.size) {
return
}
// Direct buffers do not allow calling `.array()` on them, see `io.netty.buffer.UnpooledDirectByteBuf.array`
val incomingArray = Unpooled.copiedBuffer(inByteBuf).array()
val zipped = healthCheckPhraseArray.zip(incomingArray)
if (zipped.all { it.first == it.second }) {
// Matched the healthCheckPhrase
currentMode = TriState.ECHO_MODE
log.info("Echo mode activated for connection ${ctx.channel().id()}")
// Cancel scheduled action to avoid SSL handshake timeout, which starts "ticking" upon connection is established,
// namely upon call to `io.netty.handler.ssl.SslHandler#handlerAdded` is made
ctx.pipeline().get(SslHandler::class.java)?.handshakeFuture()?.cancel(false)
ctx.echoBack(inByteBuf)
} else {
currentMode = TriState.NORMAL_MODE
// Remove self from pipeline and replay all the messages received down the pipeline
// It is important to bump-up reference count as pipeline removal decrements it by one.
inByteBuf.retain()
ctx.pipeline().remove(this)
ctx.fireChannelRead(inByteBuf)
}
}
}

73
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/NettyServerEventLogger.kt Normal file
View File

@ -0,0 +1,73 @@
package net.corda.nodeapi.internal.protonwrapper.netty
import io.netty.channel.ChannelDuplexHandler
import io.netty.channel.ChannelHandler
import io.netty.channel.ChannelHandlerContext
import io.netty.channel.ChannelPromise
import io.netty.handler.logging.LogLevel
import io.netty.util.internal.logging.InternalLogLevel
import io.netty.util.internal.logging.InternalLogger
import io.netty.util.internal.logging.InternalLoggerFactory
import java.net.SocketAddress
@ChannelHandler.Sharable
class NettyServerEventLogger(level: LogLevel = DEFAULT_LEVEL, val silencedIPs: Set<String> = emptySet()) : ChannelDuplexHandler() {
companion object {
val DEFAULT_LEVEL: LogLevel = LogLevel.DEBUG
}
private val logger: InternalLogger = InternalLoggerFactory.getInstance(javaClass)
private val internalLevel: InternalLogLevel = level.toInternalLevel()
@Throws(Exception::class)
override fun channelActive(ctx: ChannelHandlerContext) {
if (logger.isEnabled(internalLevel)) {
logger.log(internalLevel, "Server socket ${ctx.channel()} ACTIVE")
}
ctx.fireChannelActive()
}
@Throws(Exception::class)
override fun channelInactive(ctx: ChannelHandlerContext) {
if (logger.isEnabled(internalLevel)) {
logger.log(internalLevel, "Server socket ${ctx.channel()} INACTIVE")
}
ctx.fireChannelInactive()
}
@Suppress("OverridingDeprecatedMember")
@Throws(Exception::class)
override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) {
if (logger.isEnabled(internalLevel)) {
logger.log(internalLevel, "Server socket ${ctx.channel()} EXCEPTION ${cause.message}", cause)
}
ctx.fireExceptionCaught(cause)
}
@Throws(Exception::class)
override fun bind(ctx: ChannelHandlerContext, localAddress: SocketAddress, promise: ChannelPromise) {
if (logger.isEnabled(internalLevel)) {
logger.log(internalLevel, "Server socket ${ctx.channel()} BIND $localAddress")
}
ctx.bind(localAddress, promise)
}
@Throws(Exception::class)
override fun close(ctx: ChannelHandlerContext, promise: ChannelPromise) {
if (logger.isEnabled(internalLevel)) {
logger.log(internalLevel, "Server socket ${ctx.channel()} CLOSE")
}
ctx.close(promise)
}
@Throws(Exception::class)
override fun channelRead(ctx: ChannelHandlerContext, msg: Any) {
val level = if (msg is io.netty.channel.socket.SocketChannel) { // Should always be the case as this is a server socket, but be defensive
if (msg.remoteAddress()?.hostString !in silencedIPs) internalLevel else InternalLogLevel.TRACE
} else internalLevel
if (logger.isEnabled(level)) {
logger.log(level, "Server socket ${ctx.channel()} ACCEPTED $msg")
}
ctx.fireChannelRead(msg)
}
}

83
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/RevocationConfig.kt Normal file
View File

@ -0,0 +1,83 @@
package net.corda.nodeapi.internal.protonwrapper.netty
import com.typesafe.config.Config
import net.corda.nodeapi.internal.config.ConfigParser
import net.corda.nodeapi.internal.config.CustomConfigParser
/**
* Data structure for controlling the way how Certificate Revocation Lists are handled.
*/
@CustomConfigParser(RevocationConfigParser::class)
interface RevocationConfig {
enum class Mode {
/**
* @see java.security.cert.PKIXRevocationChecker.Option.SOFT_FAIL
*/
SOFT_FAIL,
/**
* Opposite of SOFT_FAIL - i.e. most rigorous check.
* Among other things, this check requires that CRL checking URL is available on every level of certificate chain.
* This is also known as Strict mode.
*/
HARD_FAIL,
/**
* CRLs are obtained from external source
* @see ExternalCrlSource
*/
EXTERNAL_SOURCE,
/**
* Switch CRL check off.
*/
OFF
}
val mode: Mode
/**
* Optional `ExternalCrlSource` which only makes sense with `mode` = `EXTERNAL_SOURCE`
*/
val externalCrlSource: ExternalCrlSource?
/**
* Creates a copy of `RevocationConfig` with ExternalCrlSource enriched
*/
fun enrichExternalCrlSource(sourceFunc: (() -> ExternalCrlSource)?): RevocationConfig
}
/**
* Maintained for legacy purposes to convert old style `crlCheckSoftFail`.
*/
fun Boolean.toRevocationConfig() = if(this) RevocationConfigImpl(RevocationConfig.Mode.SOFT_FAIL) else RevocationConfigImpl(RevocationConfig.Mode.HARD_FAIL)
data class RevocationConfigImpl(override val mode: RevocationConfig.Mode, override val externalCrlSource: ExternalCrlSource? = null) : RevocationConfig {
override fun enrichExternalCrlSource(sourceFunc: (() -> ExternalCrlSource)?): RevocationConfig {
return if(mode != RevocationConfig.Mode.EXTERNAL_SOURCE) {
this
} else {
assert(sourceFunc != null) { "There should be a way to obtain ExternalCrlSource" }
copy(externalCrlSource = sourceFunc!!())
}
}
}
class RevocationConfigParser : ConfigParser<RevocationConfig> {
override fun parse(config: Config): RevocationConfig {
val oneAndTheOnly = "mode"
val allKeys = config.entrySet().map { it.key }
require(allKeys.size == 1 && allKeys.contains(oneAndTheOnly)) {"For RevocationConfig, it is expected to have '$oneAndTheOnly' property only. " +
"Actual set of properties: $allKeys. Please check 'revocationConfig' section."}
val mode = config.getString(oneAndTheOnly)
return when (mode.toUpperCase()) {
"SOFT_FAIL" -> RevocationConfigImpl(RevocationConfig.Mode.SOFT_FAIL)
"HARD_FAIL" -> RevocationConfigImpl(RevocationConfig.Mode.HARD_FAIL)
"EXTERNAL_SOURCE" -> RevocationConfigImpl(RevocationConfig.Mode.EXTERNAL_SOURCE, null) // null for now till `enrichExternalCrlSource` is called
"OFF" -> RevocationConfigImpl(RevocationConfig.Mode.OFF)
else -> throw IllegalArgumentException("Unsupported mode : '$mode'")
}
}
}

112
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SNIKeyManager.kt Normal file
View File

@ -0,0 +1,112 @@
package net.corda.nodeapi.internal.protonwrapper.netty
import net.corda.core.identity.CordaX500Name
import net.corda.core.utilities.contextLogger
import net.corda.nodeapi.internal.config.CertificateStore
import net.corda.nodeapi.internal.crypto.x509
import org.slf4j.MDC
import java.net.Socket
import java.security.Principal
import javax.net.ssl.SNIMatcher
import javax.net.ssl.SSLEngine
import javax.net.ssl.SSLSocket
import javax.net.ssl.X509ExtendedKeyManager
import javax.net.ssl.X509KeyManager
internal class SNIKeyManager(private val keyManager: X509ExtendedKeyManager, private val amqpConfig: AMQPConfiguration) : X509ExtendedKeyManager(), X509KeyManager by keyManager, AliasProvidingKeyMangerWrapper {
companion object {
private val log = contextLogger()
}
override var lastAlias: String? = null
private fun withMDC(block: () -> Unit) {
val oldMDC = MDC.getCopyOfContextMap()
try {
MDC.put("lastAlias", lastAlias)
MDC.put("isServer", amqpConfig.sourceX500Name.isNullOrEmpty().toString())
MDC.put("sourceX500Name", amqpConfig.sourceX500Name)
MDC.put("useOpenSSL", amqpConfig.useOpenSsl.toString())
block()
} finally {
MDC.setContextMap(oldMDC)
}
}
private fun logDebugWithMDC(msg: () -> String) {
if (log.isDebugEnabled) {
withMDC { log.debug(msg()) }
}
}
override fun chooseClientAlias(keyType: Array<out String>, issuers: Array<out Principal>, socket: Socket): String? {
return storeIfNotNull { chooseClientAlias(amqpConfig.keyStore, amqpConfig.sourceX500Name) }
}
override fun chooseEngineClientAlias(keyType: Array<out String>, issuers: Array<out Principal>, engine: SSLEngine): String? {
return storeIfNotNull { chooseClientAlias(amqpConfig.keyStore, amqpConfig.sourceX500Name) }
}
override fun chooseServerAlias(keyType: String?, issuers: Array<out Principal>?, socket: Socket): String? {
return storeIfNotNull {
val matcher = (socket as SSLSocket).sslParameters.sniMatchers.first()
chooseServerAlias(keyType, issuers, matcher)
}
}
override fun chooseEngineServerAlias(keyType: String?, issuers: Array<out Principal>?, engine: SSLEngine?): String? {
return storeIfNotNull {
val matcher = engine?.sslParameters?.sniMatchers?.first()
chooseServerAlias(keyType, issuers, matcher)
}
}
private fun chooseServerAlias(keyType: String?, issuers: Array<out Principal>?, matcher: SNIMatcher?): String? {
val aliases = keyManager.getServerAliases(keyType, issuers)
if (aliases == null || aliases.isEmpty()) {
logDebugWithMDC { "Keystore doesn't contain any aliases for key type $keyType and issuers $issuers." }
return null
}
log.debug("Checking aliases: $aliases.")
matcher?.let {
val matchedAlias = (it as ServerSNIMatcher).matchedAlias
if (aliases.contains(matchedAlias)) {
logDebugWithMDC { "Found match for $matchedAlias." }
return matchedAlias
}
}
logDebugWithMDC { "Unable to find a matching alias." }
return null
}
private fun chooseClientAlias(keyStore: CertificateStore, clientLegalName: String?): String? {
clientLegalName?.let {
val aliases = keyStore.aliases()
if (aliases.isEmpty()) {
logDebugWithMDC { "Keystore doesn't contain any entries." }
}
aliases.forEach { alias ->
val x500Name = keyStore[alias].x509.subjectX500Principal
val aliasCordaX500Name = CordaX500Name.build(x500Name)
val clientCordaX500Name = CordaX500Name.parse(it)
if (clientCordaX500Name == aliasCordaX500Name) {
logDebugWithMDC { "Found alias $alias for $clientCordaX500Name." }
return alias
}
}
}
return null
}
private fun storeIfNotNull(func: () -> String?): String? {
val alias = func()
if (alias != null) {
lastAlias = alias
}
return alias
}
}

230
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelper.kt
View File

@ -1,28 +1,91 @@
package net.corda.nodeapi.internal.protonwrapper.netty
import io.netty.buffer.ByteBufAllocator
import io.netty.handler.ssl.ClientAuth
import io.netty.handler.ssl.SniHandler
import io.netty.handler.ssl.SslContextBuilder
import io.netty.handler.ssl.SslHandler
import io.netty.handler.ssl.SslProvider
import io.netty.util.DomainNameMappingBuilder
import net.corda.core.crypto.SecureHash
import net.corda.core.crypto.newSecureRandom
import net.corda.core.identity.CordaX500Name
import net.corda.core.internal.VisibleForTesting
import net.corda.core.utilities.NetworkHostAndPort
import net.corda.core.utilities.contextLogger
import net.corda.core.utilities.toHex
import net.corda.nodeapi.internal.ArtemisTcpTransport
import net.corda.nodeapi.internal.config.CertificateStore
import net.corda.nodeapi.internal.crypto.toBc
import net.corda.nodeapi.internal.crypto.x509
import net.corda.nodeapi.internal.protonwrapper.netty.revocation.ExternalSourceRevocationChecker
import org.bouncycastle.asn1.ASN1InputStream
import org.bouncycastle.asn1.DERIA5String
import org.bouncycastle.asn1.DEROctetString
import org.bouncycastle.asn1.x509.AuthorityKeyIdentifier
import org.bouncycastle.asn1.x509.CRLDistPoint
import org.bouncycastle.asn1.x509.DistributionPointName
import org.bouncycastle.asn1.x509.Extension
import org.bouncycastle.asn1.x509.GeneralName
import org.bouncycastle.asn1.x509.GeneralNames
import org.bouncycastle.asn1.x509.SubjectKeyIdentifier
import org.slf4j.LoggerFactory
import java.io.ByteArrayInputStream
import java.net.Socket
import java.security.KeyStore
import java.security.cert.*
import java.util.*
import java.util.concurrent.Executor
import javax.net.ssl.*
import kotlin.system.measureTimeMillis
private const val HOSTNAME_FORMAT = "%s.corda.net"
private const val SSL_HANDSHAKE_TIMEOUT_PROP_NAME = "corda.netty.sslHelper.handshakeTimeout"
private const val DEFAULT_SSL_TIMEOUT = 20000 // Aligned with sun.security.provider.certpath.URICertStore.DEFAULT_CRL_CONNECT_TIMEOUT
internal const val DEFAULT = "default"
internal class LoggingTrustManagerWrapper(val wrapped: X509ExtendedTrustManager) : X509ExtendedTrustManager() {
internal const val DP_DEFAULT_ANSWER = "NO CRLDP ext"
internal val logger = LoggerFactory.getLogger("net.corda.nodeapi.internal.protonwrapper.netty.SSLHelper")
fun X509Certificate.distributionPoints() : Set<String>? {
logger.debug("Checking CRLDPs for $subjectX500Principal")
val crldpExtBytes = getExtensionValue(Extension.cRLDistributionPoints.id)
if (crldpExtBytes == null) {
logger.debug(DP_DEFAULT_ANSWER)
return emptySet()
}
val derObjCrlDP = ASN1InputStream(ByteArrayInputStream(crldpExtBytes)).readObject()
val dosCrlDP = derObjCrlDP as? DEROctetString
if (dosCrlDP == null) {
logger.error("Expected to have DEROctetString, actual type: ${derObjCrlDP.javaClass}")
return emptySet()
}
val crldpExtOctetsBytes = dosCrlDP.octets
val dpObj = ASN1InputStream(ByteArrayInputStream(crldpExtOctetsBytes)).readObject()
val distPoint = CRLDistPoint.getInstance(dpObj)
if (distPoint == null) {
logger.error("Could not instantiate CRLDistPoint, from: $dpObj")
return emptySet()
}
val dpNames = distPoint.distributionPoints.mapNotNull { it.distributionPoint }.filter { it.type == DistributionPointName.FULL_NAME }
val generalNames = dpNames.flatMap { GeneralNames.getInstance(it.name).names.asList() }
return generalNames.filter { it.tagNo == GeneralName.uniformResourceIdentifier}.map { DERIA5String.getInstance(it.name).string }.toSet()
}
fun X509Certificate.distributionPointsToString() : String {
return with(distributionPoints()) {
if(this == null || isEmpty()) {
DP_DEFAULT_ANSWER
} else {
sorted().joinToString()
}
}
}
@VisibleForTesting
class LoggingTrustManagerWrapper(val wrapped: X509ExtendedTrustManager) : X509ExtendedTrustManager() {
companion object {
val log = contextLogger()
}
@ -45,12 +108,11 @@ internal class LoggingTrustManagerWrapper(val wrapped: X509ExtendedTrustManager)
} catch (ex: Exception) {
"null"
}
" $subject[$keyIdentifier] issued by $issuer[$authorityKeyIdentifier]"
" $subject[$keyIdentifier] issued by $issuer[$authorityKeyIdentifier] [${it.distributionPointsToString()}]"
}
return certs.joinToString("\r\n")
}
private fun certPathToStringFull(chain: Array<out X509Certificate>?): String {
if (chain == null) {
return "<empty certpath>"
@ -107,6 +169,33 @@ internal class LoggingTrustManagerWrapper(val wrapped: X509ExtendedTrustManager)
}
private object LoggingImmediateExecutor : Executor {
override fun execute(command: Runnable?) {
val log = LoggerFactory.getLogger(javaClass)
if (command == null) {
log.error("SSL handler executor called with a null command")
throw NullPointerException("command")
}
@Suppress("TooGenericExceptionCaught", "MagicNumber") // log and rethrow all exceptions
try {
val commandName = command::class.qualifiedName?.let { "[$it]" } ?: ""
log.debug("Entering SSL command $commandName")
val elapsedTime = measureTimeMillis { command.run() }
log.debug("Exiting SSL command $elapsedTime millis")
if (elapsedTime > 100) {
log.info("Command: $commandName took $elapsedTime millis to execute")
}
}
catch (ex: Exception) {
log.error("Caught exception in SSL handler executor", ex)
throw ex
}
}
}
internal fun createClientSslHelper(target: NetworkHostAndPort,
expectedRemoteLegalNames: Set<CordaX500Name>,
keyManagerFactory: KeyManagerFactory,
@ -125,13 +214,31 @@ internal fun createClientSslHelper(target: NetworkHostAndPort,
sslParameters.serverNames = listOf(SNIHostName(x500toHostName(expectedRemoteLegalNames.single())))
sslEngine.sslParameters = sslParameters
}
val sslHandler = SslHandler(sslEngine)
sslHandler.handshakeTimeoutMillis = Integer.getInteger(SSL_HANDSHAKE_TIMEOUT_PROP_NAME, DEFAULT_SSL_TIMEOUT).toLong()
return sslHandler
@Suppress("DEPRECATION")
return SslHandler(sslEngine, false, LoggingImmediateExecutor)
}
internal fun createServerSslHelper(keyManagerFactory: KeyManagerFactory,
trustManagerFactory: TrustManagerFactory): SslHandler {
internal fun createClientOpenSslHandler(target: NetworkHostAndPort,
expectedRemoteLegalNames: Set<CordaX500Name>,
keyManagerFactory: KeyManagerFactory,
trustManagerFactory: TrustManagerFactory,
alloc: ByteBufAllocator): SslHandler {
val sslContext = SslContextBuilder.forClient().sslProvider(SslProvider.OPENSSL).keyManager(keyManagerFactory).trustManager(LoggingTrustManagerFactoryWrapper(trustManagerFactory)).build()
val sslEngine = sslContext.newEngine(alloc, target.host, target.port)
sslEngine.enabledProtocols = ArtemisTcpTransport.TLS_VERSIONS.toTypedArray()
sslEngine.enabledCipherSuites = ArtemisTcpTransport.CIPHER_SUITES.toTypedArray()
if (expectedRemoteLegalNames.size == 1) {
val sslParameters = sslEngine.sslParameters
sslParameters.serverNames = listOf(SNIHostName(x500toHostName(expectedRemoteLegalNames.single())))
sslEngine.sslParameters = sslParameters
}
@Suppress("DEPRECATION")
return SslHandler(sslEngine, false, LoggingImmediateExecutor)
}
internal fun createServerSslHandler(keyStore: CertificateStore,
keyManagerFactory: KeyManagerFactory,
trustManagerFactory: TrustManagerFactory): SslHandler {
val sslContext = SSLContext.getInstance("TLS")
val keyManagers = keyManagerFactory.keyManagers
val trustManagers = trustManagerFactory.trustManagers.filterIsInstance(X509ExtendedTrustManager::class.java).map { LoggingTrustManagerWrapper(it) }.toTypedArray()
@ -142,35 +249,106 @@ internal fun createServerSslHelper(keyManagerFactory: KeyManagerFactory,
sslEngine.enabledProtocols = ArtemisTcpTransport.TLS_VERSIONS.toTypedArray()
sslEngine.enabledCipherSuites = ArtemisTcpTransport.CIPHER_SUITES.toTypedArray()
sslEngine.enableSessionCreation = true
val sslHandler = SslHandler(sslEngine)
sslHandler.handshakeTimeoutMillis = Integer.getInteger(SSL_HANDSHAKE_TIMEOUT_PROP_NAME, DEFAULT_SSL_TIMEOUT).toLong()
return sslHandler
val sslParameters = sslEngine.sslParameters
sslParameters.sniMatchers = listOf(ServerSNIMatcher(keyStore))
sslEngine.sslParameters = sslParameters
@Suppress("DEPRECATION")
return SslHandler(sslEngine, false, LoggingImmediateExecutor)
}
internal fun initialiseTrustStoreAndEnableCrlChecking(trustStore: CertificateStore, crlCheckSoftFail: Boolean): ManagerFactoryParameters {
val certPathBuilder = CertPathBuilder.getInstance("PKIX")
val revocationChecker = certPathBuilder.revocationChecker as PKIXRevocationChecker
revocationChecker.options = EnumSet.of(
// Prefer CRL over OCSP
PKIXRevocationChecker.Option.PREFER_CRLS,
// Don't fall back to OCSP checking
PKIXRevocationChecker.Option.NO_FALLBACK)
if (crlCheckSoftFail) {
// Allow revocation check to succeed if the revocation status cannot be determined for one of
// the following reasons: The CRL or OCSP response cannot be obtained because of a network error.
revocationChecker.options = revocationChecker.options + PKIXRevocationChecker.Option.SOFT_FAIL
}
@VisibleForTesting
fun initialiseTrustStoreAndEnableCrlChecking(trustStore: CertificateStore, revocationConfig: RevocationConfig): ManagerFactoryParameters {
val pkixParams = PKIXBuilderParameters(trustStore.value.internal, X509CertSelector())
val revocationChecker = when (revocationConfig.mode) {
RevocationConfig.Mode.OFF -> AllowAllRevocationChecker // Custom PKIXRevocationChecker skipping CRL check
RevocationConfig.Mode.EXTERNAL_SOURCE -> {
require(revocationConfig.externalCrlSource != null) { "externalCrlSource must not be null" }
ExternalSourceRevocationChecker(revocationConfig.externalCrlSource!!) { Date() } // Custom PKIXRevocationChecker which uses `externalCrlSource`
}
else -> {
val certPathBuilder = CertPathBuilder.getInstance("PKIX")
val pkixRevocationChecker = certPathBuilder.revocationChecker as PKIXRevocationChecker
pkixRevocationChecker.options = EnumSet.of(
// Prefer CRL over OCSP
PKIXRevocationChecker.Option.PREFER_CRLS,
// Don't fall back to OCSP checking
PKIXRevocationChecker.Option.NO_FALLBACK)
if (revocationConfig.mode == RevocationConfig.Mode.SOFT_FAIL) {
// Allow revocation check to succeed if the revocation status cannot be determined for one of
// the following reasons: The CRL or OCSP response cannot be obtained because of a network error.
pkixRevocationChecker.options = pkixRevocationChecker.options + PKIXRevocationChecker.Option.SOFT_FAIL
}
pkixRevocationChecker
}
}
pkixParams.addCertPathChecker(revocationChecker)
return CertPathTrustManagerParameters(pkixParams)
}
internal fun createServerOpenSslHandler(keyManagerFactory: KeyManagerFactory,
trustManagerFactory: TrustManagerFactory,
alloc: ByteBufAllocator): SslHandler {
val sslContext = getServerSslContextBuilder(keyManagerFactory, trustManagerFactory).build()
val sslEngine = sslContext.newEngine(alloc)
sslEngine.useClientMode = false
@Suppress("DEPRECATION")
return SslHandler(sslEngine, false, LoggingImmediateExecutor)
}
/**
* Creates a special SNI handler used only when openSSL is used for AMQPServer
*/
internal fun createServerSNIOpenSslHandler(keyManagerFactoriesMap: Map<String, KeyManagerFactory>,
trustManagerFactory: TrustManagerFactory): SniHandler {
// Default value can be any in the map.
val sslCtxBuilder = getServerSslContextBuilder(keyManagerFactoriesMap.values.first(), trustManagerFactory)
val mapping = DomainNameMappingBuilder(sslCtxBuilder.build())
keyManagerFactoriesMap.forEach {
mapping.add(it.key, sslCtxBuilder.keyManager(it.value).build())
}
return SniHandler(mapping.build())
}
@Suppress("SpreadOperator")
private fun getServerSslContextBuilder(keyManagerFactory: KeyManagerFactory, trustManagerFactory: TrustManagerFactory): SslContextBuilder {
return SslContextBuilder.forServer(keyManagerFactory)
.sslProvider(SslProvider.OPENSSL)
.trustManager(LoggingTrustManagerFactoryWrapper(trustManagerFactory))
.clientAuth(ClientAuth.REQUIRE)
.ciphers(ArtemisTcpTransport.CIPHER_SUITES)
.protocols(*ArtemisTcpTransport.TLS_VERSIONS.toTypedArray())
}
internal fun splitKeystore(config: AMQPConfiguration): Map<String, CertHoldingKeyManagerFactoryWrapper> {
val keyStore = config.keyStore.value.internal
val password = config.keyStore.entryPassword.toCharArray()
return keyStore.aliases().toList().map { alias ->
val key = keyStore.getKey(alias, password)
val certs = keyStore.getCertificateChain(alias)
val x500Name = keyStore.getCertificate(alias).x509.subjectX500Principal
val cordaX500Name = CordaX500Name.build(x500Name)
val newKeyStore = KeyStore.getInstance("JKS")
newKeyStore.load(null)
newKeyStore.setKeyEntry(alias, key, password, certs)
val newKeyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm())
newKeyManagerFactory.init(newKeyStore, password)
x500toHostName(cordaX500Name) to CertHoldingKeyManagerFactoryWrapper(newKeyManagerFactory, config)
}.toMap()
}
// As per Javadoc in: https://docs.oracle.com/javase/8/docs/api/javax/net/ssl/KeyManagerFactory.html `init` method
// 2nd parameter `password` - the password for recovering keys in the KeyStore
fun KeyManagerFactory.init(keyStore: CertificateStore) = init(keyStore.value.internal, keyStore.entryPassword.toCharArray())
fun TrustManagerFactory.init(trustStore: CertificateStore) = init(trustStore.value.internal)
/**
* Method that converts a [CordaX500Name] to a a valid hostname (RFC-1035). It's used for SNI to indicate the target
* when trying to communicate with nodes that reside behind the same firewall. This is a solution to TLS's extension not
* yet supporting x500 names as server names
*/
internal fun x500toHostName(x500Name: CordaX500Name): String {
val secureHash = SecureHash.sha256(x500Name.toString())
// RFC 1035 specifies a limit 255 bytes for hostnames with each label being 63 bytes or less. Due to this, the string

47
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/ServerSNIMatcher.kt Normal file
View File

@ -0,0 +1,47 @@
package net.corda.nodeapi.internal.protonwrapper.netty
import net.corda.core.identity.CordaX500Name
import net.corda.core.utilities.contextLogger
import net.corda.nodeapi.internal.config.CertificateStore
import net.corda.nodeapi.internal.crypto.x509
import javax.net.ssl.SNIHostName
import javax.net.ssl.SNIMatcher
import javax.net.ssl.SNIServerName
import javax.net.ssl.StandardConstants
class ServerSNIMatcher(private val keyStore: CertificateStore) : SNIMatcher(0) {
companion object {
val log = contextLogger()
}
var matchedAlias: String? = null
private set
var matchedServerName: String? = null
private set
override fun matches(serverName: SNIServerName): Boolean {
if (serverName.type == StandardConstants.SNI_HOST_NAME) {
keyStore.aliases().forEach { alias ->
val x500Name = keyStore[alias].x509.subjectX500Principal
val cordaX500Name = CordaX500Name.build(x500Name)
// Convert the CordaX500Name into the expected host name and compare
// E.g. O=Corda B, L=London, C=GB becomes 3c6dd991936308edb210555103ffc1bb.corda.net
if ((serverName as SNIHostName).asciiName == x500toHostName(cordaX500Name)) {
matchedAlias = alias
matchedServerName = serverName.asciiName
return true
}
}
}
val knownSNIValues = keyStore.aliases().joinToString {
val x500Name = keyStore[it].x509.subjectX500Principal
val cordaX500Name = CordaX500Name.build(x500Name)
"hostname = ${x500toHostName(cordaX500Name)} alias = $it"
}
val requestedSNIValue = "hostname = ${(serverName as SNIHostName).asciiName}"
log.warn("The requested SNI value [$requestedSNIValue] does not match any of the following known SNI values [$knownSNIValues]")
return false
}
}

40
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/TrustManagerFactoryWrapper.kt Normal file
View File

@ -0,0 +1,40 @@
package net.corda.nodeapi.internal.protonwrapper.netty
import java.security.KeyStore
import javax.net.ssl.ManagerFactoryParameters
import javax.net.ssl.TrustManager
import javax.net.ssl.TrustManagerFactory
import javax.net.ssl.TrustManagerFactorySpi
import javax.net.ssl.X509ExtendedTrustManager
class LoggingTrustManagerFactorySpiWrapper(private val factorySpi: TrustManagerFactorySpi) : TrustManagerFactorySpi() {
override fun engineGetTrustManagers(): Array<TrustManager> {
val engineGetTrustManagersMethod = TrustManagerFactorySpi::class.java.getDeclaredMethod("engineGetTrustManagers")
engineGetTrustManagersMethod.isAccessible = true
@Suppress("UNCHECKED_CAST")
val trustManagers = engineGetTrustManagersMethod.invoke(factorySpi) as Array<TrustManager>
return if (factorySpi is LoggingTrustManagerFactorySpiWrapper) trustManagers else trustManagers.filterIsInstance(X509ExtendedTrustManager::class.java).map { LoggingTrustManagerWrapper(it) }.toTypedArray()
}
override fun engineInit(ks: KeyStore?) {
val engineInitMethod = TrustManagerFactorySpi::class.java.getDeclaredMethod("engineInit", KeyStore::class.java)
engineInitMethod.isAccessible = true
engineInitMethod.invoke(factorySpi, ks)
}
override fun engineInit(spec: ManagerFactoryParameters?) {
val engineInitMethod = TrustManagerFactorySpi::class.java.getDeclaredMethod("engineInit", ManagerFactoryParameters::class.java)
engineInitMethod.isAccessible = true
engineInitMethod.invoke(factorySpi, spec)
}
}
class LoggingTrustManagerFactoryWrapper(factory: TrustManagerFactory) : TrustManagerFactory(getFactorySpi(factory), factory.provider, factory.algorithm) {
companion object {
private fun getFactorySpi(factory: TrustManagerFactory): TrustManagerFactorySpi {
val spiField = TrustManagerFactory::class.java.getDeclaredField("factorySpi")
spiField.isAccessible = true
return LoggingTrustManagerFactorySpiWrapper(spiField.get(factory) as TrustManagerFactorySpi)
}
}
}

88
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/revocation/ExternalSourceRevocationChecker.kt Normal file
View File

@ -0,0 +1,88 @@
package net.corda.nodeapi.internal.protonwrapper.netty.revocation
import net.corda.core.utilities.contextLogger
import net.corda.nodeapi.internal.protonwrapper.netty.ExternalCrlSource
import org.bouncycastle.asn1.x509.Extension
import java.security.cert.CRLReason
import java.security.cert.CertPathValidatorException
import java.security.cert.Certificate
import java.security.cert.CertificateRevokedException
import java.security.cert.PKIXRevocationChecker
import java.security.cert.X509CRL
import java.security.cert.X509Certificate
import java.util.*
/**
* Implementation of [PKIXRevocationChecker] which determines whether certificate is revoked using [externalCrlSource] which knows how to
* obtain a set of CRLs for a given certificate from an external source
*/
class ExternalSourceRevocationChecker(private val externalCrlSource: ExternalCrlSource, private val dateSource: () -> Date) : PKIXRevocationChecker() {
companion object {
private val logger = contextLogger()
}
override fun check(cert: Certificate, unresolvedCritExts: MutableCollection<String>?) {
val x509Certificate = cert as X509Certificate
checkApprovedCRLs(x509Certificate, externalCrlSource.fetch(x509Certificate))
}
/**
* Borrowed from `RevocationChecker.checkApprovedCRLs()`
*/
@Suppress("NestedBlockDepth")
@Throws(CertPathValidatorException::class)
private fun checkApprovedCRLs(cert: X509Certificate, approvedCRLs: Set<X509CRL>) {
// See if the cert is in the set of approved crls.
logger.debug("ExternalSourceRevocationChecker.checkApprovedCRLs() cert SN: ${cert.serialNumber}")
for (crl in approvedCRLs) {
val entry = crl.getRevokedCertificate(cert)
if (entry != null) {
logger.debug("ExternalSourceRevocationChecker.checkApprovedCRLs() CRL entry: $entry")
/*
* Abort CRL validation and throw exception if there are any
* unrecognized critical CRL entry extensions (see section
* 5.3 of RFC 5280).
*/
val unresCritExts = entry.criticalExtensionOIDs
if (unresCritExts != null && !unresCritExts.isEmpty()) {
/* remove any that we will process */
unresCritExts.remove(Extension.cRLDistributionPoints.id)
unresCritExts.remove(Extension.certificateIssuer.id)
if (!unresCritExts.isEmpty()) {
throw CertPathValidatorException(
"Unrecognized critical extension(s) in revoked CRL entry: $unresCritExts")
}
}
val reasonCode = entry.revocationReason ?: CRLReason.UNSPECIFIED
val revocationDate = entry.revocationDate
if (revocationDate.before(dateSource())) {
val t = CertificateRevokedException(
revocationDate, reasonCode,
crl.issuerX500Principal, mutableMapOf())
throw CertPathValidatorException(
t.message, t, null, -1, CertPathValidatorException.BasicReason.REVOKED)
}
}
}
}
override fun isForwardCheckingSupported(): Boolean {
return true
}
override fun getSupportedExtensions(): MutableSet<String>? {
return null
}
override fun init(forward: Boolean) {
// Nothing to do
}
override fun getSoftFailExceptions(): MutableList<CertPathValidatorException> {
return LinkedList()
}
}

1
node-api/src/test/kotlin/net/corda/nodeapi/internal/config/ConfigParsingTest.kt
View File

@ -9,7 +9,6 @@ import net.corda.core.identity.CordaX500Name
import net.corda.core.internal.div
import net.corda.core.utilities.NetworkHostAndPort
import org.assertj.core.api.Assertions.*
import org.hibernate.exception.DataException
import org.junit.Test
import java.net.URL
import java.nio.file.Path

15
node-api/src/test/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelperTest.kt
View File

@ -4,7 +4,10 @@ import net.corda.core.crypto.SecureHash
import net.corda.core.identity.CordaX500Name
import net.corda.core.utilities.NetworkHostAndPort
import net.corda.coretesting.internal.configureTestSSL
import net.corda.nodeapi.internal.DEV_CA_KEY_STORE_PASS
import net.corda.nodeapi.internal.DEV_CA_PRIVATE_KEY_PASS
import net.corda.nodeapi.internal.config.CertificateStore
import net.corda.nodeapi.internal.crypto.X509Utilities.CORDA_CLIENT_TLS
import org.junit.Test
import javax.net.ssl.KeyManagerFactory
import javax.net.ssl.SNIHostName
@ -23,7 +26,7 @@ class SSLHelperTest {
val keyStore = sslConfig.keyStore
keyManagerFactory.init(CertificateStore.fromFile(keyStore.path, keyStore.storePassword, keyStore.entryPassword, false))
val trustStore = sslConfig.trustStore
trustManagerFactory.init(initialiseTrustStoreAndEnableCrlChecking(CertificateStore.fromFile(trustStore.path, trustStore.storePassword, trustStore.entryPassword, false), false))
trustManagerFactory.init(initialiseTrustStoreAndEnableCrlChecking(CertificateStore.fromFile(trustStore.path, trustStore.storePassword, trustStore.entryPassword, false), RevocationConfigImpl(RevocationConfig.Mode.HARD_FAIL)))
val sslHandler = createClientSslHelper(NetworkHostAndPort("localhost", 1234), setOf(legalName), keyManagerFactory, trustManagerFactory)
val legalNameHash = SecureHash.sha256(legalName.toString()).toString().take(32).toLowerCase()
@ -34,4 +37,14 @@ class SSLHelperTest {
assertEquals(1, sslHandler.engine().sslParameters.serverNames.size)
assertEquals("$legalNameHash.corda.net", (sslHandler.engine().sslParameters.serverNames.first() as SNIHostName).asciiName)
}
@Test(timeout=300_000)
fun `test distributionPointsToString`() {
val certStore = CertificateStore.fromResource(
"net/corda/nodeapi/internal/protonwrapper/netty/sslkeystore_Revoked.jks",
DEV_CA_KEY_STORE_PASS, DEV_CA_PRIVATE_KEY_PASS)
val distPoints = certStore.query { getCertificateChain(CORDA_CLIENT_TLS).map { it.distributionPointsToString() } }
assertEquals(listOf("NO CRLDP ext", "http://day-v3-doorman.cordaconnect.io/doorman",
"http://day3-doorman.cordaconnect.io/doorman", "http://day3-doorman.cordaconnect.io/subordinate", "NO CRLDP ext"), distPoints)
}
}

56
node-api/src/test/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/revocation/ExternalSourceRevocationCheckerTest.kt Normal file
View File

@ -0,0 +1,56 @@
package net.corda.nodeapi.internal.protonwrapper.netty.revocation
import net.corda.core.utilities.Try
import net.corda.nodeapi.internal.DEV_CA_KEY_STORE_PASS
import net.corda.nodeapi.internal.DEV_CA_PRIVATE_KEY_PASS
import net.corda.nodeapi.internal.config.CertificateStore
import net.corda.nodeapi.internal.crypto.X509Utilities
import net.corda.nodeapi.internal.protonwrapper.netty.ExternalCrlSource
import org.bouncycastle.jcajce.provider.asymmetric.x509.CertificateFactory
import org.junit.Test
import java.math.BigInteger
import java.security.cert.X509CRL
import java.security.cert.X509Certificate
import java.sql.Date
import kotlin.test.assertEquals
import kotlin.test.assertTrue
class ExternalSourceRevocationCheckerTest {
@Test(timeout=300_000)
fun checkRevoked() {
val checkResult = performCheckOnDate(Date.valueOf("2019-09-27"))
val failedChecks = checkResult.filterNot { it.second.isSuccess }
assertEquals(1, failedChecks.size)
assertEquals(BigInteger.valueOf(8310484079152632582), failedChecks.first().first.serialNumber)
}
@Test(timeout=300_000)
fun checkTooEarly() {
val checkResult = performCheckOnDate(Date.valueOf("2019-08-27"))
assertTrue(checkResult.all { it.second.isSuccess })
}
private fun performCheckOnDate(date: Date): List<Pair<X509Certificate, Try<Unit>>> {
val certStore = CertificateStore.fromResource(
"net/corda/nodeapi/internal/protonwrapper/netty/sslkeystore_Revoked.jks",
DEV_CA_KEY_STORE_PASS, DEV_CA_PRIVATE_KEY_PASS)
val resourceAsStream = javaClass.getResourceAsStream("/net/corda/nodeapi/internal/protonwrapper/netty/doorman.crl")
val crl = CertificateFactory().engineGenerateCRL(resourceAsStream) as X509CRL
//val crlHolder = X509CRLHolder(resourceAsStream)
//crlHolder.revokedCertificates as X509CRLEntryHolder
val instance = ExternalSourceRevocationChecker(object : ExternalCrlSource {
override fun fetch(certificate: X509Certificate): Set<X509CRL> = setOf(crl)
}) { date }
return certStore.query {
getCertificateChain(X509Utilities.CORDA_CLIENT_TLS).map {
Pair(it, Try.on { instance.check(it, mutableListOf()) })
}
}
}
}

3
node-api/src/test/resources/net/corda/nodeapi/internal/protonwrapper/netty/Readme.txt Normal file
View File

@ -0,0 +1,3 @@
Represents some test data which contains real certificates produced by DayWatch Doorman as well as CRL list file.
For all the keystores the password is "cordacadevpass".

BIN
node-api/src/test/resources/net/corda/nodeapi/internal/protonwrapper/netty/doorman.crl Normal file
View File

Binary file not shown.

BIN
node-api/src/test/resources/net/corda/nodeapi/internal/protonwrapper/netty/sslkeystore_Revoked.jks Normal file
View File

Binary file not shown.

BIN
node-api/src/test/resources/net/corda/nodeapi/internal/serialization/amqp/networkParamsWrite Normal file
View File

Binary file not shown.

16
node/src/integration-test/kotlin/net/corda/node/amqp/AMQPBridgeTest.kt
View File

@ -22,6 +22,7 @@ import net.corda.testing.core.TestIdentity
import net.corda.testing.driver.internal.incrementalPortAllocation
import net.corda.coretesting.internal.rigorousMock
import net.corda.coretesting.internal.stubs.CertificateStoreStubs
import net.corda.nodeapi.internal.protonwrapper.netty.toRevocationConfig
import org.apache.activemq.artemis.api.core.Message.HDR_DUPLICATE_DETECTION_ID
import org.apache.activemq.artemis.api.core.RoutingType
import org.apache.activemq.artemis.api.core.SimpleString
@ -206,13 +207,22 @@ class AMQPBridgeTest {
val artemisClient = ArtemisMessagingClient(artemisConfig.p2pSslOptions, artemisAddress, MAX_MESSAGE_SIZE)
artemisServer.start()
artemisClient.start()
val bridgeManager = AMQPBridgeManager(artemisConfig.p2pSslOptions, artemisAddress, MAX_MESSAGE_SIZE, artemisConfig.crlCheckSoftFail)
val bridgeManager = AMQPBridgeManager(
artemisConfig.p2pSslOptions.keyStore.get(),
artemisConfig.p2pSslOptions.trustStore.get(),
false,
null,
MAX_MESSAGE_SIZE,
artemisConfig.crlCheckSoftFail.toRevocationConfig(),
false, { ArtemisMessagingClient(artemisConfig.p2pSslOptions, artemisAddress, MAX_MESSAGE_SIZE) }, trace = false,
sslHandshakeTimeout = null,
bridgeConnectionTTLSeconds = 0)
bridgeManager.start()
val artemis = artemisClient.started!!
if (sourceQueueName != null) {
// Local queue for outgoing messages
artemis.session.createQueue(sourceQueueName, RoutingType.ANYCAST, sourceQueueName, true)
bridgeManager.deployBridge(sourceQueueName, listOf(amqpAddress), setOf(BOB.name))
bridgeManager.deployBridge(ALICE_NAME.toString(), sourceQueueName, listOf(amqpAddress), setOf(BOB.name))
}
return Triple(artemisServer, artemisClient, bridgeManager)
}
@ -228,7 +238,6 @@ class AMQPBridgeTest {
doReturn(certificatesDirectory).whenever(it).certificatesDirectory
doReturn(signingCertificateStore).whenever(it).signingCertificateStore
doReturn(p2pSslConfiguration).whenever(it).p2pSslOptions
doReturn(true).whenever(it).crlCheckSoftFail
}
serverConfig.configureWithDevSSLCertificate()
@ -238,7 +247,6 @@ class AMQPBridgeTest {
override val trustStore = serverConfig.p2pSslOptions.trustStore.get()
override val trace: Boolean = true
override val maxMessageSize: Int = maxMessageSize
override val crlCheckSoftFail: Boolean = serverConfig.crlCheckSoftFail
}
return AMQPServer("0.0.0.0",
amqpAddress.port,

8
node/src/integration-test/kotlin/net/corda/node/amqp/CertificateRevocationListNodeTests.kt
View File

@ -29,6 +29,8 @@ import net.corda.coretesting.internal.DEV_INTERMEDIATE_CA
import net.corda.coretesting.internal.DEV_ROOT_CA
import net.corda.coretesting.internal.rigorousMock
import net.corda.coretesting.internal.stubs.CertificateStoreStubs
import net.corda.nodeapi.internal.protonwrapper.netty.RevocationConfig
import net.corda.nodeapi.internal.protonwrapper.netty.toRevocationConfig
import org.assertj.core.api.Assertions.assertThatIllegalArgumentException
import org.bouncycastle.asn1.x500.X500Name
import org.bouncycastle.asn1.x509.*
@ -305,7 +307,7 @@ class CertificateRevocationListNodeTests {
}
@Test(timeout=300_000)
fun `Revocation status chceck fails when the CRL distribution point is not set and soft fail is disabled`() {
fun `Revocation status check fails when the CRL distribution point is not set and soft fail is disabled`() {
val crlCheckSoftFail = false
val (amqpServer, _) = createServer(
serverPort,
@ -380,7 +382,6 @@ class CertificateRevocationListNodeTests {
val amqpConfig = object : AMQPConfiguration {
override val keyStore = keyStore
override val trustStore = clientConfig.p2pSslOptions.trustStore.get()
override val crlCheckSoftFail: Boolean = crlCheckSoftFail
override val maxMessageSize: Int = maxMessageSize
}
return Pair(AMQPClient(
@ -404,7 +405,6 @@ class CertificateRevocationListNodeTests {
doReturn(name).whenever(it).myLegalName
doReturn(p2pSslConfiguration).whenever(it).p2pSslOptions
doReturn(signingCertificateStore).whenever(it).signingCertificateStore
doReturn(crlCheckSoftFail).whenever(it).crlCheckSoftFail
}
serverConfig.configureWithDevSSLCertificate()
val nodeCert = (signingCertificateStore to p2pSslConfiguration).recreateNodeCaAndTlsCertificates(nodeCrlDistPoint, tlsCrlDistPoint)
@ -412,7 +412,7 @@ class CertificateRevocationListNodeTests {
val amqpConfig = object : AMQPConfiguration {
override val keyStore = keyStore
override val trustStore = serverConfig.p2pSslOptions.trustStore.get()
override val crlCheckSoftFail: Boolean = crlCheckSoftFail
override val revocationConfig = crlCheckSoftFail.toRevocationConfig()
override val maxMessageSize: Int = maxMessageSize
}
return Pair(AMQPServer(

11
node/src/integration-test/kotlin/net/corda/node/amqp/ProtonWrapperTests.kt
View File

@ -32,6 +32,7 @@ import net.corda.testing.driver.internal.incrementalPortAllocation
import net.corda.testing.internal.createDevIntermediateCaCertPath
import net.corda.coretesting.internal.rigorousMock
import net.corda.coretesting.internal.stubs.CertificateStoreStubs
import net.corda.nodeapi.internal.protonwrapper.netty.toRevocationConfig
import org.apache.activemq.artemis.api.core.RoutingType
import org.assertj.core.api.Assertions.assertThatThrownBy
import org.junit.Assert.assertArrayEquals
@ -39,6 +40,7 @@ import org.junit.Rule
import org.junit.Test
import org.junit.rules.TemporaryFolder
import java.security.cert.X509Certificate
import java.util.concurrent.TimeUnit
import javax.net.ssl.*
import kotlin.concurrent.thread
import kotlin.test.assertEquals
@ -341,6 +343,7 @@ class ProtonWrapperTests {
val connection1ID = CordaX500Name.build(connection1.remoteCert!!.subjectX500Principal)
assertEquals("client 0", connection1ID.organisationUnit)
val source1 = connection1.remoteAddress
val client2Connected = amqpClient2.onConnection.toFuture()
amqpClient2.start()
val connection2 = connectionEvents.next()
assertEquals(true, connection2.connected)
@ -353,6 +356,7 @@ class ProtonWrapperTests {
assertEquals(false, connection3.connected)
assertEquals(source1, connection3.remoteAddress)
assertEquals(false, amqpClient1.connected)
client2Connected.get(60, TimeUnit.SECONDS)
assertEquals(true, amqpClient2.connected)
// Now shutdown both
amqpClient2.stop()
@ -362,11 +366,13 @@ class ProtonWrapperTests {
assertEquals(false, amqpClient1.connected)
assertEquals(false, amqpClient2.connected)
// Now restarting one should work
val client1Connected = amqpClient1.onConnection.toFuture()
amqpClient1.start()
val connection5 = connectionEvents.next()
assertEquals(true, connection5.connected)
val connection5ID = CordaX500Name.build(connection5.remoteCert!!.subjectX500Principal)
assertEquals("client 0", connection5ID.organisationUnit)
client1Connected.get(60, TimeUnit.SECONDS)
assertEquals(true, amqpClient1.connected)
assertEquals(false, amqpClient2.connected)
// Cleanup
@ -447,7 +453,6 @@ class ProtonWrapperTests {
override val trustStore = clientTruststore
override val trace: Boolean = true
override val maxMessageSize: Int = maxMessageSize
override val crlCheckSoftFail: Boolean = clientConfig.crlCheckSoftFail
}
return AMQPClient(
listOf(NetworkHostAndPort("localhost", serverPort),
@ -479,7 +484,6 @@ class ProtonWrapperTests {
override val trustStore = clientTruststore
override val trace: Boolean = true
override val maxMessageSize: Int = maxMessageSize
override val crlCheckSoftFail: Boolean = clientConfig.crlCheckSoftFail
}
return AMQPClient(
listOf(NetworkHostAndPort("localhost", serverPort)),
@ -502,7 +506,6 @@ class ProtonWrapperTests {
doReturn(name).whenever(it).myLegalName
doReturn(signingCertificateStore).whenever(it).signingCertificateStore
doReturn(p2pSslConfiguration).whenever(it).p2pSslOptions
doReturn(crlCheckSoftFail).whenever(it).crlCheckSoftFail
}
serverConfig.configureWithDevSSLCertificate()
@ -512,8 +515,8 @@ class ProtonWrapperTests {
override val keyStore = serverKeystore
override val trustStore = serverTruststore
override val trace: Boolean = true
override val revocationConfig = crlCheckSoftFail.toRevocationConfig()
override val maxMessageSize: Int = maxMessageSize
override val crlCheckSoftFail: Boolean = serverConfig.crlCheckSoftFail
}
return AMQPServer(
"0.0.0.0",

11
node/src/main/kotlin/net/corda/node/internal/Node.kt
View File

@ -79,6 +79,7 @@ import net.corda.nodeapi.internal.bridging.BridgeControlListener
import net.corda.nodeapi.internal.config.User
import net.corda.nodeapi.internal.crypto.X509Utilities
import net.corda.nodeapi.internal.persistence.CouldNotCreateDataSourceException
import net.corda.nodeapi.internal.protonwrapper.netty.toRevocationConfig
import net.corda.serialization.internal.AMQP_P2P_CONTEXT
import net.corda.serialization.internal.AMQP_RPC_CLIENT_CONTEXT
import net.corda.serialization.internal.AMQP_RPC_SERVER_CONTEXT
@ -417,7 +418,15 @@ open class Node(configuration: NodeConfiguration,
failoverCallback = { errorAndTerminate("ArtemisMessagingClient failed. Shutting down.", null) }
)
}
return BridgeControlListener(configuration.p2pSslOptions, networkParameters.maxMessageSize, configuration.crlCheckSoftFail, artemisMessagingClientFactory)
return BridgeControlListener(
configuration.p2pSslOptions.keyStore.get(),
configuration.p2pSslOptions.trustStore.get(),
false,
null,
networkParameters.maxMessageSize,
configuration.crlCheckSoftFail.toRevocationConfig(),
false,
artemisMessagingClientFactory)
}
private fun startLocalRpcBroker(securityManager: RPCSecurityManager): BrokerAddresses? {

13
node/src/main/kotlin/net/corda/node/services/messaging/P2PMessagingClient.kt
View File

@ -197,7 +197,7 @@ class P2PMessagingClient(val config: NodeConfiguration,
inboxes += RemoteInboxAddress(it).queueName
}
inboxes.forEach { createQueueIfAbsent(it, producerSession!!, exclusive = true) }
inboxes.forEach { createQueueIfAbsent(it, producerSession!!, exclusive = true, isServiceAddress = false) }
p2pConsumer = P2PMessagingConsumer(inboxes, createNewSession, isDrainingModeOn, drainingModeWasChangedEvents, metricRegistry)
@ -267,7 +267,7 @@ class P2PMessagingClient(val config: NodeConfiguration,
return state.locked {
node.legalIdentitiesAndCerts.map {
val messagingAddress = NodeAddress(it.party.owningKey)
BridgeEntry(messagingAddress.queueName, node.addresses, node.legalIdentities.map { it.name })
BridgeEntry(messagingAddress.queueName, node.addresses, node.legalIdentities.map { it.name }, serviceAddress = false)
}.filter { producerSession!!.queueQuery(SimpleString(it.queueName)).isExists }.asSequence()
}
}
@ -306,7 +306,7 @@ class P2PMessagingClient(val config: NodeConfiguration,
val keyHash = queueName.substring(PEERS_PREFIX.length)
val peers = networkMap.getNodesByOwningKeyIndex(keyHash)
for (node in peers) {
val bridge = BridgeEntry(queueName.toString(), node.addresses, node.legalIdentities.map { it.name })
val bridge = BridgeEntry(queueName.toString(), node.addresses, node.legalIdentities.map { it.name }, serviceAddress = false)
requiredBridges += bridge
knownQueues += queueName.toString()
}
@ -527,19 +527,20 @@ class P2PMessagingClient(val config: NodeConfiguration,
val internalTargetQueue = (address as? ArtemisAddress)?.queueName
?: throw IllegalArgumentException("Not an Artemis address")
state.locked {
createQueueIfAbsent(internalTargetQueue, producerSession!!, exclusive = address !is ServiceAddress)
val serviceAddress = address is ServiceAddress
createQueueIfAbsent(internalTargetQueue, producerSession!!, exclusive = !serviceAddress, isServiceAddress = serviceAddress)
}
internalTargetQueue
}
}
/** Attempts to create a durable queue on the broker which is bound to an address of the same name. */
private fun createQueueIfAbsent(queueName: String, session: ClientSession, exclusive: Boolean) {
private fun createQueueIfAbsent(queueName: String, session: ClientSession, exclusive: Boolean, isServiceAddress: Boolean) {
fun sendBridgeCreateMessage() {
val keyHash = queueName.substring(PEERS_PREFIX.length)
val peers = networkMap.getNodesByOwningKeyIndex(keyHash)
for (node in peers) {
val bridge = BridgeEntry(queueName, node.addresses, node.legalIdentities.map { it.name })
val bridge = BridgeEntry(queueName, node.addresses, node.legalIdentities.map { it.name }, isServiceAddress)
val createBridgeMessage = BridgeControl.Create(config.myLegalName.toString(), bridge)
sendBridgeControl(createBridgeMessage)
}