diff --git a/.ci/dev/pr-code-checks/Jenkinsfile b/.ci/dev/pr-code-checks/Jenkinsfile index c8b0a0c41b..261dc6f374 100644 --- a/.ci/dev/pr-code-checks/Jenkinsfile +++ b/.ci/dev/pr-code-checks/Jenkinsfile @@ -47,6 +47,7 @@ pipeline { GRADLE_USER_HOME = "/host_tmp/gradle" } steps { + authenticateGradleWrapper() sh 'mkdir -p ${GRADLE_USER_HOME}' authenticateGradleWrapper() snykDeltaScan(env.SNYK_API_TOKEN, env.C4_OS_SNYK_ORG_ID) diff --git a/client/rpc/src/main/kotlin/net/corda/client/rpc/internal/RPCClient.kt b/client/rpc/src/main/kotlin/net/corda/client/rpc/internal/RPCClient.kt index fe34aa057c..3d8a8cc2bd 100644 --- a/client/rpc/src/main/kotlin/net/corda/client/rpc/internal/RPCClient.kt +++ b/client/rpc/src/main/kotlin/net/corda/client/rpc/internal/RPCClient.kt @@ -17,7 +17,6 @@ import net.corda.core.serialization.internal.nodeSerializationEnv import net.corda.core.utilities.NetworkHostAndPort import net.corda.core.utilities.contextLogger import net.corda.nodeapi.internal.ArtemisTcpTransport.Companion.rpcConnectorTcpTransport -import net.corda.nodeapi.internal.ArtemisTcpTransport.Companion.rpcConnectorTcpTransportsFromList import net.corda.nodeapi.internal.ArtemisTcpTransport.Companion.rpcInternalClientTcpTransport import net.corda.nodeapi.internal.RoundRobinConnectionPolicy import net.corda.nodeapi.internal.config.SslConfiguration @@ -62,8 +61,12 @@ class RPCClient( sslConfiguration: ClientRpcSslOptions? = null, configuration: CordaRPCClientConfiguration = CordaRPCClientConfiguration.DEFAULT, serializationContext: SerializationContext = SerializationDefaults.RPC_CLIENT_CONTEXT - ) : this(rpcConnectorTcpTransport(haAddressPool.first(), sslConfiguration), - configuration, serializationContext, rpcConnectorTcpTransportsFromList(haAddressPool, sslConfiguration)) + ) : this( + rpcConnectorTcpTransport(haAddressPool.first(), sslConfiguration), + configuration, + serializationContext, + haAddressPool.map { rpcConnectorTcpTransport(it, sslConfiguration) } + ) companion object { private val log = contextLogger() diff --git a/node-api/build.gradle b/node-api/build.gradle index 7299cfa23e..c4cdfdd906 100644 --- a/node-api/build.gradle +++ b/node-api/build.gradle @@ -31,7 +31,7 @@ dependencies { // SQL connection pooling library compile "com.zaxxer:HikariCP:$hikari_version" - + // ClassGraph: classpath scanning compile "io.github.classgraph:classgraph:$class_graph_version" @@ -56,6 +56,9 @@ dependencies { testRuntimeOnly "org.junit.vintage:junit-vintage-engine:${junit_vintage_version}" testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${junit_jupiter_version}" testRuntimeOnly "org.junit.platform:junit-platform-launcher:${junit_platform_version}" + + testCompile project(':node-driver') + // Unit testing helpers. testCompile "org.assertj:assertj-core:$assertj_version" testCompile "org.jetbrains.kotlin:kotlin-test:$kotlin_version" diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisMessagingClient.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisMessagingClient.kt index 4a381c21ab..750b5857f5 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisMessagingClient.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisMessagingClient.kt @@ -5,7 +5,6 @@ import net.corda.core.utilities.NetworkHostAndPort import net.corda.core.utilities.loggerFor import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.NODE_P2P_USER import net.corda.nodeapi.internal.ArtemisTcpTransport.Companion.p2pConnectorTcpTransport -import net.corda.nodeapi.internal.ArtemisTcpTransport.Companion.p2pConnectorTcpTransportFromList import net.corda.nodeapi.internal.config.MessagingServerConnectionConfiguration import net.corda.nodeapi.internal.config.MutualSslConfiguration import org.apache.activemq.artemis.api.core.client.* @@ -41,7 +40,7 @@ class ArtemisMessagingClient(private val config: MutualSslConfiguration, override fun start(): Started = synchronized(this) { check(started == null) { "start can't be called twice" } val tcpTransport = p2pConnectorTcpTransport(serverAddress, config) - val backupTransports = p2pConnectorTcpTransportFromList(backupServerAddressPool, config) + val backupTransports = backupServerAddressPool.map { p2pConnectorTcpTransport(it, config) } log.info("Connecting to message broker: $serverAddress") if (backupTransports.isNotEmpty()) { diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisTcpTransport.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisTcpTransport.kt index cdf5ea8cc9..bf2ff89bf9 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisTcpTransport.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ArtemisTcpTransport.kt @@ -5,16 +5,14 @@ import net.corda.core.serialization.internal.nodeSerializationEnv import net.corda.core.utilities.NetworkHostAndPort import net.corda.nodeapi.BrokerRpcSslOptions import net.corda.nodeapi.internal.config.CertificateStore -import net.corda.nodeapi.internal.config.FileBasedCertificateStoreSupplier +import net.corda.nodeapi.internal.config.DEFAULT_SSL_HANDSHAKE_TIMEOUT import net.corda.nodeapi.internal.config.MutualSslConfiguration import net.corda.nodeapi.internal.config.SslConfiguration import org.apache.activemq.artemis.api.core.TransportConfiguration -import org.apache.activemq.artemis.core.remoting.impl.netty.NettyAcceptorFactory import org.apache.activemq.artemis.core.remoting.impl.netty.NettyConnectorFactory import org.apache.activemq.artemis.core.remoting.impl.netty.TransportConstants import java.nio.file.Path -// This avoids internal types from leaking in the public API. The "external" ArtemisTcpTransport delegates to this internal one. class ArtemisTcpTransport { companion object { val CIPHER_SUITES = listOf( @@ -24,6 +22,9 @@ class ArtemisTcpTransport { val TLS_VERSIONS = listOf("TLSv1.2") + const val SSL_HANDSHAKE_TIMEOUT_NAME = "SSLHandshakeTimeout" + const val TRACE_NAME = "trace" + // Turn on AMQP support, which needs the protocol jar on the classpath. // Unfortunately we cannot disable core protocol as artemis only uses AMQP for interop. // It does not use AMQP messages for its own messages e.g. topology and heartbeats. @@ -46,17 +47,7 @@ class ArtemisTcpTransport { TransportConstants.ENABLED_CIPHER_SUITES_PROP_NAME to CIPHER_SUITES.joinToString(","), TransportConstants.ENABLED_PROTOCOLS_PROP_NAME to TLS_VERSIONS.joinToString(",")) - private fun SslConfiguration.toTransportOptions(): Map { - - val options = mutableMapOf() - (keyStore to trustStore).addToTransportOptions(options) - return options - } - - private fun Pair.addToTransportOptions(options: MutableMap) { - - val keyStore = first - val trustStore = second + private fun SslConfiguration.addToTransportOptions(options: MutableMap) { keyStore?.let { with (it) { path.requireOnDefaultFileSystem() @@ -69,6 +60,8 @@ class ArtemisTcpTransport { options.putAll(get().toTrustStoreTransportOptions(path)) } } + options[TransportConstants.SSL_PROVIDER] = if (useOpenSsl) TransportConstants.OPENSSL_PROVIDER else TransportConstants.DEFAULT_SSL_PROVIDER + options[SSL_HANDSHAKE_TIMEOUT_NAME] = handshakeTimeout ?: DEFAULT_SSL_HANDSHAKE_TIMEOUT } private fun CertificateStore.toKeyStoreTransportOptions(path: Path) = mapOf( @@ -98,96 +91,97 @@ class ArtemisTcpTransport { TransportConstants.KEYSTORE_PASSWORD_PROP_NAME to keyStorePassword, TransportConstants.NEED_CLIENT_AUTH_PROP_NAME to false) - private val acceptorFactoryClassName = NettyAcceptorFactory::class.java.name - private val connectorFactoryClassName = NettyConnectorFactory::class.java.name - - fun p2pAcceptorTcpTransport(hostAndPort: NetworkHostAndPort, config: MutualSslConfiguration?, enableSSL: Boolean = true): TransportConfiguration { - - return p2pAcceptorTcpTransport(hostAndPort, config?.keyStore, config?.trustStore, enableSSL = enableSSL, useOpenSsl = config?.useOpenSsl ?: false) - } - - fun p2pConnectorTcpTransport(hostAndPort: NetworkHostAndPort, config: MutualSslConfiguration?, enableSSL: Boolean = true, keyStoreType: String? = null): TransportConfiguration { - - return p2pConnectorTcpTransport(hostAndPort, config?.keyStore, config?.trustStore, enableSSL = enableSSL, useOpenSsl = config?.useOpenSsl ?: false, keyStoreType = keyStoreType) - } - - fun p2pAcceptorTcpTransport(hostAndPort: NetworkHostAndPort, keyStore: FileBasedCertificateStoreSupplier?, trustStore: FileBasedCertificateStoreSupplier?, enableSSL: Boolean = true, useOpenSsl: Boolean = false): TransportConfiguration { - - val options = defaultArtemisOptions(hostAndPort, P2P_PROTOCOLS).toMutableMap() + fun p2pAcceptorTcpTransport(hostAndPort: NetworkHostAndPort, + config: MutualSslConfiguration?, + enableSSL: Boolean = true, + trace: Boolean = false): TransportConfiguration { + val options = mutableMapOf() if (enableSSL) { - options.putAll(defaultSSLOptions) - (keyStore to trustStore).addToTransportOptions(options) - options[TransportConstants.SSL_PROVIDER] = if (useOpenSsl) TransportConstants.OPENSSL_PROVIDER else TransportConstants.DEFAULT_SSL_PROVIDER + config?.addToTransportOptions(options) } - options[TransportConstants.HANDSHAKE_TIMEOUT] = 0 // Suppress core.server.lambda$channelActive$0 - AMQ224088 error from load balancer type connections - return TransportConfiguration(acceptorFactoryClassName, options) + return createAcceptorTransport(hostAndPort, P2P_PROTOCOLS, options, enableSSL, trace) } - @Suppress("LongParameterList") - fun p2pConnectorTcpTransport(hostAndPort: NetworkHostAndPort, keyStore: FileBasedCertificateStoreSupplier?, trustStore: FileBasedCertificateStoreSupplier?, enableSSL: Boolean = true, useOpenSsl: Boolean = false, keyStoreType: String? = null): TransportConfiguration { - - val options = defaultArtemisOptions(hostAndPort, P2P_PROTOCOLS).toMutableMap() + fun p2pConnectorTcpTransport(hostAndPort: NetworkHostAndPort, + config: MutualSslConfiguration?, + enableSSL: Boolean = true, + keyStoreType: String? = null): TransportConfiguration { + val options = mutableMapOf() if (enableSSL) { - options.putAll(defaultSSLOptions) - (keyStore to trustStore).addToTransportOptions(options) - options[TransportConstants.SSL_PROVIDER] = if (useOpenSsl) TransportConstants.OPENSSL_PROVIDER else TransportConstants.DEFAULT_SSL_PROVIDER - keyStoreType?.let { options.put(TransportConstants.KEYSTORE_TYPE_PROP_NAME, keyStoreType) } - // This is required to stop Client checking URL address vs. Server provided certificate - options[TransportConstants.VERIFY_HOST_PROP_NAME] = false + config?.addToTransportOptions(options) + options += asMap(keyStoreType) } - return TransportConfiguration(connectorFactoryClassName, options) + return createConnectorTransport(hostAndPort, P2P_PROTOCOLS, options, enableSSL) } - fun p2pConnectorTcpTransportFromList(hostAndPortList: List, config: MutualSslConfiguration?, enableSSL: Boolean = true, keyStoreType: String? = null): List = hostAndPortList.map { - p2pConnectorTcpTransport(it, config, enableSSL, keyStoreType) - } - - fun rpcAcceptorTcpTransport(hostAndPort: NetworkHostAndPort, config: BrokerRpcSslOptions?, enableSSL: Boolean = true): TransportConfiguration { - val options = defaultArtemisOptions(hostAndPort, RPC_PROTOCOLS).toMutableMap() - + fun rpcAcceptorTcpTransport(hostAndPort: NetworkHostAndPort, + config: BrokerRpcSslOptions?, + enableSSL: Boolean = true, + trace: Boolean = false): TransportConfiguration { + val options = mutableMapOf() if (config != null && enableSSL) { config.keyStorePath.requireOnDefaultFileSystem() options.putAll(config.toTransportOptions()) - options.putAll(defaultSSLOptions) } - options[TransportConstants.HANDSHAKE_TIMEOUT] = 0 // Suppress core.server.lambda$channelActive$0 - AMQ224088 error from load balancer type connections - return TransportConfiguration(acceptorFactoryClassName, options) + return createAcceptorTransport(hostAndPort, RPC_PROTOCOLS, options, enableSSL, trace) } fun rpcConnectorTcpTransport(hostAndPort: NetworkHostAndPort, config: ClientRpcSslOptions?, enableSSL: Boolean = true): TransportConfiguration { - val options = defaultArtemisOptions(hostAndPort, RPC_PROTOCOLS).toMutableMap() - + val options = mutableMapOf() if (config != null && enableSSL) { config.trustStorePath.requireOnDefaultFileSystem() options.putAll(config.toTransportOptions()) - options.putAll(defaultSSLOptions) - // This is required to stop Client checking URL address vs. Server provided certificate - options[TransportConstants.VERIFY_HOST_PROP_NAME] = false } - return TransportConfiguration(connectorFactoryClassName, options) + return createConnectorTransport(hostAndPort, RPC_PROTOCOLS, options, enableSSL) } - fun rpcConnectorTcpTransportsFromList(hostAndPortList: List, config: ClientRpcSslOptions?, enableSSL: Boolean = true): List = hostAndPortList.map { - rpcConnectorTcpTransport(it, config, enableSSL) + fun rpcInternalClientTcpTransport(hostAndPort: NetworkHostAndPort, config: SslConfiguration, keyStoreProvider: String? = null): TransportConfiguration { + val options = mutableMapOf() + config.addToTransportOptions(options) + options += asMap(keyStoreProvider) + return createConnectorTransport(hostAndPort, RPC_PROTOCOLS, options, enableSSL = true) } - fun rpcInternalClientTcpTransport(hostAndPort: NetworkHostAndPort, config: SslConfiguration, keyStoreType: String? = null): TransportConfiguration { - val options = defaultArtemisOptions(hostAndPort, RPC_PROTOCOLS).toMutableMap() - options.putAll(defaultSSLOptions) - options.putAll(config.toTransportOptions()) - options.putAll(asMap(keyStoreType)) - // This is required to stop Client checking URL address vs. Server provided certificate - options[TransportConstants.VERIFY_HOST_PROP_NAME] = false - return TransportConfiguration(connectorFactoryClassName, options) - } - - fun rpcInternalAcceptorTcpTransport(hostAndPort: NetworkHostAndPort, config: SslConfiguration, keyStoreType: String? = null): TransportConfiguration { - return TransportConfiguration(acceptorFactoryClassName, defaultArtemisOptions(hostAndPort, RPC_PROTOCOLS) + defaultSSLOptions + - config.toTransportOptions() + (TransportConstants.HANDSHAKE_TIMEOUT to 0) + asMap(keyStoreType)) + fun rpcInternalAcceptorTcpTransport(hostAndPort: NetworkHostAndPort, + config: SslConfiguration, + keyStoreType: String? = null, + trace: Boolean = false): TransportConfiguration { + val options = mutableMapOf() + config.addToTransportOptions(options) + options += asMap(keyStoreType) + return createAcceptorTransport(hostAndPort, RPC_PROTOCOLS, options, enableSSL = true, trace = trace) } private fun asMap(keyStoreType: String?): Map { return keyStoreType?.let { mutableMapOf(TransportConstants.KEYSTORE_TYPE_PROP_NAME to it) } ?: emptyMap() } + + private fun createAcceptorTransport(hostAndPort: NetworkHostAndPort, + protocols: String, + options: MutableMap, + enableSSL: Boolean, + trace: Boolean): TransportConfiguration { + options += defaultArtemisOptions(hostAndPort, protocols) + if (enableSSL) { + options += defaultSSLOptions + } + // Suppress core.server.lambda$channelActive$0 - AMQ224088 error from load balancer type connections + options[TransportConstants.HANDSHAKE_TIMEOUT] = 0 + options[TRACE_NAME] = trace + return TransportConfiguration("net.corda.node.services.messaging.NodeNettyAcceptorFactory", options) + } + + private fun createConnectorTransport(hostAndPort: NetworkHostAndPort, + protocols: String, + options: MutableMap, + enableSSL: Boolean): TransportConfiguration { + options += defaultArtemisOptions(hostAndPort, protocols) + if (enableSSL) { + options += defaultSSLOptions + // This is required to stop Client checking URL address vs. Server provided certificate + options[TransportConstants.VERIFY_HOST_PROP_NAME] = false + } + return TransportConfiguration(NettyConnectorFactory::class.java.name, options) + } } -} \ No newline at end of file +} diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/AMQPBridgeManager.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/AMQPBridgeManager.kt index 40523033f2..9b24adc538 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/AMQPBridgeManager.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/AMQPBridgeManager.kt @@ -29,6 +29,7 @@ import org.apache.activemq.artemis.api.core.client.ClientMessage import org.apache.activemq.artemis.api.core.client.ClientSession import org.slf4j.MDC import rx.Subscription +import java.time.Duration import java.util.concurrent.Executors import java.util.concurrent.ScheduledExecutorService import java.util.concurrent.ScheduledFuture @@ -54,7 +55,7 @@ open class AMQPBridgeManager(keyStore: CertificateStore, private val artemisMessageClientFactory: () -> ArtemisSessionProvider, private val bridgeMetricsService: BridgeMetricsService? = null, trace: Boolean, - sslHandshakeTimeout: Long?, + sslHandshakeTimeout: Duration?, private val bridgeConnectionTTLSeconds: Int) : BridgeManager { private val lock = ReentrantLock() @@ -69,8 +70,8 @@ open class AMQPBridgeManager(keyStore: CertificateStore, override val enableSNI: Boolean, override val sourceX500Name: String? = null, override val trace: Boolean, - private val _sslHandshakeTimeout: Long?) : AMQPConfiguration { - override val sslHandshakeTimeout: Long + private val _sslHandshakeTimeout: Duration?) : AMQPConfiguration { + override val sslHandshakeTimeout: Duration get() = _sslHandshakeTimeout ?: super.sslHandshakeTimeout } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/BridgeControlListener.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/BridgeControlListener.kt index 422ecc1c7b..0fee8f1fba 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/BridgeControlListener.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/BridgeControlListener.kt @@ -5,16 +5,13 @@ import net.corda.core.identity.CordaX500Name import net.corda.core.serialization.SerializationDefaults import net.corda.core.serialization.deserialize import net.corda.core.serialization.serialize -import net.corda.core.utilities.NetworkHostAndPort import net.corda.core.utilities.contextLogger -import net.corda.nodeapi.internal.ArtemisMessagingClient import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.BRIDGE_CONTROL import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.BRIDGE_NOTIFY import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.P2P_PREFIX import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.PEERS_PREFIX import net.corda.nodeapi.internal.ArtemisSessionProvider import net.corda.nodeapi.internal.config.CertificateStore -import net.corda.nodeapi.internal.config.MutualSslConfiguration import net.corda.nodeapi.internal.crypto.x509 import net.corda.nodeapi.internal.protonwrapper.netty.ProxyConfig import net.corda.nodeapi.internal.protonwrapper.netty.RevocationConfig @@ -28,6 +25,7 @@ import org.apache.activemq.artemis.api.core.client.ClientMessage import org.apache.activemq.artemis.api.core.client.ClientSession import rx.Observable import rx.subjects.PublishSubject +import java.time.Duration import java.util.* class BridgeControlListener(private val keyStore: CertificateStore, @@ -40,7 +38,7 @@ class BridgeControlListener(private val keyStore: CertificateStore, private val artemisMessageClientFactory: () -> ArtemisSessionProvider, bridgeMetricsService: BridgeMetricsService? = null, trace: Boolean = false, - sslHandshakeTimeout: Long? = null, + sslHandshakeTimeout: Duration? = null, bridgeConnectionTTLSeconds: Int = 0) : AutoCloseable { private val bridgeId: String = UUID.randomUUID().toString() private var bridgeControlQueue = "$BRIDGE_CONTROL.$bridgeId" @@ -58,13 +56,6 @@ class BridgeControlListener(private val keyStore: CertificateStore, private var controlConsumer: ClientConsumer? = null private var notifyConsumer: ClientConsumer? = null - constructor(config: MutualSslConfiguration, - p2pAddress: NetworkHostAndPort, - maxMessageSize: Int, - revocationConfig: RevocationConfig, - enableSNI: Boolean, - proxy: ProxyConfig? = null) : this(config.keyStore.get(), config.trustStore.get(), config.useOpenSsl, proxy, maxMessageSize, revocationConfig, enableSNI, { ArtemisMessagingClient(config, p2pAddress, maxMessageSize) }) - companion object { private val log = contextLogger() } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/LoopbackBridgeManager.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/LoopbackBridgeManager.kt index fc27029584..e9ac1ca522 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/LoopbackBridgeManager.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/LoopbackBridgeManager.kt @@ -23,6 +23,7 @@ import org.apache.activemq.artemis.api.core.client.ClientMessage import org.apache.activemq.artemis.api.core.client.ClientProducer import org.apache.activemq.artemis.api.core.client.ClientSession import org.slf4j.MDC +import java.time.Duration /** * The LoopbackBridgeManager holds the list of independent LoopbackBridge objects that actively loopback messages to local Artemis @@ -40,7 +41,7 @@ class LoopbackBridgeManager(keyStore: CertificateStore, private val bridgeMetricsService: BridgeMetricsService? = null, private val isLocalInbox: (String) -> Boolean, trace: Boolean, - sslHandshakeTimeout: Long? = null, + sslHandshakeTimeout: Duration? = null, bridgeConnectionTTLSeconds: Int = 0) : AMQPBridgeManager(keyStore, trustStore, useOpenSSL, proxyConfig, maxMessageSize, revocationConfig, enableSNI, artemisMessageClientFactory, bridgeMetricsService, diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/config/SslConfiguration.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/config/SslConfiguration.kt index e4433b4e00..fdb8e9aea0 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/config/SslConfiguration.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/config/SslConfiguration.kt @@ -1,16 +1,20 @@ package net.corda.nodeapi.internal.config +import net.corda.core.utilities.seconds +import java.time.Duration + interface SslConfiguration { val keyStore: FileBasedCertificateStoreSupplier? val trustStore: FileBasedCertificateStoreSupplier? val useOpenSsl: Boolean + val handshakeTimeout: Duration? companion object { - - fun mutual(keyStore: FileBasedCertificateStoreSupplier, trustStore: FileBasedCertificateStoreSupplier): MutualSslConfiguration { - - return MutualSslOptions(keyStore, trustStore) + fun mutual(keyStore: FileBasedCertificateStoreSupplier, + trustStore: FileBasedCertificateStoreSupplier, + handshakeTimeout: Duration? = null): MutualSslConfiguration { + return MutualSslOptions(keyStore, trustStore, handshakeTimeout) } } } @@ -21,9 +25,10 @@ interface MutualSslConfiguration : SslConfiguration { } private class MutualSslOptions(override val keyStore: FileBasedCertificateStoreSupplier, - override val trustStore: FileBasedCertificateStoreSupplier) : MutualSslConfiguration { + override val trustStore: FileBasedCertificateStoreSupplier, + override val handshakeTimeout: Duration?) : MutualSslConfiguration { override val useOpenSsl: Boolean = false } -const val DEFAULT_SSL_HANDSHAKE_TIMEOUT_MILLIS = 60000L // Set at least 3 times higher than sun.security.provider.certpath.URICertStore.DEFAULT_CRL_CONNECT_TIMEOUT which is 15 sec - +@Suppress("MagicNumber") +val DEFAULT_SSL_HANDSHAKE_TIMEOUT: Duration = 60.seconds // Set at least 3 times higher than sun.security.provider.certpath.URICertStore.DEFAULT_CRL_CONNECT_TIMEOUT which is 15 sec diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/crypto/X509Utilities.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/crypto/X509Utilities.kt index aff65d7987..ff115ec78c 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/crypto/X509Utilities.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/crypto/X509Utilities.kt @@ -438,6 +438,8 @@ class X509CertificateFactory { fun generateCertPath(vararg certificates: X509Certificate): CertPath = generateCertPath(certificates.asList()) fun generateCertPath(certificates: List): CertPath = delegate.generateCertPath(certificates) + + fun generateCRL(input: InputStream): X509CRL = delegate.generateCRL(input) as X509CRL } enum class CertificateType(val keyUsage: KeyUsage, vararg val purposes: KeyPurposeId, val isCA: Boolean, val role: CertRole?) { diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPChannelHandler.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPChannelHandler.kt index 4cba1c6d95..e42cedcbb3 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPChannelHandler.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPChannelHandler.kt @@ -186,7 +186,7 @@ internal class AMQPChannelHandler(private val serverMode: Boolean, } } - @Suppress("OverridingDeprecatedMember") + @Deprecated("Deprecated in Java") override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { logWarnWithMDC("Closing channel due to nonrecoverable exception ${cause.message}") if (log.isTraceEnabled) { @@ -298,16 +298,16 @@ internal class AMQPChannelHandler(private val serverMode: Boolean, cause is ClosedChannelException -> logWarnWithMDC("SSL Handshake closed early.") cause is SslHandshakeTimeoutException -> logWarnWithMDC("SSL Handshake timed out") // Sadly the exception thrown by Netty wrapper requires that we check the message. - cause is SSLException && (cause.message?.contains("close_notify") == true) - -> logWarnWithMDC("Received close_notify during handshake") + cause is SSLException && (cause.message?.contains("close_notify") == true) -> logWarnWithMDC("Received close_notify during handshake") // io.netty.handler.ssl.SslHandler.setHandshakeFailureTransportFailure() cause is SSLException && (cause.message?.contains("writing TLS control frames") == true) -> logWarnWithMDC(cause.message!!) cause is SSLException && (cause.message?.contains("internal_error") == true) -> logWarnWithMDC("Received internal_error during handshake") else -> connectionResult = ConnectionResult.HANDSHAKE_FAILURE } - logWarnWithMDC("Handshake failure: ${evt.cause().message}") if (log.isTraceEnabled) { - withMDC { log.trace("Handshake failure", evt.cause()) } + withMDC { log.trace("Handshake failure", cause) } + } else { + logWarnWithMDC("Handshake failure: ${cause.message}") } ctx.close() } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPClient.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPClient.kt index 4c8b78d57f..c14ce5e820 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPClient.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPClient.kt @@ -26,6 +26,7 @@ import rx.Observable import rx.subjects.PublishSubject import java.lang.Long.min import java.net.InetSocketAddress +import java.security.cert.CertPathValidatorException import java.time.Duration import java.util.concurrent.TimeUnit import java.util.concurrent.locks.ReentrantLock @@ -54,7 +55,7 @@ data class ProxyConfig(val version: ProxyVersion, val proxyAddress: NetworkHostA * otherwise it creates a self-contained Netty thraed pool and socket objects. * Once connected it can accept application packets to send via the AMQP protocol. */ -class AMQPClient(val targets: List, +class AMQPClient(private val targets: List, val allowedRemoteLegalNames: Set, private val configuration: AMQPConfiguration, private val sharedThreadPool: EventLoopGroup? = null) : AutoCloseable { @@ -84,6 +85,7 @@ class AMQPClient(val targets: List, private var targetIndex = 0 private var currentTarget: NetworkHostAndPort = targets.first() private var retryInterval = MIN_RETRY_INTERVAL + private val revocationChecker = configuration.revocationConfig.createPKIXRevocationChecker() private val handshakeFailureRetryTargets = mutableSetOf() private var retryingHandshakeFailures = false private var retryOffset = 0 @@ -158,24 +160,22 @@ class AMQPClient(val targets: List, log.info("Retry connect to ${targets[targetIndex]} in [$retryInterval] ms") } - private val connectListener = object : ChannelFutureListener { - override fun operationComplete(future: ChannelFuture) { - amqpActive = false - if (!future.isSuccess) { - log.info("Failed to connect to $currentTarget", future.cause()) + private val connectListener = ChannelFutureListener { future -> + amqpActive = false + if (!future.isSuccess) { + log.info("Failed to connect to $currentTarget", future.cause()) - if (started) { - workerGroup?.schedule({ - nextTarget() - restart() - }, retryInterval, TimeUnit.MILLISECONDS) - } - } else { - // Connection established successfully - clientChannel = future.channel() - clientChannel?.closeFuture()?.addListener(closeListener) - log.info("Connected to $currentTarget, Local address: $localAddressString") + if (started) { + workerGroup?.schedule({ + nextTarget() + restart() + }, retryInterval, TimeUnit.MILLISECONDS) } + } else { + // Connection established successfully + clientChannel = future.channel() + clientChannel?.closeFuture()?.addListener(closeListener) + log.info("Connected to $currentTarget, Local address: $localAddressString") } } @@ -201,7 +201,7 @@ class AMQPClient(val targets: List, init { keyManagerFactory.init(conf.keyStore) - trustManagerFactory.init(initialiseTrustStoreAndEnableCrlChecking(conf.trustStore, conf.revocationConfig)) + trustManagerFactory.init(initialiseTrustStoreAndEnableCrlChecking(conf.trustStore, parent.revocationChecker)) } @Suppress("ComplexMethod") @@ -245,12 +245,13 @@ class AMQPClient(val targets: List, val handler = if (parent.configuration.useOpenSsl) { createClientOpenSslHandler(target, parent.allowedRemoteLegalNames, wrappedKeyManagerFactory, trustManagerFactory, ch.alloc()) } else { - createClientSslHelper(target, parent.allowedRemoteLegalNames, wrappedKeyManagerFactory, trustManagerFactory) + createClientSslHandler(target, parent.allowedRemoteLegalNames, wrappedKeyManagerFactory, trustManagerFactory) } - handler.handshakeTimeoutMillis = conf.sslHandshakeTimeout + handler.handshakeTimeoutMillis = conf.sslHandshakeTimeout.toMillis() pipeline.addLast("sslHandler", handler) if (conf.trace) pipeline.addLast("logger", LoggingHandler(LogLevel.INFO)) - amqpChannelHandler = AMQPChannelHandler(false, + amqpChannelHandler = AMQPChannelHandler( + false, parent.allowedRemoteLegalNames, // Single entry, key can be anything. mapOf(DEFAULT to wrappedKeyManagerFactory), @@ -258,37 +259,41 @@ class AMQPClient(val targets: List, conf.password, conf.trace, false, - onOpen = { _, change -> - parent.run { - amqpActive = true - successfullyConnected() - _onConnection.onNext(change) - } - }, - onClose = { _, change -> - if (parent.amqpChannelHandler == amqpChannelHandler) { - parent.run { - _onConnection.onNext(change) - if (change.connectionResult == ConnectionResult.HANDSHAKE_FAILURE) { - log.warn("Handshake failure with $target target; will retry later") - handshakeFailureRetryTargets += target - } - - if (started && amqpActive) { - log.debug { "Scheduling restart of $currentTarget (AMQP active)" } - workerGroup?.schedule({ - nextTarget() - restart() - }, retryInterval, TimeUnit.MILLISECONDS) - } - amqpActive = false - } - } - }, - onReceive = { rcv -> parent._onReceive.onNext(rcv) }) + onOpen = { _, change -> onChannelOpen(change) }, + onClose = { _, change -> onChannelClose(change, target) }, + onReceive = parent._onReceive::onNext + ) parent.amqpChannelHandler = amqpChannelHandler pipeline.addLast(amqpChannelHandler) } + + private fun onChannelOpen(change: ConnectionChange) { + parent.run { + amqpActive = true + successfullyConnected() + _onConnection.onNext(change) + } + } + + private fun onChannelClose(change: ConnectionChange, target: NetworkHostAndPort) { + if (parent.amqpChannelHandler != amqpChannelHandler) return + parent.run { + _onConnection.onNext(change) + if (change.connectionResult == ConnectionResult.HANDSHAKE_FAILURE) { + log.warn("Handshake failure with $target target; will retry later") + handshakeFailureRetryTargets += target + } + + if (started && amqpActive) { + log.debug { "Scheduling restart of $currentTarget (AMQP active)" } + workerGroup?.schedule({ + nextTarget() + restart() + }, retryInterval, TimeUnit.MILLISECONDS) + } + amqpActive = false + } + } } fun start() { @@ -372,4 +377,6 @@ class AMQPClient(val targets: List, private val _onConnection = PublishSubject.create().toSerialized() val onConnection: Observable get() = _onConnection + + val softFailExceptions: List get() = revocationChecker.softFailExceptions } \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPConfiguration.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPConfiguration.kt index db0dd8023c..c992dd55e4 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPConfiguration.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPConfiguration.kt @@ -2,7 +2,8 @@ package net.corda.nodeapi.internal.protonwrapper.netty import net.corda.nodeapi.internal.ArtemisMessagingComponent import net.corda.nodeapi.internal.config.CertificateStore -import net.corda.nodeapi.internal.config.DEFAULT_SSL_HANDSHAKE_TIMEOUT_MILLIS +import net.corda.nodeapi.internal.config.DEFAULT_SSL_HANDSHAKE_TIMEOUT +import java.time.Duration interface AMQPConfiguration { /** @@ -67,8 +68,8 @@ interface AMQPConfiguration { get() = false @JvmDefault - val sslHandshakeTimeout: Long - get() = DEFAULT_SSL_HANDSHAKE_TIMEOUT_MILLIS // Aligned with sun.security.provider.certpath.URICertStore.DEFAULT_CRL_CONNECT_TIMEOUT + val sslHandshakeTimeout: Duration + get() = DEFAULT_SSL_HANDSHAKE_TIMEOUT // Aligned with sun.security.provider.certpath.URICertStore.DEFAULT_CRL_CONNECT_TIMEOUT /** * An optional Health Check Phrase which if passed through the channel will cause AMQP Server to echo it back instead of doing normal pipeline processing diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPServer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPServer.kt index 20834a2041..126e47a3e6 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPServer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPServer.kt @@ -25,6 +25,7 @@ import rx.Observable import rx.subjects.PublishSubject import java.net.BindException import java.net.InetSocketAddress +import java.security.cert.CertPathValidatorException import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.locks.ReentrantLock import javax.net.ssl.KeyManagerFactory @@ -55,6 +56,7 @@ class AMQPServer(val hostName: String, private var bossGroup: EventLoopGroup? = null private var workerGroup: EventLoopGroup? = null private var serverChannel: Channel? = null + private val revocationChecker = configuration.revocationConfig.createPKIXRevocationChecker() private val clientChannels = ConcurrentHashMap() private class ServerChannelInitializer(val parent: AMQPServer) : ChannelInitializer() { @@ -64,7 +66,7 @@ class AMQPServer(val hostName: String, init { keyManagerFactory.init(conf.keyStore.value.internal, conf.keyStore.entryPassword.toCharArray()) - trustManagerFactory.init(initialiseTrustStoreAndEnableCrlChecking(conf.trustStore, conf.revocationConfig)) + trustManagerFactory.init(initialiseTrustStoreAndEnableCrlChecking(conf.trustStore, parent.revocationChecker)) } override fun initChannel(ch: SocketChannel) { @@ -75,7 +77,8 @@ class AMQPServer(val hostName: String, pipeline.addLast("sslHandler", sslHandler) if (conf.trace) pipeline.addLast("logger", LoggingHandler(LogLevel.INFO)) val suppressLogs = ch.remoteAddress()?.hostString in amqpConfiguration.silencedIPs - pipeline.addLast(AMQPChannelHandler(true, + pipeline.addLast(AMQPChannelHandler( + true, null, // Passing a mapping of legal names to key managers to be able to pick the correct one after // SNI completion event is fired up. @@ -84,27 +87,32 @@ class AMQPServer(val hostName: String, conf.password, conf.trace, suppressLogs, - onOpen = { channel, change -> - parent.run { - clientChannels[channel.remoteAddress()] = channel - _onConnection.onNext(change) - } - }, - onClose = { channel, change -> - parent.run { - val remoteAddress = channel.remoteAddress() - clientChannels.remove(remoteAddress) - _onConnection.onNext(change) - } - }, - onReceive = { rcv -> parent._onReceive.onNext(rcv) })) + onOpen = ::onChannelOpen, + onClose = ::onChannelClose, + onReceive = parent._onReceive::onNext + )) + } + + private fun onChannelOpen(channel: SocketChannel, change: ConnectionChange) { + parent.run { + clientChannels[channel.remoteAddress()] = channel + _onConnection.onNext(change) + } + } + + private fun onChannelClose(channel: SocketChannel, change: ConnectionChange) { + parent.run { + val remoteAddress = channel.remoteAddress() + clientChannels.remove(remoteAddress) + _onConnection.onNext(change) + } } private fun createSSLHandler(amqpConfig: AMQPConfiguration, ch: SocketChannel): Pair> { return if (amqpConfig.useOpenSsl && amqpConfig.enableSNI && amqpConfig.keyStore.aliases().size > 1) { val keyManagerFactoriesMap = splitKeystore(amqpConfig) // SNI matching needed only when multiple nodes exist behind the server. - Pair(createServerSNIOpenSslHandler(keyManagerFactoriesMap, trustManagerFactory), keyManagerFactoriesMap) + Pair(createServerSNIOpenSniHandler(keyManagerFactoriesMap, trustManagerFactory), keyManagerFactoriesMap) } else { val keyManagerFactory = CertHoldingKeyManagerFactoryWrapper(keyManagerFactory, amqpConfig) val handler = if (amqpConfig.useOpenSsl) { @@ -113,7 +121,7 @@ class AMQPServer(val hostName: String, // For javaSSL, SNI matching is handled at key manager level. createServerSslHandler(amqpConfig.keyStore, keyManagerFactory, trustManagerFactory) } - handler.handshakeTimeoutMillis = amqpConfig.sslHandshakeTimeout + handler.handshakeTimeoutMillis = amqpConfig.sslHandshakeTimeout.toMillis() Pair(handler, mapOf(DEFAULT to keyManagerFactory)) } } @@ -217,4 +225,6 @@ class AMQPServer(val hostName: String, private val _onConnection = PublishSubject.create().toSerialized() val onConnection: Observable get() = _onConnection + + val softFailExceptions: List get() = revocationChecker.softFailExceptions } \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AllowAllRevocationChecker.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AllowAllRevocationChecker.kt index 30e0445689..538bb17a1e 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AllowAllRevocationChecker.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AllowAllRevocationChecker.kt @@ -11,7 +11,7 @@ object AllowAllRevocationChecker : PKIXRevocationChecker() { private val logger = LoggerFactory.getLogger(AllowAllRevocationChecker::class.java) - override fun check(cert: Certificate?, unresolvedCritExts: MutableCollection?) { + override fun check(cert: Certificate, unresolvedCritExts: Collection) { logger.debug {"Passing certificate check for: $cert"} // Nothing to do } @@ -20,7 +20,7 @@ object AllowAllRevocationChecker : PKIXRevocationChecker() { return true } - override fun getSupportedExtensions(): MutableSet? { + override fun getSupportedExtensions(): Set? { return null } @@ -28,7 +28,7 @@ object AllowAllRevocationChecker : PKIXRevocationChecker() { // Nothing to do } - override fun getSoftFailExceptions(): MutableList { - return LinkedList() + override fun getSoftFailExceptions(): List { + return Collections.emptyList() } } \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/ExternalCrlSource.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/CrlSource.kt similarity index 70% rename from node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/ExternalCrlSource.kt rename to node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/CrlSource.kt index 654ead24a0..a0cbd7079c 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/ExternalCrlSource.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/CrlSource.kt @@ -3,10 +3,11 @@ package net.corda.nodeapi.internal.protonwrapper.netty import java.security.cert.X509CRL import java.security.cert.X509Certificate -interface ExternalCrlSource { +@FunctionalInterface +interface CrlSource { /** * Given certificate provides a set of CRLs, potentially performing remote communication. */ - fun fetch(certificate: X509Certificate) : Set -} \ No newline at end of file + fun fetch(certificate: X509Certificate): Set +} diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/RevocationConfig.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/RevocationConfig.kt index f444421128..6e8c695387 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/RevocationConfig.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/RevocationConfig.kt @@ -3,6 +3,9 @@ package net.corda.nodeapi.internal.protonwrapper.netty import com.typesafe.config.Config import net.corda.nodeapi.internal.config.ConfigParser import net.corda.nodeapi.internal.config.CustomConfigParser +import net.corda.nodeapi.internal.revocation.CertDistPointCrlSource +import net.corda.nodeapi.internal.revocation.CordaRevocationChecker +import java.security.cert.PKIXRevocationChecker /** * Data structure for controlling the way how Certificate Revocation Lists are handled. @@ -26,7 +29,7 @@ interface RevocationConfig { /** * CRLs are obtained from external source - * @see ExternalCrlSource + * @see CrlSource */ EXTERNAL_SOURCE, @@ -39,14 +42,21 @@ interface RevocationConfig { val mode: Mode /** - * Optional `ExternalCrlSource` which only makes sense with `mode` = `EXTERNAL_SOURCE` + * Optional [CrlSource] which only makes sense with `mode` = `EXTERNAL_SOURCE` */ - val externalCrlSource: ExternalCrlSource? + val externalCrlSource: CrlSource? - /** - * Creates a copy of `RevocationConfig` with ExternalCrlSource enriched - */ - fun enrichExternalCrlSource(sourceFunc: (() -> ExternalCrlSource)?): RevocationConfig + fun createPKIXRevocationChecker(): PKIXRevocationChecker { + return when (mode) { + Mode.OFF -> AllowAllRevocationChecker + Mode.EXTERNAL_SOURCE -> { + val externalCrlSource = requireNotNull(externalCrlSource) { "externalCrlSource must be specfied for EXTERNAL_SOURCE" } + CordaRevocationChecker(externalCrlSource, softFail = true) + } + Mode.SOFT_FAIL -> CordaRevocationChecker(CertDistPointCrlSource(), softFail = true) + Mode.HARD_FAIL -> CordaRevocationChecker(CertDistPointCrlSource(), softFail = false) + } + } } /** @@ -54,16 +64,7 @@ interface RevocationConfig { */ fun Boolean.toRevocationConfig() = if(this) RevocationConfigImpl(RevocationConfig.Mode.SOFT_FAIL) else RevocationConfigImpl(RevocationConfig.Mode.HARD_FAIL) -data class RevocationConfigImpl(override val mode: RevocationConfig.Mode, override val externalCrlSource: ExternalCrlSource? = null) : RevocationConfig { - override fun enrichExternalCrlSource(sourceFunc: (() -> ExternalCrlSource)?): RevocationConfig { - return if(mode != RevocationConfig.Mode.EXTERNAL_SOURCE) { - this - } else { - assert(sourceFunc != null) { "There should be a way to obtain ExternalCrlSource" } - copy(externalCrlSource = sourceFunc!!()) - } - } -} +data class RevocationConfigImpl(override val mode: RevocationConfig.Mode, override val externalCrlSource: CrlSource? = null) : RevocationConfig class RevocationConfigParser : ConfigParser { override fun parse(config: Config): RevocationConfig { @@ -80,4 +81,4 @@ class RevocationConfigParser : ConfigParser { else -> throw IllegalArgumentException("Unsupported mode : '$mode'") } } -} \ No newline at end of file +} diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelper.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelper.kt index 87e6a3e4cb..ab13048d67 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelper.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelper.kt @@ -13,15 +13,17 @@ import net.corda.core.identity.CordaX500Name import net.corda.core.internal.VisibleForTesting import net.corda.core.utilities.NetworkHostAndPort import net.corda.core.utilities.contextLogger +import net.corda.core.utilities.debug import net.corda.core.utilities.toHex import net.corda.nodeapi.internal.ArtemisTcpTransport import net.corda.nodeapi.internal.config.CertificateStore import net.corda.nodeapi.internal.crypto.toBc import net.corda.nodeapi.internal.crypto.x509 -import net.corda.nodeapi.internal.protonwrapper.netty.revocation.ExternalSourceRevocationChecker -import org.bouncycastle.asn1.ASN1IA5String import org.bouncycastle.asn1.ASN1InputStream +import org.bouncycastle.asn1.ASN1Primitive +import org.bouncycastle.asn1.ASN1IA5String import org.bouncycastle.asn1.DEROctetString +import org.bouncycastle.asn1.x500.X500Name import org.bouncycastle.asn1.x509.AuthorityKeyIdentifier import org.bouncycastle.asn1.x509.CRLDistPoint import org.bouncycastle.asn1.x509.DistributionPointName @@ -30,13 +32,15 @@ import org.bouncycastle.asn1.x509.GeneralName import org.bouncycastle.asn1.x509.GeneralNames import org.bouncycastle.asn1.x509.SubjectKeyIdentifier import org.slf4j.LoggerFactory -import java.io.ByteArrayInputStream import java.net.Socket +import java.net.URI import java.security.KeyStore import java.security.cert.* import java.util.* import java.util.concurrent.Executor import javax.net.ssl.* +import javax.security.auth.x500.X500Principal +import kotlin.collections.HashMap import kotlin.system.measureTimeMillis private const val HOSTNAME_FORMAT = "%s.corda.net" @@ -46,44 +50,61 @@ internal const val DP_DEFAULT_ANSWER = "NO CRLDP ext" internal val logger = LoggerFactory.getLogger("net.corda.nodeapi.internal.protonwrapper.netty.SSLHelper") -fun X509Certificate.distributionPoints() : Set? { - logger.debug("Checking CRLDPs for $subjectX500Principal") +/** + * Returns all the CRL distribution points in the certificate as [URI]s along with the CRL issuer names, if any. + */ +@Suppress("ComplexMethod") +fun X509Certificate.distributionPoints(): Map?> { + logger.debug { "Checking CRLDPs for $subjectX500Principal" } val crldpExtBytes = getExtensionValue(Extension.cRLDistributionPoints.id) if (crldpExtBytes == null) { logger.debug(DP_DEFAULT_ANSWER) - return emptySet() + return emptyMap() } - val derObjCrlDP = ASN1InputStream(ByteArrayInputStream(crldpExtBytes)).readObject() + val derObjCrlDP = crldpExtBytes.toAsn1Object() val dosCrlDP = derObjCrlDP as? DEROctetString if (dosCrlDP == null) { logger.error("Expected to have DEROctetString, actual type: ${derObjCrlDP.javaClass}") - return emptySet() + return emptyMap() } - val crldpExtOctetsBytes = dosCrlDP.octets - val dpObj = ASN1InputStream(ByteArrayInputStream(crldpExtOctetsBytes)).readObject() - val distPoint = CRLDistPoint.getInstance(dpObj) - if (distPoint == null) { + val dpObj = dosCrlDP.octets.toAsn1Object() + val crlDistPoint = CRLDistPoint.getInstance(dpObj) + if (crlDistPoint == null) { logger.error("Could not instantiate CRLDistPoint, from: $dpObj") - return emptySet() + return emptyMap() } - val dpNames = distPoint.distributionPoints.mapNotNull { it.distributionPoint }.filter { it.type == DistributionPointName.FULL_NAME } - val generalNames = dpNames.flatMap { GeneralNames.getInstance(it.name).names.asList() } - return generalNames.filter { it.tagNo == GeneralName.uniformResourceIdentifier}.map { ASN1IA5String.getInstance(it.name).string }.toSet() -} - -fun X509Certificate.distributionPointsToString() : String { - return with(distributionPoints()) { - if(this == null || isEmpty()) { - DP_DEFAULT_ANSWER - } else { - sorted().joinToString() + val dpMap = HashMap?>() + for (distributionPoint in crlDistPoint.distributionPoints) { + val distributionPointName = distributionPoint.distributionPoint + if (distributionPointName?.type != DistributionPointName.FULL_NAME) continue + val issuerNames = distributionPoint.crlIssuer?.names?.mapNotNull { + if (it.tagNo == GeneralName.directoryName) { + X500Principal(X500Name.getInstance(it.name).encoded) + } else { + null + } + } + for (generalName in GeneralNames.getInstance(distributionPointName.name).names) { + if (generalName.tagNo == GeneralName.uniformResourceIdentifier) { + val uri = URI(ASN1IA5String.getInstance(generalName.name).string) + dpMap[uri] = issuerNames + } } } + return dpMap } +fun X509Certificate.distributionPointsToString(): String { + return with(distributionPoints().keys) { + if (isEmpty()) DP_DEFAULT_ANSWER else sorted().joinToString() + } +} + +fun ByteArray.toAsn1Object(): ASN1Primitive = ASN1InputStream(this).readObject() + fun certPathToString(certPath: Array?): String { if (certPath == null) { return "" @@ -117,7 +138,7 @@ class LoggingTrustManagerWrapper(val wrapped: X509ExtendedTrustManager) : X509Ex if (chain == null) { return "" } - return chain.map { it.toString() }.joinToString(", ") + return chain.joinToString(", ") { it.toString() } } private fun logErrors(chain: Array?, block: () -> Unit) { @@ -171,14 +192,9 @@ class LoggingTrustManagerWrapper(val wrapped: X509ExtendedTrustManager) : X509Ex private object LoggingImmediateExecutor : Executor { - override fun execute(command: Runnable?) { + override fun execute(command: Runnable) { val log = LoggerFactory.getLogger(javaClass) - if (command == null) { - log.error("SSL handler executor called with a null command") - throw NullPointerException("command") - } - @Suppress("TooGenericExceptionCaught", "MagicNumber") // log and rethrow all exceptions try { val commandName = command::class.qualifiedName?.let { "[$it]" } ?: "" @@ -196,10 +212,10 @@ private object LoggingImmediateExecutor : Executor { } } -internal fun createClientSslHelper(target: NetworkHostAndPort, - expectedRemoteLegalNames: Set, - keyManagerFactory: KeyManagerFactory, - trustManagerFactory: TrustManagerFactory): SslHandler { +internal fun createClientSslHandler(target: NetworkHostAndPort, + expectedRemoteLegalNames: Set, + keyManagerFactory: KeyManagerFactory, + trustManagerFactory: TrustManagerFactory): SslHandler { val sslContext = createAndInitSslContext(keyManagerFactory, trustManagerFactory) val sslEngine = sslContext.createSSLEngine(target.host, target.port) sslEngine.useClientMode = true @@ -211,7 +227,6 @@ internal fun createClientSslHelper(target: NetworkHostAndPort, sslParameters.serverNames = listOf(SNIHostName(x500toHostName(expectedRemoteLegalNames.single()))) sslEngine.sslParameters = sslParameters } - @Suppress("DEPRECATION") return SslHandler(sslEngine, false, LoggingImmediateExecutor) } @@ -229,7 +244,6 @@ internal fun createClientOpenSslHandler(target: NetworkHostAndPort, sslParameters.serverNames = listOf(SNIHostName(x500toHostName(expectedRemoteLegalNames.single()))) sslEngine.sslParameters = sslParameters } - @Suppress("DEPRECATION") return SslHandler(sslEngine, false, LoggingImmediateExecutor) } @@ -246,7 +260,15 @@ internal fun createServerSslHandler(keyStore: CertificateStore, val sslParameters = sslEngine.sslParameters sslParameters.sniMatchers = listOf(ServerSNIMatcher(keyStore)) sslEngine.sslParameters = sslParameters - @Suppress("DEPRECATION") + return SslHandler(sslEngine, false, LoggingImmediateExecutor) +} + +internal fun createServerOpenSslHandler(keyManagerFactory: KeyManagerFactory, + trustManagerFactory: TrustManagerFactory, + alloc: ByteBufAllocator): SslHandler { + val sslContext = getServerSslContextBuilder(keyManagerFactory, trustManagerFactory).build() + val sslEngine = sslContext.newEngine(alloc) + sslEngine.useClientMode = false return SslHandler(sslEngine, false, LoggingImmediateExecutor) } @@ -259,52 +281,23 @@ fun createAndInitSslContext(keyManagerFactory: KeyManagerFactory, trustManagerFa return sslContext } -@VisibleForTesting -fun initialiseTrustStoreAndEnableCrlChecking(trustStore: CertificateStore, revocationConfig: RevocationConfig): ManagerFactoryParameters { - val pkixParams = PKIXBuilderParameters(trustStore.value.internal, X509CertSelector()) - val revocationChecker = when (revocationConfig.mode) { - RevocationConfig.Mode.OFF -> AllowAllRevocationChecker // Custom PKIXRevocationChecker skipping CRL check - RevocationConfig.Mode.EXTERNAL_SOURCE -> { - require(revocationConfig.externalCrlSource != null) { "externalCrlSource must not be null" } - ExternalSourceRevocationChecker(revocationConfig.externalCrlSource!!) { Date() } // Custom PKIXRevocationChecker which uses `externalCrlSource` - } - else -> { - val certPathBuilder = CertPathBuilder.getInstance("PKIX") - val pkixRevocationChecker = certPathBuilder.revocationChecker as PKIXRevocationChecker - pkixRevocationChecker.options = EnumSet.of( - // Prefer CRL over OCSP - PKIXRevocationChecker.Option.PREFER_CRLS, - // Don't fall back to OCSP checking - PKIXRevocationChecker.Option.NO_FALLBACK) - if (revocationConfig.mode == RevocationConfig.Mode.SOFT_FAIL) { - // Allow revocation check to succeed if the revocation status cannot be determined for one of - // the following reasons: The CRL or OCSP response cannot be obtained because of a network error. - pkixRevocationChecker.options = pkixRevocationChecker.options + PKIXRevocationChecker.Option.SOFT_FAIL - } - pkixRevocationChecker - } - } - pkixParams.addCertPathChecker(revocationChecker) - return CertPathTrustManagerParameters(pkixParams) +fun initialiseTrustStoreAndEnableCrlChecking(trustStore: CertificateStore, + revocationConfig: RevocationConfig): CertPathTrustManagerParameters { + return initialiseTrustStoreAndEnableCrlChecking(trustStore, revocationConfig.createPKIXRevocationChecker()) } -internal fun createServerOpenSslHandler(keyManagerFactory: KeyManagerFactory, - trustManagerFactory: TrustManagerFactory, - alloc: ByteBufAllocator): SslHandler { - - val sslContext = getServerSslContextBuilder(keyManagerFactory, trustManagerFactory).build() - val sslEngine = sslContext.newEngine(alloc) - sslEngine.useClientMode = false - @Suppress("DEPRECATION") - return SslHandler(sslEngine, false, LoggingImmediateExecutor) +fun initialiseTrustStoreAndEnableCrlChecking(trustStore: CertificateStore, + revocationChecker: PKIXRevocationChecker): CertPathTrustManagerParameters { + val pkixParams = PKIXBuilderParameters(trustStore.value.internal, X509CertSelector()) + pkixParams.addCertPathChecker(revocationChecker) + return CertPathTrustManagerParameters(pkixParams) } /** * Creates a special SNI handler used only when openSSL is used for AMQPServer */ -internal fun createServerSNIOpenSslHandler(keyManagerFactoriesMap: Map, +internal fun createServerSNIOpenSniHandler(keyManagerFactoriesMap: Map, trustManagerFactory: TrustManagerFactory): SniHandler { - // Default value can be any in the map. val sslCtxBuilder = getServerSslContextBuilder(keyManagerFactoriesMap.values.first(), trustManagerFactory) val mapping = DomainWildcardMappingBuilder(sslCtxBuilder.build()) @@ -327,7 +320,7 @@ private fun getServerSslContextBuilder(keyManagerFactory: KeyManagerFactory, tru internal fun splitKeystore(config: AMQPConfiguration): Map { val keyStore = config.keyStore.value.internal val password = config.keyStore.entryPassword.toCharArray() - return keyStore.aliases().toList().map { alias -> + return keyStore.aliases().toList().associate { alias -> val key = keyStore.getKey(alias, password) val certs = keyStore.getCertificateChain(alias) val x500Name = keyStore.getCertificate(alias).x509.subjectX500Principal @@ -338,7 +331,7 @@ internal fun splitKeystore(config: AMQPConfiguration): Map = Caffeine.newBuilder() + .maximumSize(java.lang.Long.getLong("net.corda.dpcrl.cache.size", DEFAULT_CACHE_SIZE)) + .expireAfterWrite(java.lang.Long.getLong("net.corda.dpcrl.cache.expiry", DEFAULT_CACHE_EXPIRY), TimeUnit.MILLISECONDS) + .build(::retrieveCRL) + + private val connectTimeout = Integer.getInteger("net.corda.dpcrl.connect.timeout", DEFAULT_CONNECT_TIMEOUT) + private val readTimeout = Integer.getInteger("net.corda.dpcrl.read.timeout", DEFAULT_READ_TIMEOUT) + + private fun retrieveCRL(uri: URI): X509CRL { + val bytes = run { + val conn = uri.toURL().openConnection() + conn.connectTimeout = connectTimeout + conn.readTimeout = readTimeout + // Read all bytes first and then pass them into the CertificateFactory. This may seem unnecessary when generateCRL already takes + // in an InputStream, but the JDK implementation (sun.security.provider.X509Factory.engineGenerateCRL) converts any IOException + // into CRLException and drops the cause chain. + conn.getInputStream().readFully() + } + return X509CertificateFactory().generateCRL(bytes.inputStream()) + } + } + + @Suppress("TooGenericExceptionCaught") + override fun fetch(certificate: X509Certificate): Set { + val approvedCRLs = HashSet() + var exception: Exception? = null + for ((distPointUri, issuerNames) in certificate.distributionPoints()) { + try { + val possibleCRL = getPossibleCRL(distPointUri) + if (verifyCRL(possibleCRL, certificate, issuerNames)) { + approvedCRLs += possibleCRL + } + } catch (e: Exception) { + if (exception == null) { + exception = e + } else { + exception.addSuppressed(e) + } + } + } + // Only throw if no CRLs are retrieved + if (exception != null && approvedCRLs.isEmpty()) { + throw exception + } else { + return approvedCRLs + } + } + + private fun getPossibleCRL(uri: URI): X509CRL { + return cache[uri]!! + } + + // DistributionPointFetcher.verifyCRL + private fun verifyCRL(crl: X509CRL, certificate: X509Certificate, distPointIssuerNames: List?): Boolean { + val crlIssuer = crl.issuerX500Principal + return distPointIssuerNames?.any { it == crlIssuer } ?: (certificate.issuerX500Principal == crlIssuer) + } +} diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/revocation/ExternalSourceRevocationChecker.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/revocation/CordaRevocationChecker.kt similarity index 50% rename from node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/revocation/ExternalSourceRevocationChecker.kt rename to node-api/src/main/kotlin/net/corda/nodeapi/internal/revocation/CordaRevocationChecker.kt index 23af94ca3d..b90bde624e 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/revocation/ExternalSourceRevocationChecker.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/revocation/CordaRevocationChecker.kt @@ -1,30 +1,53 @@ -package net.corda.nodeapi.internal.protonwrapper.netty.revocation +package net.corda.nodeapi.internal.revocation import net.corda.core.utilities.contextLogger -import net.corda.nodeapi.internal.protonwrapper.netty.ExternalCrlSource +import net.corda.nodeapi.internal.protonwrapper.netty.CrlSource import org.bouncycastle.asn1.x509.Extension import java.security.cert.CRLReason import java.security.cert.CertPathValidatorException +import java.security.cert.CertPathValidatorException.BasicReason import java.security.cert.Certificate import java.security.cert.CertificateRevokedException import java.security.cert.PKIXRevocationChecker import java.security.cert.X509CRL import java.security.cert.X509Certificate import java.util.* +import kotlin.collections.ArrayList /** - * Implementation of [PKIXRevocationChecker] which determines whether certificate is revoked using [externalCrlSource] which knows how to - * obtain a set of CRLs for a given certificate from an external source + * Custom [PKIXRevocationChecker] which delegates to a plugable [CrlSource] to retrieve the CRLs for certificate revocation checks. */ -class ExternalSourceRevocationChecker(private val externalCrlSource: ExternalCrlSource, private val dateSource: () -> Date) : PKIXRevocationChecker() { - +class CordaRevocationChecker(private val crlSource: CrlSource, + private val softFail: Boolean, + private val dateSource: () -> Date = ::Date) : PKIXRevocationChecker() { companion object { private val logger = contextLogger() } + private val softFailExceptions = ArrayList() + override fun check(cert: Certificate, unresolvedCritExts: MutableCollection?) { val x509Certificate = cert as X509Certificate - checkApprovedCRLs(x509Certificate, externalCrlSource.fetch(x509Certificate)) + checkApprovedCRLs(x509Certificate, getCRLs(x509Certificate)) + } + + @Suppress("TooGenericExceptionCaught") + private fun getCRLs(cert: X509Certificate): Set { + val crls = try { + crlSource.fetch(cert) + } catch (e: Exception) { + if (softFail) { + addSoftFailException(e) + return emptySet() + } else { + throw undeterminedRevocationException("Unable to retrieve CRLs", e) + } + } + if (crls.isNotEmpty() || softFail) { + return crls + } + // Note, the JDK tries to find a valid CRL from a different signing key before giving up (RevocationChecker.verifyWithSeparateSigningKey) + throw undeterminedRevocationException("Could not find any valid CRLs", null) } /** @@ -47,11 +70,11 @@ class ExternalSourceRevocationChecker(private val externalCrlSource: ExternalCrl * 5.3 of RFC 5280). */ val unresCritExts = entry.criticalExtensionOIDs - if (unresCritExts != null && !unresCritExts.isEmpty()) { + if (unresCritExts != null && unresCritExts.isNotEmpty()) { /* remove any that we will process */ unresCritExts.remove(Extension.cRLDistributionPoints.id) unresCritExts.remove(Extension.certificateIssuer.id) - if (!unresCritExts.isEmpty()) { + if (unresCritExts.isNotEmpty()) { throw CertPathValidatorException( "Unrecognized critical extension(s) in revoked CRL entry: $unresCritExts") } @@ -64,14 +87,22 @@ class ExternalSourceRevocationChecker(private val externalCrlSource: ExternalCrl revocationDate, reasonCode, crl.issuerX500Principal, mutableMapOf()) throw CertPathValidatorException( - t.message, t, null, -1, CertPathValidatorException.BasicReason.REVOKED) + t.message, t, null, -1, BasicReason.REVOKED) } } } } + /** + * This is set to false intentionally for security reasons. + * It ensures that certificates are provided in reverse direction (from most-trusted CA to target certificate) + * after the necessary validation checks have already been performed. + * + * If that wasn't the case, we could be reaching out to CRL endpoints for invalid certificates, which would open security holes + * e.g. systems that are not part of a Corda network could force a Corda firewall to initiate outbound requests to systems under their control. + */ override fun isForwardCheckingSupported(): Boolean { - return true + return false } override fun getSupportedExtensions(): MutableSet? { @@ -79,10 +110,19 @@ class ExternalSourceRevocationChecker(private val externalCrlSource: ExternalCrl } override fun init(forward: Boolean) { - // Nothing to do + softFailExceptions.clear() } override fun getSoftFailExceptions(): MutableList { - return LinkedList() + return Collections.unmodifiableList(softFailExceptions) + } + + private fun addSoftFailException(e: Exception) { + logger.debug("Soft fail exception", e) + softFailExceptions += undeterminedRevocationException(e.message, e) + } + + private fun undeterminedRevocationException(message: String?, cause: Throwable?): CertPathValidatorException { + return CertPathValidatorException(message, cause, null, -1, BasicReason.UNDETERMINED_REVOCATION_STATUS) } } \ No newline at end of file diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelperTest.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelperTest.kt index 782d8b8abe..8fbe5fd9d9 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelperTest.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelperTest.kt @@ -28,7 +28,7 @@ class SSLHelperTest { val trustStore = sslConfig.trustStore trustManagerFactory.init(initialiseTrustStoreAndEnableCrlChecking(CertificateStore.fromFile(trustStore.path, trustStore.storePassword, trustStore.entryPassword, false), RevocationConfigImpl(RevocationConfig.Mode.HARD_FAIL))) - val sslHandler = createClientSslHelper(NetworkHostAndPort("localhost", 1234), setOf(legalName), keyManagerFactory, trustManagerFactory) + val sslHandler = createClientSslHandler(NetworkHostAndPort("localhost", 1234), setOf(legalName), keyManagerFactory, trustManagerFactory) val legalNameHash = SecureHash.sha256(legalName.toString()).toString().take(32).toLowerCase() // These hardcoded values must not be changed, something is broken if you have to change these hardcoded values. diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/revocation/CertDistPointCrlSourceTest.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/revocation/CertDistPointCrlSourceTest.kt new file mode 100644 index 0000000000..489c66de32 --- /dev/null +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/revocation/CertDistPointCrlSourceTest.kt @@ -0,0 +1,51 @@ +package net.corda.nodeapi.internal.revocation + +import net.corda.core.crypto.Crypto +import net.corda.core.utilities.NetworkHostAndPort +import net.corda.nodeapi.internal.createDevNodeCa +import net.corda.testing.core.ALICE_NAME +import net.corda.testing.node.internal.network.CrlServer +import org.assertj.core.api.Assertions.assertThat +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.junit.After +import org.junit.Before +import org.junit.Test +import java.math.BigInteger + +class CertDistPointCrlSourceTest { + private lateinit var crlServer: CrlServer + + @Before + fun setUp() { + // Do not use Security.addProvider(BouncyCastleProvider()) to avoid EdDSA signature disruption in other tests. + Crypto.findProvider(BouncyCastleProvider.PROVIDER_NAME) + crlServer = CrlServer(NetworkHostAndPort("localhost", 0)) + crlServer.start() + } + + @After + fun tearDown() { + if (::crlServer.isInitialized) { + crlServer.close() + } + } + + @Test(timeout=300_000) + fun `happy path`() { + val crlSource = CertDistPointCrlSource() + + with(crlSource.fetch(crlServer.intermediateCa.certificate)) { + assertThat(size).isEqualTo(1) + assertThat(single().revokedCertificates).isNull() + } + + val nodeCaCert = crlServer.replaceNodeCertDistPoint(createDevNodeCa(crlServer.intermediateCa, ALICE_NAME).certificate) + + crlServer.revokedNodeCerts += listOf(BigInteger.ONE, BigInteger.TEN) + with(crlSource.fetch(nodeCaCert)) { // Use a different cert to avoid the cache + assertThat(size).isEqualTo(1) + val revokedCertificates = single().revokedCertificates + assertThat(revokedCertificates.map { it.serialNumber }).containsExactlyInAnyOrder(BigInteger.ONE, BigInteger.TEN) + } + } +} diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/revocation/ExternalSourceRevocationCheckerTest.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/revocation/CordaRevocationCheckerTest.kt similarity index 66% rename from node-api/src/test/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/revocation/ExternalSourceRevocationCheckerTest.kt rename to node-api/src/test/kotlin/net/corda/nodeapi/internal/revocation/CordaRevocationCheckerTest.kt index 7be350a525..6ccf8ce594 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/revocation/ExternalSourceRevocationCheckerTest.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/revocation/CordaRevocationCheckerTest.kt @@ -1,26 +1,27 @@ -package net.corda.nodeapi.internal.protonwrapper.netty.revocation +package net.corda.nodeapi.internal.revocation import net.corda.core.utilities.Try import net.corda.nodeapi.internal.DEV_CA_KEY_STORE_PASS import net.corda.nodeapi.internal.DEV_CA_PRIVATE_KEY_PASS import net.corda.nodeapi.internal.config.CertificateStore import net.corda.nodeapi.internal.crypto.X509Utilities -import net.corda.nodeapi.internal.protonwrapper.netty.ExternalCrlSource +import net.corda.nodeapi.internal.protonwrapper.netty.CrlSource import org.bouncycastle.jcajce.provider.asymmetric.x509.CertificateFactory import org.junit.Test import java.math.BigInteger - import java.security.cert.X509CRL import java.security.cert.X509Certificate -import java.sql.Date +import java.time.LocalDate +import java.time.ZoneOffset +import java.util.* import kotlin.test.assertEquals import kotlin.test.assertTrue -class ExternalSourceRevocationCheckerTest { +class CordaRevocationCheckerTest { @Test(timeout=300_000) fun checkRevoked() { - val checkResult = performCheckOnDate(Date.valueOf("2019-09-27")) + val checkResult = performCheckOnDate(LocalDate.of(2019, 9, 27)) val failedChecks = checkResult.filterNot { it.second.isSuccess } assertEquals(1, failedChecks.size) assertEquals(BigInteger.valueOf(8310484079152632582), failedChecks.first().first.serialNumber) @@ -28,11 +29,11 @@ class ExternalSourceRevocationCheckerTest { @Test(timeout=300_000) fun checkTooEarly() { - val checkResult = performCheckOnDate(Date.valueOf("2019-08-27")) + val checkResult = performCheckOnDate(LocalDate.of(2019, 8, 27)) assertTrue(checkResult.all { it.second.isSuccess }) } - private fun performCheckOnDate(date: Date): List>> { + private fun performCheckOnDate(date: LocalDate): List>> { val certStore = CertificateStore.fromResource( "net/corda/nodeapi/internal/protonwrapper/netty/sslkeystore_Revoked.jks", DEV_CA_KEY_STORE_PASS, DEV_CA_PRIVATE_KEY_PASS) @@ -40,16 +41,17 @@ class ExternalSourceRevocationCheckerTest { val resourceAsStream = javaClass.getResourceAsStream("/net/corda/nodeapi/internal/protonwrapper/netty/doorman.crl") val crl = CertificateFactory().engineGenerateCRL(resourceAsStream) as X509CRL - //val crlHolder = X509CRLHolder(resourceAsStream) - //crlHolder.revokedCertificates as X509CRLEntryHolder - - val instance = ExternalSourceRevocationChecker(object : ExternalCrlSource { + val crlSource = object : CrlSource { override fun fetch(certificate: X509Certificate): Set = setOf(crl) - }) { date } + } + val checker = CordaRevocationChecker(crlSource, + softFail = true, + dateSource = { Date.from(date.atStartOfDay().toInstant(ZoneOffset.UTC)) } + ) return certStore.query { getCertificateChain(X509Utilities.CORDA_CLIENT_TLS).map { - Pair(it, Try.on { instance.check(it, mutableListOf()) }) + Pair(it, Try.on { checker.check(it, mutableListOf()) }) } } } diff --git a/node/build.gradle b/node/build.gradle index c788acb66c..2f2ba885ad 100644 --- a/node/build.gradle +++ b/node/build.gradle @@ -270,6 +270,8 @@ tasks.register('integrationTest', Test) { testClassesDirs = sourceSets.integrationTest.output.classesDirs classpath = sourceSets.integrationTest.runtimeClasspath maxParallelForks = (System.env.CORDA_NODE_INT_TESTING_FORKS == null) ? 1 : "$System.env.CORDA_NODE_INT_TESTING_FORKS".toInteger() + // CertificateRevocationListNodeTests + systemProperty 'net.corda.dpcrl.connect.timeout', '4000' } tasks.register('slowIntegrationTest', Test) { diff --git a/node/src/integration-test/kotlin/net/corda/node/amqp/AMQPClientSslErrorsTest.kt b/node/src/integration-test/kotlin/net/corda/node/amqp/AMQPClientSslErrorsTest.kt index dd85bae8e7..f3fb153013 100644 --- a/node/src/integration-test/kotlin/net/corda/node/amqp/AMQPClientSslErrorsTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/amqp/AMQPClientSslErrorsTest.kt @@ -28,6 +28,7 @@ import org.junit.Test import org.junit.rules.TemporaryFolder import org.junit.runner.RunWith import org.junit.runners.Parameterized +import java.time.Duration import javax.net.ssl.KeyManagerFactory import javax.net.ssl.TrustManagerFactory import kotlin.test.assertEquals @@ -125,7 +126,7 @@ class AMQPClientSslErrorsTest(@Suppress("unused") private val iteration: Int) { override val keyStore = keyStore override val trustStore = clientConfig.p2pSslOptions.trustStore.get() override val maxMessageSize: Int = MAX_MESSAGE_SIZE - override val sslHandshakeTimeout: Long = 3000 + override val sslHandshakeTimeout: Duration = 3.seconds } clientKeyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()) diff --git a/node/src/integration-test/kotlin/net/corda/node/amqp/CertificateRevocationListNodeTests.kt b/node/src/integration-test/kotlin/net/corda/node/amqp/CertificateRevocationListNodeTests.kt index 97a2795df6..36b1a14ae1 100644 --- a/node/src/integration-test/kotlin/net/corda/node/amqp/CertificateRevocationListNodeTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/amqp/CertificateRevocationListNodeTests.kt @@ -5,67 +5,49 @@ import com.nhaarman.mockito_kotlin.whenever import net.corda.core.crypto.Crypto import net.corda.core.identity.CordaX500Name import net.corda.core.internal.div +import net.corda.core.internal.rootCause +import net.corda.core.internal.times import net.corda.core.toFuture import net.corda.core.utilities.NetworkHostAndPort -import net.corda.core.utilities.days -import net.corda.core.utilities.minutes -import net.corda.core.utilities.seconds +import net.corda.coretesting.internal.rigorousMock +import net.corda.coretesting.internal.stubs.CertificateStoreStubs import net.corda.node.services.config.NodeConfiguration import net.corda.node.services.config.configureWithDevSSLCertificate +import net.corda.node.services.messaging.ArtemisMessagingServer +import net.corda.nodeapi.internal.ArtemisMessagingClient import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.P2P_PREFIX import net.corda.nodeapi.internal.config.CertificateStoreSupplier import net.corda.nodeapi.internal.config.MutualSslConfiguration -import net.corda.nodeapi.internal.crypto.* +import net.corda.nodeapi.internal.crypto.X509Utilities import net.corda.nodeapi.internal.protonwrapper.messages.MessageStatus import net.corda.nodeapi.internal.protonwrapper.netty.AMQPClient import net.corda.nodeapi.internal.protonwrapper.netty.AMQPConfiguration import net.corda.nodeapi.internal.protonwrapper.netty.AMQPServer +import net.corda.nodeapi.internal.protonwrapper.netty.toRevocationConfig import net.corda.testing.core.ALICE_NAME import net.corda.testing.core.BOB_NAME import net.corda.testing.core.CHARLIE_NAME import net.corda.testing.core.MAX_MESSAGE_SIZE import net.corda.testing.driver.internal.incrementalPortAllocation -import net.corda.coretesting.internal.DEV_INTERMEDIATE_CA -import net.corda.coretesting.internal.DEV_ROOT_CA -import net.corda.coretesting.internal.rigorousMock -import net.corda.coretesting.internal.stubs.CertificateStoreStubs -import net.corda.node.services.messaging.ArtemisMessagingServer -import net.corda.nodeapi.internal.ArtemisMessagingClient -import net.corda.nodeapi.internal.protonwrapper.netty.toRevocationConfig +import net.corda.testing.node.internal.network.CrlServer +import net.corda.testing.node.internal.network.CrlServer.Companion.EMPTY_CRL +import net.corda.testing.node.internal.network.CrlServer.Companion.FORBIDDEN_CRL +import net.corda.testing.node.internal.network.CrlServer.Companion.NODE_CRL +import net.corda.testing.node.internal.network.CrlServer.Companion.withCrlDistPoint import org.apache.activemq.artemis.api.core.QueueConfiguration import org.apache.activemq.artemis.api.core.RoutingType +import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatIllegalArgumentException -import org.bouncycastle.asn1.x500.X500Name -import org.bouncycastle.asn1.x509.* -import org.bouncycastle.cert.jcajce.JcaX509CRLConverter -import org.bouncycastle.cert.jcajce.JcaX509ExtensionUtils -import org.bouncycastle.cert.jcajce.JcaX509v2CRLBuilder import org.bouncycastle.jce.provider.BouncyCastleProvider -import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder -import org.eclipse.jetty.server.Server -import org.eclipse.jetty.server.ServerConnector -import org.eclipse.jetty.server.handler.HandlerCollection -import org.eclipse.jetty.servlet.ServletContextHandler -import org.eclipse.jetty.servlet.ServletHolder -import org.glassfish.jersey.server.ResourceConfig -import org.glassfish.jersey.servlet.ServletContainer import org.junit.After import org.junit.Before import org.junit.Rule import org.junit.Test import org.junit.rules.TemporaryFolder -import java.io.Closeable -import java.math.BigInteger -import java.net.InetSocketAddress -import java.security.KeyPair -import java.security.PrivateKey -import java.security.cert.X509CRL +import java.net.SocketTimeoutException import java.security.cert.X509Certificate -import java.util.* -import javax.ws.rs.GET -import javax.ws.rs.Path -import javax.ws.rs.Produces -import javax.ws.rs.core.Response +import java.time.Duration +import java.util.concurrent.atomic.AtomicInteger import kotlin.test.assertEquals class CertificateRevocationListNodeTests { @@ -73,45 +55,27 @@ class CertificateRevocationListNodeTests { @JvmField val temporaryFolder = TemporaryFolder() - private val ROOT_CA = DEV_ROOT_CA - private lateinit var INTERMEDIATE_CA: CertificateAndKeyPair - private val portAllocation = incrementalPortAllocation() private val serverPort = portAllocation.nextPort() - private lateinit var server: CrlServer - - private val revokedNodeCerts: MutableList = mutableListOf() - private val revokedIntermediateCerts: MutableList = mutableListOf() + private lateinit var crlServer: CrlServer + private lateinit var amqpServer: AMQPServer + private lateinit var amqpClient: AMQPClient private abstract class AbstractNodeConfiguration : NodeConfiguration companion object { + private val unreachableIpCounter = AtomicInteger(1) - const val FORBIDDEN_CRL = "forbidden.crl" + private val crlConnectTimeout = Duration.ofMillis(System.getProperty("net.corda.dpcrl.connect.timeout").toLong()) - fun createRevocationList(clrServer: CrlServer, signatureAlgorithm: String, caCertificate: X509Certificate, - caPrivateKey: PrivateKey, - endpoint: String, - indirect: Boolean, - vararg serialNumbers: BigInteger): X509CRL { - println("Generating CRL for $endpoint") - val builder = JcaX509v2CRLBuilder(caCertificate.subjectX500Principal, Date(System.currentTimeMillis() - 1.minutes.toMillis())) - val extensionUtils = JcaX509ExtensionUtils() - builder.addExtension(Extension.authorityKeyIdentifier, - false, extensionUtils.createAuthorityKeyIdentifier(caCertificate)) - val issuingDistPointName = GeneralName( - GeneralName.uniformResourceIdentifier, - "http://${clrServer.hostAndPort.host}:${clrServer.hostAndPort.port}/crl/$endpoint") - // This is required and needs to match the certificate settings with respect to being indirect - val issuingDistPoint = IssuingDistributionPoint(DistributionPointName(GeneralNames(issuingDistPointName)), indirect, false) - builder.addExtension(Extension.issuingDistributionPoint, true, issuingDistPoint) - builder.setNextUpdate(Date(System.currentTimeMillis() + 1.seconds.toMillis())) - serialNumbers.forEach { - builder.addCRLEntry(it, Date(System.currentTimeMillis() - 10.minutes.toMillis()), ReasonFlags.certificateHold) - } - val signer = JcaContentSignerBuilder(signatureAlgorithm).setProvider(Crypto.findProvider("BC")).build(caPrivateKey) - return JcaX509CRLConverter().setProvider(Crypto.findProvider("BC")).getCRL(builder.build(signer)) + /** + * Use this method to get a unqiue unreachable IP address. Subsequent uses of the same IP for connection timeout testing purposes + * may not work as the OS process may cache the timeout result. + */ + private fun newUnreachableIpAddress(): String { + check(unreachableIpCounter.get() != 255) + return "10.255.255.${unreachableIpCounter.getAndIncrement()}" } } @@ -119,253 +83,253 @@ class CertificateRevocationListNodeTests { fun setUp() { // Do not use Security.addProvider(BouncyCastleProvider()) to avoid EdDSA signature disruption in other tests. Crypto.findProvider(BouncyCastleProvider.PROVIDER_NAME) - revokedNodeCerts.clear() - server = CrlServer(NetworkHostAndPort("localhost", 0)) - server.start() - INTERMEDIATE_CA = CertificateAndKeyPair(replaceCrlDistPointCaCertificate( - DEV_INTERMEDIATE_CA.certificate, - CertificateType.INTERMEDIATE_CA, - ROOT_CA.keyPair, - "http://${server.hostAndPort}/crl/intermediate.crl"), DEV_INTERMEDIATE_CA.keyPair) + crlServer = CrlServer(NetworkHostAndPort("localhost", 0)) + crlServer.start() } @After fun tearDown() { - server.close() - revokedNodeCerts.clear() - } - - @Test(timeout=300_000) - fun `Simple AMPQ Client to Server connection works and soft fail is enabled`() { - val crlCheckSoftFail = true - val (amqpServer, _) = createServer(serverPort, crlCheckSoftFail = crlCheckSoftFail) - amqpServer.use { - amqpServer.start() - val receiveSubs = amqpServer.onReceive.subscribe { - assertEquals(BOB_NAME.toString(), it.sourceLegalName) - assertEquals(P2P_PREFIX + "Test", it.topic) - assertEquals("Test", String(it.payload)) - it.complete(true) - } - val (amqpClient, _) = createClient(serverPort, crlCheckSoftFail) - amqpClient.use { - val serverConnected = amqpServer.onConnection.toFuture() - val clientConnected = amqpClient.onConnection.toFuture() - amqpClient.start() - val serverConnect = serverConnected.get() - assertEquals(true, serverConnect.connected) - val clientConnect = clientConnected.get() - assertEquals(true, clientConnect.connected) - val msg = amqpClient.createMessage("Test".toByteArray(), - P2P_PREFIX + "Test", - ALICE_NAME.toString(), - emptyMap()) - amqpClient.write(msg) - assertEquals(MessageStatus.Acknowledged, msg.onComplete.get()) - receiveSubs.unsubscribe() - } + if (::amqpClient.isInitialized) { + amqpClient.close() + } + if (::amqpServer.isInitialized) { + amqpServer.close() + } + if (::crlServer.isInitialized) { + crlServer.close() } } @Test(timeout=300_000) - fun `Simple AMPQ Client to Server connection works and soft fail is disabled`() { - val crlCheckSoftFail = false - val (amqpServer, _) = createServer(serverPort, crlCheckSoftFail = crlCheckSoftFail) - amqpServer.use { - amqpServer.start() - val receiveSubs = amqpServer.onReceive.subscribe { - assertEquals(BOB_NAME.toString(), it.sourceLegalName) - assertEquals(P2P_PREFIX + "Test", it.topic) - assertEquals("Test", String(it.payload)) - it.complete(true) - } - val (amqpClient, _) = createClient(serverPort, crlCheckSoftFail) - amqpClient.use { - val serverConnected = amqpServer.onConnection.toFuture() - val clientConnected = amqpClient.onConnection.toFuture() - amqpClient.start() - val serverConnect = serverConnected.get() - assertEquals(true, serverConnect.connected) - val clientConnect = clientConnected.get() - assertEquals(true, clientConnect.connected) - val msg = amqpClient.createMessage("Test".toByteArray(), - P2P_PREFIX + "Test", - ALICE_NAME.toString(), - emptyMap()) - amqpClient.write(msg) - assertEquals(MessageStatus.Acknowledged, msg.onComplete.get()) - receiveSubs.unsubscribe() - } - } + fun `AMQP server connection works and soft fail is enabled`() { + verifyAMQPConnection( + crlCheckSoftFail = true, + expectedConnectStatus = true + ) } @Test(timeout=300_000) - fun `AMPQ Client to Server connection fails when client's certificate is revoked and soft fail is enabled`() { - val crlCheckSoftFail = true - val (amqpServer, _) = createServer(serverPort, crlCheckSoftFail = crlCheckSoftFail) - amqpServer.use { - amqpServer.start() - amqpServer.onReceive.subscribe { - it.complete(true) - } - val (amqpClient, clientCert) = createClient(serverPort, crlCheckSoftFail) - revokedNodeCerts.add(clientCert.serialNumber) - amqpClient.use { - val serverConnected = amqpServer.onConnection.toFuture() - amqpClient.onConnection.toFuture() - amqpClient.start() - val serverConnect = serverConnected.get() - assertEquals(false, serverConnect.connected) - } - } + fun `AMQP server connection works and soft fail is disabled`() { + verifyAMQPConnection( + crlCheckSoftFail = false, + expectedConnectStatus = true + ) } @Test(timeout=300_000) - fun `AMPQ Client to Server connection fails when client's certificate is revoked and soft fail is disabled`() { - val crlCheckSoftFail = false - val (amqpServer, _) = createServer(serverPort, crlCheckSoftFail = crlCheckSoftFail) - amqpServer.use { - amqpServer.start() - amqpServer.onReceive.subscribe { - it.complete(true) - } - val (amqpClient, clientCert) = createClient(serverPort, crlCheckSoftFail) - revokedNodeCerts.add(clientCert.serialNumber) - amqpClient.use { - val serverConnected = amqpServer.onConnection.toFuture() - amqpClient.onConnection.toFuture() - amqpClient.start() - val serverConnect = serverConnected.get() - assertEquals(false, serverConnect.connected) - } - } + fun `AMQP server connection fails when client's certificate is revoked and soft fail is enabled`() { + verifyAMQPConnection( + crlCheckSoftFail = true, + revokeClientCert = true, + expectedConnectStatus = false + ) } @Test(timeout=300_000) - fun `AMPQ Client to Server connection fails when servers's certificate is revoked`() { - val crlCheckSoftFail = true - val (amqpServer, serverCert) = createServer(serverPort, crlCheckSoftFail = crlCheckSoftFail) - revokedNodeCerts.add(serverCert.serialNumber) - amqpServer.use { - amqpServer.start() - amqpServer.onReceive.subscribe { - it.complete(true) - } - val (amqpClient, _) = createClient(serverPort, crlCheckSoftFail) - amqpClient.use { - val serverConnected = amqpServer.onConnection.toFuture() - amqpClient.onConnection.toFuture() - amqpClient.start() - val serverConnect = serverConnected.get() - assertEquals(false, serverConnect.connected) - } - } + fun `AMQP server connection fails when client's certificate is revoked and soft fail is disabled`() { + verifyAMQPConnection( + crlCheckSoftFail = false, + revokeClientCert = true, + expectedConnectStatus = false + ) } @Test(timeout=300_000) - fun `AMPQ Client to Server connection fails when servers's certificate is revoked and soft fail is enabled`() { - val crlCheckSoftFail = true - val (amqpServer, serverCert) = createServer(serverPort, crlCheckSoftFail = crlCheckSoftFail) - revokedNodeCerts.add(serverCert.serialNumber) - amqpServer.use { - amqpServer.start() - amqpServer.onReceive.subscribe { - it.complete(true) - } - val (amqpClient, _) = createClient(serverPort, crlCheckSoftFail) - amqpClient.use { - val serverConnected = amqpServer.onConnection.toFuture() - amqpClient.onConnection.toFuture() - amqpClient.start() - val serverConnect = serverConnected.get() - assertEquals(false, serverConnect.connected) - } - } + fun `AMQP server connection fails when servers's certificate is revoked and soft fail is enabled`() { + verifyAMQPConnection( + crlCheckSoftFail = true, + revokeServerCert = true, + expectedConnectStatus = false + ) } @Test(timeout=300_000) - fun `AMPQ Client to Server connection succeeds when CRL cannot be obtained and soft fail is enabled`() { - val crlCheckSoftFail = true - val (amqpServer, _) = createServer( - serverPort, - crlCheckSoftFail = crlCheckSoftFail, - nodeCrlDistPoint = "http://${server.hostAndPort}/crl/invalid.crl") - amqpServer.use { - amqpServer.start() - amqpServer.onReceive.subscribe { - it.complete(true) - } - val (amqpClient, _) = createClient( - serverPort, - crlCheckSoftFail, - nodeCrlDistPoint = "http://${server.hostAndPort}/crl/invalid.crl") - amqpClient.use { - val serverConnected = amqpServer.onConnection.toFuture() - amqpClient.onConnection.toFuture() - amqpClient.start() - val serverConnect = serverConnected.get() - assertEquals(true, serverConnect.connected) - } - } + fun `AMQP server connection fails when servers's certificate is revoked and soft fail is disabled`() { + verifyAMQPConnection( + crlCheckSoftFail = false, + revokeServerCert = true, + expectedConnectStatus = false + ) } @Test(timeout=300_000) - fun `Revocation status check fails when the CRL distribution point is not set and soft fail is disabled`() { - val crlCheckSoftFail = false - val (amqpServer, _) = createServer( - serverPort, - crlCheckSoftFail = crlCheckSoftFail, - tlsCrlDistPoint = null) - amqpServer.use { - amqpServer.start() - amqpServer.onReceive.subscribe { - it.complete(true) - } - val (amqpClient, _) = createClient( - serverPort, - crlCheckSoftFail, - tlsCrlDistPoint = null) - amqpClient.use { - val serverConnected = amqpServer.onConnection.toFuture() - amqpClient.onConnection.toFuture() - amqpClient.start() - val serverConnect = serverConnected.get() - assertEquals(false, serverConnect.connected) - } - } + fun `AMQP server connection succeeds when CRL cannot be obtained and soft fail is enabled`() { + verifyAMQPConnection( + crlCheckSoftFail = true, + nodeCrlDistPoint = "http://${crlServer.hostAndPort}/crl/invalid.crl", + expectedConnectStatus = true + ) } @Test(timeout=300_000) - fun `Revocation status chceck succeds when the CRL distribution point is not set and soft fail is enabled`() { - val crlCheckSoftFail = true - val (amqpServer, _) = createServer( - serverPort, - crlCheckSoftFail = crlCheckSoftFail, - tlsCrlDistPoint = null) - amqpServer.use { - amqpServer.start() - amqpServer.onReceive.subscribe { - it.complete(true) - } - val (amqpClient, _) = createClient( - serverPort, - crlCheckSoftFail, - tlsCrlDistPoint = null) - amqpClient.use { - val serverConnected = amqpServer.onConnection.toFuture() - amqpClient.onConnection.toFuture() - amqpClient.start() - val serverConnect = serverConnected.get() - assertEquals(true, serverConnect.connected) - } - } + fun `AMQP server connection fails when CRL cannot be obtained and soft fail is disabled`() { + verifyAMQPConnection( + crlCheckSoftFail = false, + nodeCrlDistPoint = "http://${crlServer.hostAndPort}/crl/invalid.crl", + expectedConnectStatus = false + ) } - private fun createClient(targetPort: Int, - crlCheckSoftFail: Boolean, - nodeCrlDistPoint: String = "http://${server.hostAndPort}/crl/node.crl", - tlsCrlDistPoint: String? = "http://${server.hostAndPort}/crl/empty.crl", - maxMessageSize: Int = MAX_MESSAGE_SIZE): Pair { + @Test(timeout=300_000) + fun `AMQP server connection succeeds when CRL is not defined and soft fail is enabled`() { + verifyAMQPConnection( + crlCheckSoftFail = true, + nodeCrlDistPoint = null, + expectedConnectStatus = true + ) + } + + @Test(timeout=300_000) + fun `AMQP server connection fails when CRL is not defined and soft fail is disabled`() { + verifyAMQPConnection( + crlCheckSoftFail = false, + nodeCrlDistPoint = null, + expectedConnectStatus = false + ) + } + + @Test(timeout=300_000) + fun `AMQP server connection succeeds when CRL retrieval is forbidden and soft fail is enabled`() { + verifyAMQPConnection( + crlCheckSoftFail = true, + nodeCrlDistPoint = "http://${crlServer.hostAndPort}/crl/$FORBIDDEN_CRL", + expectedConnectStatus = true + ) + } + + @Test(timeout=300_000) + fun `AMQP server connection succeeds when CRL endpoint is unreachable, soft fail is enabled and CRL timeouts are within SSL handshake timeout`() { + verifyAMQPConnection( + crlCheckSoftFail = true, + nodeCrlDistPoint = "http://${newUnreachableIpAddress()}/crl/unreachable.crl", + sslHandshakeTimeout = crlConnectTimeout * 3, + expectedConnectStatus = true + ) + val timeoutExceptions = (amqpServer.softFailExceptions + amqpClient.softFailExceptions) + .map { it.rootCause } + .filterIsInstance() + assertThat(timeoutExceptions).isNotEmpty + } + + @Test(timeout=300_000) + fun `AMQP server connection fails when CRL endpoint is unreachable, despite soft fail enabled, when CRL timeouts are not within SSL handshake timeout`() { + verifyAMQPConnection( + crlCheckSoftFail = true, + nodeCrlDistPoint = "http://${newUnreachableIpAddress()}/crl/unreachable.crl", + sslHandshakeTimeout = crlConnectTimeout / 2, + expectedConnectStatus = false + ) + } + + @Test(timeout=300_000) + fun `verify CRL algorithms`() { + val crl = crlServer.createRevocationList( + "SHA256withECDSA", + crlServer.rootCa, + EMPTY_CRL, + true, + emptyList() + ) + // This should pass. + crl.verify(crlServer.rootCa.keyPair.public) + + // Try changing the algorithm to EC will fail. + assertThatIllegalArgumentException().isThrownBy { + crlServer.createRevocationList( + "EC", + crlServer.rootCa, + EMPTY_CRL, + true, + emptyList() + ) + }.withMessage("Unknown signature type requested: EC") + } + + @Test(timeout = 300_000) + fun `Artemis server connection succeeds with soft fail CRL check`() { + verifyArtemisConnection( + crlCheckSoftFail = true, + crlCheckArtemisServer = true, + expectedStatus = MessageStatus.Acknowledged + ) + } + + @Test(timeout = 300_000) + fun `Artemis server connection succeeds with hard fail CRL check`() { + verifyArtemisConnection( + crlCheckSoftFail = false, + crlCheckArtemisServer = true, + expectedStatus = MessageStatus.Acknowledged + ) + } + + @Test(timeout = 300_000) + fun `Artemis server connection succeeds with soft fail CRL check on unavailable URL`() { + verifyArtemisConnection( + crlCheckSoftFail = true, + crlCheckArtemisServer = true, + expectedStatus = MessageStatus.Acknowledged, + nodeCrlDistPoint = "http://${crlServer.hostAndPort}/crl/$FORBIDDEN_CRL" + ) + } + + @Test(timeout = 300_000) + fun `Artemis server connection succeeds with soft fail CRL check on unreachable URL if CRL timeout is within SSL handshake timeout`() { + verifyArtemisConnection( + crlCheckSoftFail = true, + crlCheckArtemisServer = true, + expectedStatus = MessageStatus.Acknowledged, + nodeCrlDistPoint = "http://${newUnreachableIpAddress()}/crl/unreachable.crl", + sslHandshakeTimeout = crlConnectTimeout * 3 + ) + } + + @Test(timeout = 300_000) + fun `Artemis server connection fails with soft fail CRL check on unreachable URL if CRL timeout is not within SSL handshake timeout`() { + verifyArtemisConnection( + crlCheckSoftFail = true, + crlCheckArtemisServer = true, + expectedConnected = false, + nodeCrlDistPoint = "http://${newUnreachableIpAddress()}/crl/unreachable.crl", + sslHandshakeTimeout = crlConnectTimeout / 2 + ) + } + + @Test(timeout = 300_000) + fun `Artemis server connection fails with hard fail CRL check on unavailable URL`() { + verifyArtemisConnection( + crlCheckSoftFail = false, + crlCheckArtemisServer = true, + expectedStatus = MessageStatus.Rejected, + nodeCrlDistPoint = "http://${crlServer.hostAndPort}/crl/$FORBIDDEN_CRL" + ) + } + + @Test(timeout = 300_000) + fun `Artemis server connection fails with soft fail CRL check on revoked node certificate`() { + verifyArtemisConnection( + crlCheckSoftFail = true, + crlCheckArtemisServer = true, + expectedStatus = MessageStatus.Rejected, + revokedNodeCert = true + ) + } + + @Test(timeout = 300_000) + fun `Artemis server connection succeeds with disabled CRL check on revoked node certificate`() { + verifyArtemisConnection( + crlCheckSoftFail = false, + crlCheckArtemisServer = false, + expectedStatus = MessageStatus.Acknowledged, + revokedNodeCert = true + ) + } + + private fun createAMQPClient(targetPort: Int, + crlCheckSoftFail: Boolean, + nodeCrlDistPoint: String? = "http://${crlServer.hostAndPort}/crl/$NODE_CRL", + tlsCrlDistPoint: String? = "http://${crlServer.hostAndPort}/crl/$EMPTY_CRL", + maxMessageSize: Int = MAX_MESSAGE_SIZE): X509Certificate { val baseDirectory = temporaryFolder.root.toPath() / "client" val certificatesDirectory = baseDirectory / "certificates" val p2pSslConfiguration = CertificateStoreStubs.P2P.withCertificatesDirectory(certificatesDirectory) @@ -379,7 +343,7 @@ class CertificateRevocationListNodeTests { doReturn(crlCheckSoftFail).whenever(it).crlCheckSoftFail } clientConfig.configureWithDevSSLCertificate() - val nodeCert = (signingCertificateStore to p2pSslConfiguration).recreateNodeCaAndTlsCertificates(nodeCrlDistPoint, tlsCrlDistPoint) + val nodeCert = recreateNodeCaAndTlsCertificates(signingCertificateStore, p2pSslConfiguration, nodeCrlDistPoint, tlsCrlDistPoint) val keyStore = clientConfig.p2pSslOptions.keyStore.get() val amqpConfig = object : AMQPConfiguration { @@ -387,17 +351,20 @@ class CertificateRevocationListNodeTests { override val trustStore = clientConfig.p2pSslOptions.trustStore.get() override val maxMessageSize: Int = maxMessageSize } - return Pair(AMQPClient( - listOf(NetworkHostAndPort("localhost", targetPort)), - setOf(ALICE_NAME, CHARLIE_NAME), - amqpConfig), nodeCert) + amqpClient = AMQPClient(listOf(NetworkHostAndPort("localhost", targetPort)), setOf(ALICE_NAME, CHARLIE_NAME), amqpConfig) + + return nodeCert } - private fun createServer(port: Int, name: CordaX500Name = ALICE_NAME, - crlCheckSoftFail: Boolean, - nodeCrlDistPoint: String = "http://${server.hostAndPort}/crl/node.crl", - tlsCrlDistPoint: String? = "http://${server.hostAndPort}/crl/empty.crl", - maxMessageSize: Int = MAX_MESSAGE_SIZE): Pair { + @Suppress("LongParameterList") + private fun createAMQPServer(port: Int, + name: CordaX500Name = ALICE_NAME, + crlCheckSoftFail: Boolean, + nodeCrlDistPoint: String? = "http://${crlServer.hostAndPort}/crl/$NODE_CRL", + tlsCrlDistPoint: String? = "http://${crlServer.hostAndPort}/crl/$EMPTY_CRL", + maxMessageSize: Int = MAX_MESSAGE_SIZE, + sslHandshakeTimeout: Duration? = null): X509Certificate { + check(!::amqpServer.isInitialized) val baseDirectory = temporaryFolder.root.toPath() / "server" val certificatesDirectory = baseDirectory / "certificates" val p2pSslConfiguration = CertificateStoreStubs.P2P.withCertificatesDirectory(certificatesDirectory) @@ -410,38 +377,41 @@ class CertificateRevocationListNodeTests { doReturn(signingCertificateStore).whenever(it).signingCertificateStore } serverConfig.configureWithDevSSLCertificate() - val nodeCert = (signingCertificateStore to p2pSslConfiguration).recreateNodeCaAndTlsCertificates(nodeCrlDistPoint, tlsCrlDistPoint) + val nodeCert = recreateNodeCaAndTlsCertificates(signingCertificateStore, p2pSslConfiguration, nodeCrlDistPoint, tlsCrlDistPoint) val keyStore = serverConfig.p2pSslOptions.keyStore.get() val amqpConfig = object : AMQPConfiguration { override val keyStore = keyStore override val trustStore = serverConfig.p2pSslOptions.trustStore.get() override val revocationConfig = crlCheckSoftFail.toRevocationConfig() override val maxMessageSize: Int = maxMessageSize + override val sslHandshakeTimeout: Duration = sslHandshakeTimeout ?: super.sslHandshakeTimeout } - return Pair(AMQPServer( - "0.0.0.0", - port, - amqpConfig), nodeCert) + amqpServer = AMQPServer("0.0.0.0", port, amqpConfig) + return nodeCert } - private fun Pair.recreateNodeCaAndTlsCertificates(nodeCaCrlDistPoint: String, tlsCrlDistPoint: String?): X509Certificate { - - val signingCertificateStore = first - val p2pSslConfiguration = second + private fun recreateNodeCaAndTlsCertificates(signingCertificateStore: CertificateStoreSupplier, + p2pSslConfiguration: MutualSslConfiguration, + nodeCaCrlDistPoint: String?, + tlsCrlDistPoint: String?): X509Certificate { val nodeKeyStore = signingCertificateStore.get() val (nodeCert, nodeKeys) = nodeKeyStore.query { getCertificateAndKeyPair(X509Utilities.CORDA_CLIENT_CA, nodeKeyStore.entryPassword) } - val newNodeCert = replaceCrlDistPointCaCertificate(nodeCert, CertificateType.NODE_CA, INTERMEDIATE_CA.keyPair, nodeCaCrlDistPoint) - val nodeCertChain = listOf(newNodeCert, INTERMEDIATE_CA.certificate, *nodeKeyStore.query { getCertificateChain(X509Utilities.CORDA_CLIENT_CA) }.drop(2).toTypedArray()) + val newNodeCert = crlServer.replaceNodeCertDistPoint(nodeCert, nodeCaCrlDistPoint) + val nodeCertChain = listOf(newNodeCert, crlServer.intermediateCa.certificate) + + nodeKeyStore.query { getCertificateChain(X509Utilities.CORDA_CLIENT_CA) }.drop(2) + nodeKeyStore.update { internal.deleteEntry(X509Utilities.CORDA_CLIENT_CA) } nodeKeyStore.update { setPrivateKey(X509Utilities.CORDA_CLIENT_CA, nodeKeys.private, nodeCertChain, nodeKeyStore.entryPassword) } + val sslKeyStore = p2pSslConfiguration.keyStore.get() val (tlsCert, tlsKeys) = sslKeyStore.query { getCertificateAndKeyPair(X509Utilities.CORDA_CLIENT_TLS, sslKeyStore.entryPassword) } - val newTlsCert = replaceCrlDistPointCaCertificate(tlsCert, CertificateType.TLS, nodeKeys, tlsCrlDistPoint, X500Name.getInstance(ROOT_CA.certificate.subjectX500Principal.encoded)) - val sslCertChain = listOf(newTlsCert, newNodeCert, INTERMEDIATE_CA.certificate, *sslKeyStore.query { getCertificateChain(X509Utilities.CORDA_CLIENT_TLS) }.drop(3).toTypedArray()) + val newTlsCert = tlsCert.withCrlDistPoint(nodeKeys, tlsCrlDistPoint, crlServer.rootCa.certificate.subjectX500Principal) + val sslCertChain = listOf(newTlsCert, newNodeCert, crlServer.intermediateCa.certificate) + + sslKeyStore.query { getCertificateChain(X509Utilities.CORDA_CLIENT_TLS) }.drop(3) sslKeyStore.update { internal.deleteEntry(X509Utilities.CORDA_CLIENT_TLS) @@ -452,203 +422,62 @@ class CertificateRevocationListNodeTests { return newNodeCert } - private fun replaceCrlDistPointCaCertificate(currentCaCert: X509Certificate, certType: CertificateType, issuerKeyPair: KeyPair, crlDistPoint: String?, crlIssuer: X500Name? = null): X509Certificate { - val signatureScheme = Crypto.findSignatureScheme(issuerKeyPair.private) - val provider = Crypto.findProvider(signatureScheme.providerName) - val issuerSigner = ContentSignerBuilder.build(signatureScheme, issuerKeyPair.private, provider) - val builder = X509Utilities.createPartialCertificate( - certType, - currentCaCert.issuerX500Principal, - issuerKeyPair.public, - currentCaCert.subjectX500Principal, - currentCaCert.publicKey, - Pair(Date(System.currentTimeMillis() - 5.minutes.toMillis()), Date(System.currentTimeMillis() + 10.days.toMillis())), - null - ) - crlDistPoint?.let { - val distPointName = DistributionPointName(GeneralNames(GeneralName(GeneralName.uniformResourceIdentifier, it))) - val crlIssuerGeneralNames = crlIssuer?.let { - GeneralNames(GeneralName(crlIssuer)) - } - val distPoint = DistributionPoint(distPointName, null, crlIssuerGeneralNames) - builder.addExtension(Extension.cRLDistributionPoints, false, CRLDistPoint(arrayOf(distPoint))) - } - return builder.build(issuerSigner).toJca() - } - - @Path("crl") - inner class CrlServlet(private val server: CrlServer) { - - private val SIGNATURE_ALGORITHM = "SHA256withECDSA" - private val NODE_CRL = "node.crl" - private val INTEMEDIATE_CRL = "intermediate.crl" - private val EMPTY_CRL = "empty.crl" - - @GET - @Path("node.crl") - @Produces("application/pkcs7-crl") - fun getNodeCRL(): Response { - return Response.ok(createRevocationList( - server, - SIGNATURE_ALGORITHM, - INTERMEDIATE_CA.certificate, - INTERMEDIATE_CA.keyPair.private, - NODE_CRL, - false, - *revokedNodeCerts.toTypedArray()).encoded) - .build() - } - - @GET - @Path(FORBIDDEN_CRL) - @Produces("application/pkcs7-crl") - fun getNodeSlowCRL(): Response { - return Response.status(Response.Status.FORBIDDEN).build() - } - - @GET - @Path("intermediate.crl") - @Produces("application/pkcs7-crl") - fun getIntermediateCRL(): Response { - return Response.ok(createRevocationList( - server, - SIGNATURE_ALGORITHM, - ROOT_CA.certificate, - ROOT_CA.keyPair.private, - INTEMEDIATE_CRL, - false, - *revokedIntermediateCerts.toTypedArray()).encoded) - .build() - } - - @GET - @Path("empty.crl") - @Produces("application/pkcs7-crl") - fun getEmptyCRL(): Response { - return Response.ok(createRevocationList( - server, - SIGNATURE_ALGORITHM, - ROOT_CA.certificate, - ROOT_CA.keyPair.private, - EMPTY_CRL, - true).encoded) - .build() - } - } - - inner class CrlServer(hostAndPort: NetworkHostAndPort) : Closeable { - - private val server: Server = Server(InetSocketAddress(hostAndPort.host, hostAndPort.port)).apply { - handler = HandlerCollection().apply { - addHandler(buildServletContextHandler()) - } - } - - val hostAndPort: NetworkHostAndPort - get() = server.connectors.mapNotNull { it as? ServerConnector } - .map { NetworkHostAndPort(it.host, it.localPort) } - .first() - - override fun close() { - println("Shutting down network management web services...") - server.stop() - server.join() - } - - fun start() { - server.start() - println("Network management web services started on $hostAndPort") - } - - private fun buildServletContextHandler(): ServletContextHandler { - val crlServer = this - return ServletContextHandler().apply { - contextPath = "/" - val resourceConfig = ResourceConfig().apply { - register(CrlServlet(crlServer)) - } - val jerseyServlet = ServletHolder(ServletContainer(resourceConfig)).apply { initOrder = 0 } - addServlet(jerseyServlet, "/*") - } - } - } - - @Test(timeout=300_000) - fun `verify CRL algorithms`() { - val ECDSA_ALGORITHM = "SHA256withECDSA" - val EC_ALGORITHM = "EC" - val EMPTY_CRL = "empty.crl" - - val crl = createRevocationList( - server, - ECDSA_ALGORITHM, - ROOT_CA.certificate, - ROOT_CA.keyPair.private, - EMPTY_CRL, - true) - // This should pass. - crl.verify(ROOT_CA.keyPair.public) - - // Try changing the algorithm to EC will fail. - assertThatIllegalArgumentException().isThrownBy { - createRevocationList( - server, - EC_ALGORITHM, - ROOT_CA.certificate, - ROOT_CA.keyPair.private, - EMPTY_CRL, - true - ) - }.withMessage("Unknown signature type requested: EC") - } - - @Test(timeout=300_000) - fun `AMPQ Client to Server connection succeeds when CRL retrieval is forbidden and soft fail is enabled`() { - val crlCheckSoftFail = true - val forbiddenUrl = "http://${server.hostAndPort}/crl/$FORBIDDEN_CRL" - val (amqpServer, _) = createServer( + @Suppress("LongParameterList") + private fun verifyAMQPConnection(crlCheckSoftFail: Boolean, + nodeCrlDistPoint: String? = "http://${crlServer.hostAndPort}/crl/$NODE_CRL", + revokeServerCert: Boolean = false, + revokeClientCert: Boolean = false, + sslHandshakeTimeout: Duration? = null, + expectedConnectStatus: Boolean) { + val serverCert = createAMQPServer( serverPort, crlCheckSoftFail = crlCheckSoftFail, - nodeCrlDistPoint = forbiddenUrl, - tlsCrlDistPoint = forbiddenUrl) - amqpServer.use { - amqpServer.start() - amqpServer.onReceive.subscribe { - it.complete(true) - } - val (amqpClient, _) = createClient( - serverPort, - crlCheckSoftFail, - nodeCrlDistPoint = forbiddenUrl, - tlsCrlDistPoint = forbiddenUrl) - amqpClient.use { - val serverConnected = amqpServer.onConnection.toFuture() - amqpClient.onConnection.toFuture() - amqpClient.start() - val serverConnect = serverConnected.get() - assertEquals(true, serverConnect.connected) - } + nodeCrlDistPoint = nodeCrlDistPoint, + sslHandshakeTimeout = sslHandshakeTimeout + ) + if (revokeServerCert) { + crlServer.revokedNodeCerts.add(serverCert.serialNumber) } + amqpServer.start() + amqpServer.onReceive.subscribe { + it.complete(true) + } + val clientCert = createAMQPClient( + serverPort, + crlCheckSoftFail = crlCheckSoftFail, + nodeCrlDistPoint = nodeCrlDistPoint + ) + if (revokeClientCert) { + crlServer.revokedNodeCerts.add(clientCert.serialNumber) + } + val serverConnected = amqpServer.onConnection.toFuture() + amqpClient.start() + val serverConnect = serverConnected.get() + assertThat(serverConnect.connected).isEqualTo(expectedConnectStatus) } - private fun createArtemisServerAndClient(port: Int, crlCheckSoftFail: Boolean, crlCheckArtemisServer: Boolean): - Pair { + private fun createArtemisServerAndClient(crlCheckSoftFail: Boolean, + crlCheckArtemisServer: Boolean, + nodeCrlDistPoint: String, + sslHandshakeTimeout: Duration?): Pair { val baseDirectory = temporaryFolder.root.toPath() / "artemis" val certificatesDirectory = baseDirectory / "certificates" val signingCertificateStore = CertificateStoreStubs.Signing.withCertificatesDirectory(certificatesDirectory) - val p2pSslConfiguration = CertificateStoreStubs.P2P.withCertificatesDirectory(certificatesDirectory) + val p2pSslConfiguration = CertificateStoreStubs.P2P.withCertificatesDirectory(certificatesDirectory, sslHandshakeTimeout = sslHandshakeTimeout) val artemisConfig = rigorousMock().also { doReturn(baseDirectory).whenever(it).baseDirectory doReturn(certificatesDirectory).whenever(it).certificatesDirectory doReturn(CHARLIE_NAME).whenever(it).myLegalName doReturn(signingCertificateStore).whenever(it).signingCertificateStore doReturn(p2pSslConfiguration).whenever(it).p2pSslOptions - doReturn(NetworkHostAndPort("0.0.0.0", port)).whenever(it).p2pAddress + doReturn(NetworkHostAndPort("0.0.0.0", serverPort)).whenever(it).p2pAddress doReturn(null).whenever(it).jmxMonitoringHttpPort doReturn(crlCheckSoftFail).whenever(it).crlCheckSoftFail doReturn(crlCheckArtemisServer).whenever(it).crlCheckArtemisServer } artemisConfig.configureWithDevSSLCertificate() + recreateNodeCaAndTlsCertificates(signingCertificateStore, p2pSslConfiguration, nodeCrlDistPoint, null) + val server = ArtemisMessagingServer(artemisConfig, artemisConfig.p2pAddress, MAX_MESSAGE_SIZE, null) val client = ArtemisMessagingClient(artemisConfig.p2pSslOptions, artemisConfig.p2pAddress, MAX_MESSAGE_SIZE) server.start() @@ -656,27 +485,31 @@ class CertificateRevocationListNodeTests { return server to client } - private fun verifyMessageToArtemis(crlCheckSoftFail: Boolean, - crlCheckArtemisServer: Boolean, - expectedStatus: MessageStatus, - revokedNodeCert: Boolean = false, - nodeCrlDistPoint: String = "http://${server.hostAndPort}/crl/node.crl") { + @Suppress("LongParameterList") + private fun verifyArtemisConnection(crlCheckSoftFail: Boolean, + crlCheckArtemisServer: Boolean, + expectedConnected: Boolean = true, + expectedStatus: MessageStatus? = null, + revokedNodeCert: Boolean = false, + nodeCrlDistPoint: String = "http://${crlServer.hostAndPort}/crl/$NODE_CRL", + sslHandshakeTimeout: Duration? = null) { val queueName = P2P_PREFIX + "Test" - val (artemisServer, artemisClient) = createArtemisServerAndClient(serverPort, crlCheckSoftFail, crlCheckArtemisServer) + val (artemisServer, artemisClient) = createArtemisServerAndClient(crlCheckSoftFail, crlCheckArtemisServer, nodeCrlDistPoint, sslHandshakeTimeout) artemisServer.use { artemisClient.started!!.session.createQueue( - QueueConfiguration(queueName).setRoutingType(RoutingType.ANYCAST).setAddress(queueName).setDurable(true)) + QueueConfiguration(queueName).setRoutingType(RoutingType.ANYCAST).setAddress(queueName).setDurable(true) + ) - val (amqpClient, nodeCert) = createClient(serverPort, true, nodeCrlDistPoint) + val nodeCert = createAMQPClient(serverPort, true, nodeCrlDistPoint) if (revokedNodeCert) { - revokedNodeCerts.add(nodeCert.serialNumber) + crlServer.revokedNodeCerts.add(nodeCert.serialNumber) } - amqpClient.use { - val clientConnected = amqpClient.onConnection.toFuture() - amqpClient.start() - val clientConnect = clientConnected.get() - assertEquals(true, clientConnect.connected) + val clientConnected = amqpClient.onConnection.toFuture() + amqpClient.start() + val clientConnect = clientConnected.get() + assertThat(clientConnect.connected).isEqualTo(expectedConnected) + if (expectedConnected) { val msg = amqpClient.createMessage("Test".toByteArray(), queueName, CHARLIE_NAME.toString(), emptyMap()) amqpClient.write(msg) assertEquals(expectedStatus, msg.onComplete.get()) @@ -684,38 +517,4 @@ class CertificateRevocationListNodeTests { artemisClient.stop() } } - - @Test(timeout = 300_000) - fun `Artemis server connection succeeds with soft fail CRL check`() { - verifyMessageToArtemis(crlCheckSoftFail = true, crlCheckArtemisServer = true, expectedStatus = MessageStatus.Acknowledged) - } - - @Test(timeout = 300_000) - fun `Artemis server connection succeeds with hard fail CRL check`() { - verifyMessageToArtemis(crlCheckSoftFail = false, crlCheckArtemisServer = true, expectedStatus = MessageStatus.Acknowledged) - } - - @Test(timeout = 300_000) - fun `Artemis server connection succeeds with soft fail CRL check on unavailable URL`() { - verifyMessageToArtemis(crlCheckSoftFail = true, crlCheckArtemisServer = true, expectedStatus = MessageStatus.Acknowledged, - nodeCrlDistPoint = "http://${server.hostAndPort}/crl/$FORBIDDEN_CRL") - } - - @Test(timeout = 300_000) - fun `Artemis server connection fails with hard fail CRL check on unavailable URL`() { - verifyMessageToArtemis(crlCheckSoftFail = false, crlCheckArtemisServer = true, expectedStatus = MessageStatus.Rejected, - nodeCrlDistPoint = "http://${server.hostAndPort}/crl/$FORBIDDEN_CRL") - } - - @Test(timeout = 300_000) - fun `Artemis server connection fails with soft fail CRL check on revoked node certificate`() { - verifyMessageToArtemis(crlCheckSoftFail = true, crlCheckArtemisServer = true, expectedStatus = MessageStatus.Rejected, - revokedNodeCert = true) - } - - @Test(timeout = 300_000) - fun `Artemis server connection succeeds with disabled CRL check on revoked node certificate`() { - verifyMessageToArtemis(crlCheckSoftFail = false, crlCheckArtemisServer = false, expectedStatus = MessageStatus.Acknowledged, - revokedNodeCert = true) - } } diff --git a/node/src/integration-test/kotlin/net/corda/node/services/messaging/ArtemisMessagingTest.kt b/node/src/integration-test/kotlin/net/corda/node/services/messaging/ArtemisMessagingTest.kt index 95b0b76662..873d1a1795 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/messaging/ArtemisMessagingTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/messaging/ArtemisMessagingTest.kt @@ -7,6 +7,8 @@ import net.corda.core.crypto.generateKeyPair import net.corda.core.internal.div import net.corda.core.utilities.NetworkHostAndPort import net.corda.core.utilities.seconds +import net.corda.coretesting.internal.rigorousMock +import net.corda.coretesting.internal.stubs.CertificateStoreStubs import net.corda.node.services.config.FlowTimeoutConfiguration import net.corda.node.services.config.NodeConfiguration import net.corda.node.services.config.configureWithDevSSLCertificate @@ -22,8 +24,6 @@ import net.corda.testing.driver.internal.incrementalPortAllocation import net.corda.testing.internal.LogHelper import net.corda.testing.internal.TestingNamedCacheFactory import net.corda.testing.internal.configureDatabase -import net.corda.coretesting.internal.rigorousMock -import net.corda.coretesting.internal.stubs.CertificateStoreStubs import net.corda.testing.node.MockServices.Companion.makeTestDataSourceProperties import net.corda.testing.node.internal.MOCK_VERSION_INFO import org.apache.activemq.artemis.api.core.ActiveMQConnectionTimedOutException @@ -57,7 +57,6 @@ class ArtemisMessagingTest { @JvmField val temporaryFolder = TemporaryFolder() - // THe private val portAllocation = incrementalPortAllocation() private val serverPort = portAllocation.nextPort() private val identity = generateKeyPair() @@ -201,7 +200,9 @@ class ArtemisMessagingTest { messagingClient!!.start(identity.public, null, maxMessageSize) } - private fun createAndStartClientAndServer(platformVersion: Int = 1, serverMaxMessageSize: Int = MAX_MESSAGE_SIZE, clientMaxMessageSize: Int = MAX_MESSAGE_SIZE): Pair> { + private fun createAndStartClientAndServer(platformVersion: Int = 1, + serverMaxMessageSize: Int = MAX_MESSAGE_SIZE, + clientMaxMessageSize: Int = MAX_MESSAGE_SIZE): Pair> { val receivedMessages = LinkedBlockingQueue() createMessagingServer(maxMessageSize = serverMaxMessageSize).start() @@ -242,7 +243,7 @@ class ArtemisMessagingTest { } private fun createMessagingServer(local: Int = serverPort, maxMessageSize: Int = MAX_MESSAGE_SIZE): ArtemisMessagingServer { - return ArtemisMessagingServer(config, NetworkHostAndPort("0.0.0.0", local), maxMessageSize, null).apply { + return ArtemisMessagingServer(config, NetworkHostAndPort("0.0.0.0", local), maxMessageSize, null, true).apply { config.configureWithDevSSLCertificate() messagingServer = this } diff --git a/node/src/main/kotlin/net/corda/node/internal/artemis/CertificateChainCheckPolicy.kt b/node/src/main/kotlin/net/corda/node/internal/artemis/CertificateChainCheckPolicy.kt index e8ef09a3c0..0acaae7d0e 100644 --- a/node/src/main/kotlin/net/corda/node/internal/artemis/CertificateChainCheckPolicy.kt +++ b/node/src/main/kotlin/net/corda/node/internal/artemis/CertificateChainCheckPolicy.kt @@ -5,15 +5,14 @@ import net.corda.core.utilities.contextLogger import net.corda.nodeapi.internal.crypto.X509CertificateFactory import net.corda.nodeapi.internal.crypto.X509Utilities import net.corda.nodeapi.internal.protonwrapper.netty.RevocationConfig +import net.corda.nodeapi.internal.protonwrapper.netty.RevocationConfigImpl import net.corda.nodeapi.internal.protonwrapper.netty.certPathToString import java.security.KeyStore import java.security.cert.CertPathValidator import java.security.cert.CertPathValidatorException import java.security.cert.CertificateException import java.security.cert.PKIXBuilderParameters -import java.security.cert.PKIXRevocationChecker import java.security.cert.X509CertSelector -import java.util.EnumSet sealed class CertificateChainCheckPolicy { companion object { @@ -94,13 +93,12 @@ sealed class CertificateChainCheckPolicy { } } - class RevocationCheck(val revocationMode: RevocationConfig.Mode) : CertificateChainCheckPolicy() { + class RevocationCheck(val revocationConfig: RevocationConfig) : CertificateChainCheckPolicy() { + constructor(revocationMode: RevocationConfig.Mode) : this(RevocationConfigImpl(revocationMode)) + override fun createCheck(keyStore: KeyStore, trustStore: KeyStore): Check { return object : Check { override fun checkCertificateChain(theirChain: Array) { - if (revocationMode == RevocationConfig.Mode.OFF) { - return - } // Convert javax.security.cert.X509Certificate to java.security.cert.X509Certificate. val chain = theirChain.map { X509CertificateFactory().generateCertificate(it.encoded.inputStream()) } log.info("Check Client Certpath:\r\n${certPathToString(chain.toTypedArray())}") @@ -110,17 +108,7 @@ sealed class CertificateChainCheckPolicy { // See PKIXValidator.engineValidate() for reference implementation. val certPath = X509Utilities.buildCertPath(chain.dropLast(1)) val certPathValidator = CertPathValidator.getInstance("PKIX") - val pkixRevocationChecker = certPathValidator.revocationChecker as PKIXRevocationChecker - pkixRevocationChecker.options = EnumSet.of( - // Prefer CRL over OCSP - PKIXRevocationChecker.Option.PREFER_CRLS, - // Don't fall back to OCSP checking - PKIXRevocationChecker.Option.NO_FALLBACK) - if (revocationMode == RevocationConfig.Mode.SOFT_FAIL) { - // Allow revocation check to succeed if the revocation status cannot be determined for one of - // the following reasons: The CRL or OCSP response cannot be obtained because of a network error. - pkixRevocationChecker.options = pkixRevocationChecker.options + PKIXRevocationChecker.Option.SOFT_FAIL - } + val pkixRevocationChecker = revocationConfig.createPKIXRevocationChecker() val params = PKIXBuilderParameters(trustStore, X509CertSelector()) params.addCertPathChecker(pkixRevocationChecker) try { diff --git a/node/src/main/kotlin/net/corda/node/services/messaging/ArtemisMessagingServer.kt b/node/src/main/kotlin/net/corda/node/services/messaging/ArtemisMessagingServer.kt index 1cb28178a8..3e79bc5966 100644 --- a/node/src/main/kotlin/net/corda/node/services/messaging/ArtemisMessagingServer.kt +++ b/node/src/main/kotlin/net/corda/node/services/messaging/ArtemisMessagingServer.kt @@ -56,7 +56,8 @@ import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag.RE class ArtemisMessagingServer(private val config: NodeConfiguration, private val messagingServerAddress: NetworkHostAndPort, private val maxMessageSize: Int, - private val journalBufferTimeout : Int?) : ArtemisBroker, SingletonSerializeAsToken() { + private val journalBufferTimeout : Int?, + private val trace: Boolean = false) : ArtemisBroker, SingletonSerializeAsToken() { companion object { private val log = contextLogger() } @@ -130,7 +131,11 @@ class ArtemisMessagingServer(private val config: NodeConfiguration, // The transaction cache is configurable, and drives other cache sizes. globalMaxSize = max(config.transactionCacheSizeBytes, 10L * maxMessageSize) - acceptorConfigurations = mutableSetOf(p2pAcceptorTcpTransport(NetworkHostAndPort(messagingServerAddress.host, messagingServerAddress.port), config.p2pSslOptions)) + acceptorConfigurations.add(p2pAcceptorTcpTransport( + NetworkHostAndPort(messagingServerAddress.host, messagingServerAddress.port), + config.p2pSslOptions, + trace = trace + )) // Enable built in message deduplication. Note we still have to do our own as the delayed commits // and our own definition of commit mean that the built in deduplication cannot remove all duplicates. idCacheSize = 2000 // Artemis Default duplicate cache size i.e. a guess diff --git a/node/src/main/kotlin/net/corda/node/services/messaging/NodeNettyAcceptorFactory.kt b/node/src/main/kotlin/net/corda/node/services/messaging/NodeNettyAcceptorFactory.kt new file mode 100644 index 0000000000..5d10099f29 --- /dev/null +++ b/node/src/main/kotlin/net/corda/node/services/messaging/NodeNettyAcceptorFactory.kt @@ -0,0 +1,68 @@ +package net.corda.node.services.messaging + +import io.netty.buffer.ByteBufAllocator +import io.netty.channel.group.ChannelGroup +import io.netty.handler.logging.LogLevel +import io.netty.handler.logging.LoggingHandler +import io.netty.handler.ssl.SslHandler +import net.corda.core.internal.declaredField +import net.corda.nodeapi.internal.ArtemisTcpTransport +import org.apache.activemq.artemis.api.core.BaseInterceptor +import org.apache.activemq.artemis.core.remoting.impl.netty.NettyAcceptor +import org.apache.activemq.artemis.core.server.balancing.RedirectHandler +import org.apache.activemq.artemis.core.server.cluster.ClusterConnection +import org.apache.activemq.artemis.spi.core.protocol.ProtocolManager +import org.apache.activemq.artemis.spi.core.remoting.Acceptor +import org.apache.activemq.artemis.spi.core.remoting.AcceptorFactory +import org.apache.activemq.artemis.spi.core.remoting.BufferHandler +import org.apache.activemq.artemis.spi.core.remoting.ServerConnectionLifeCycleListener +import org.apache.activemq.artemis.utils.actors.OrderedExecutor +import java.time.Duration +import java.util.concurrent.Executor +import java.util.concurrent.ScheduledExecutorService + +@Suppress("unused") // Used via reflection in ArtemisTcpTransport +class NodeNettyAcceptorFactory : AcceptorFactory { + override fun createAcceptor(name: String?, + clusterConnection: ClusterConnection?, + configuration: Map, + handler: BufferHandler?, + listener: ServerConnectionLifeCycleListener?, + threadPool: Executor, + scheduledThreadPool: ScheduledExecutorService?, + protocolMap: MutableMap, RedirectHandler<*>>>?): Acceptor { + val failureExecutor = OrderedExecutor(threadPool) + return NodeNettyAcceptor(name, clusterConnection, configuration, handler, listener, scheduledThreadPool, failureExecutor, protocolMap) + } + + private class NodeNettyAcceptor(name: String?, + clusterConnection: ClusterConnection?, + configuration: Map, + handler: BufferHandler?, + listener: ServerConnectionLifeCycleListener?, + scheduledThreadPool: ScheduledExecutorService?, + failureExecutor: Executor, + protocolMap: MutableMap, RedirectHandler<*>>>?) : + NettyAcceptor(name, clusterConnection, configuration, handler, listener, scheduledThreadPool, failureExecutor, protocolMap) + { + override fun start() { + super.start() + if (configuration[ArtemisTcpTransport.TRACE_NAME] == true) { + // Artemis does not seem to allow access to the underlying channel so we resort to reflection and get it via the + // serverChannelGroup field. This field is only available after start(), hence why we add the logger here. + declaredField("serverChannelGroup").value.forEach { channel -> + channel.pipeline().addLast("logger", LoggingHandler(LogLevel.INFO)) + } + } + } + + override fun getSslHandler(alloc: ByteBufAllocator?, peerHost: String?, peerPort: Int): SslHandler { + val sslHandler = super.getSslHandler(alloc, peerHost, peerPort) + val handshakeTimeout = configuration[ArtemisTcpTransport.SSL_HANDSHAKE_TIMEOUT_NAME] as Duration? + if (handshakeTimeout != null) { + sslHandler.handshakeTimeoutMillis = handshakeTimeout.toMillis() + } + return sslHandler + } + } +} diff --git a/testing/core-test-utils/src/main/kotlin/net/corda/coretesting/internal/stubs/CertificateStoreStubs.kt b/testing/core-test-utils/src/main/kotlin/net/corda/coretesting/internal/stubs/CertificateStoreStubs.kt index c23d458a80..2f90555c44 100644 --- a/testing/core-test-utils/src/main/kotlin/net/corda/coretesting/internal/stubs/CertificateStoreStubs.kt +++ b/testing/core-test-utils/src/main/kotlin/net/corda/coretesting/internal/stubs/CertificateStoreStubs.kt @@ -8,6 +8,7 @@ import net.corda.nodeapi.internal.config.FileBasedCertificateStoreSupplier import net.corda.nodeapi.internal.config.MutualSslConfiguration import net.corda.nodeapi.internal.config.SslConfiguration import java.nio.file.Path +import java.time.Duration class CertificateStoreStubs { @@ -49,11 +50,11 @@ class CertificateStoreStubs { keyStorePassword: String = KeyStore.DEFAULT_STORE_PASSWORD, keyPassword: String = keyStorePassword, trustStoreFileName: String = TrustStore.DEFAULT_STORE_FILE_NAME, trustStorePassword: String = TrustStore.DEFAULT_STORE_PASSWORD, - trustStoreKeyPassword: String = TrustStore.DEFAULT_KEY_PASSWORD): MutualSslConfiguration { - + trustStoreKeyPassword: String = TrustStore.DEFAULT_KEY_PASSWORD, + sslHandshakeTimeout: Duration? = null): MutualSslConfiguration { val keyStore = FileBasedCertificateStoreSupplier(certificatesDirectory / keyStoreFileName, keyStorePassword, keyPassword) val trustStore = FileBasedCertificateStoreSupplier(certificatesDirectory / trustStoreFileName, trustStorePassword, trustStoreKeyPassword) - return SslConfiguration.mutual(keyStore, trustStore) + return SslConfiguration.mutual(keyStore, trustStore, sslHandshakeTimeout) } @JvmStatic diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/network/CrlServer.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/network/CrlServer.kt new file mode 100644 index 0000000000..744f33c72c --- /dev/null +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/network/CrlServer.kt @@ -0,0 +1,203 @@ +@file:Suppress("MagicNumber") + +package net.corda.testing.node.internal.network + +import net.corda.core.crypto.Crypto +import net.corda.core.internal.CertRole +import net.corda.core.utilities.NetworkHostAndPort +import net.corda.core.utilities.days +import net.corda.core.utilities.minutes +import net.corda.core.utilities.seconds +import net.corda.coretesting.internal.DEV_INTERMEDIATE_CA +import net.corda.coretesting.internal.DEV_ROOT_CA +import net.corda.nodeapi.internal.crypto.CertificateAndKeyPair +import net.corda.nodeapi.internal.crypto.ContentSignerBuilder +import net.corda.nodeapi.internal.crypto.X509Utilities +import net.corda.nodeapi.internal.crypto.certificateType +import net.corda.nodeapi.internal.crypto.toJca +import org.bouncycastle.asn1.x500.X500Name +import org.bouncycastle.asn1.x509.CRLDistPoint +import org.bouncycastle.asn1.x509.DistributionPoint +import org.bouncycastle.asn1.x509.DistributionPointName +import org.bouncycastle.asn1.x509.Extension +import org.bouncycastle.asn1.x509.GeneralName +import org.bouncycastle.asn1.x509.GeneralNames +import org.bouncycastle.asn1.x509.IssuingDistributionPoint +import org.bouncycastle.asn1.x509.ReasonFlags +import org.bouncycastle.cert.jcajce.JcaX509CRLConverter +import org.bouncycastle.cert.jcajce.JcaX509ExtensionUtils +import org.bouncycastle.cert.jcajce.JcaX509v2CRLBuilder +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder +import org.eclipse.jetty.server.Server +import org.eclipse.jetty.server.ServerConnector +import org.eclipse.jetty.server.handler.HandlerCollection +import org.eclipse.jetty.servlet.ServletContextHandler +import org.eclipse.jetty.servlet.ServletHolder +import org.glassfish.jersey.server.ResourceConfig +import org.glassfish.jersey.servlet.ServletContainer +import java.io.Closeable +import java.math.BigInteger +import java.net.InetSocketAddress +import java.security.KeyPair +import java.security.cert.X509CRL +import java.security.cert.X509Certificate +import java.util.* +import javax.security.auth.x500.X500Principal +import javax.ws.rs.GET +import javax.ws.rs.Path +import javax.ws.rs.Produces +import javax.ws.rs.core.Response +import kotlin.collections.ArrayList + +class CrlServer(hostAndPort: NetworkHostAndPort) : Closeable { + companion object { + private const val SIGNATURE_ALGORITHM = "SHA256withECDSA" + + const val NODE_CRL = "node.crl" + const val FORBIDDEN_CRL = "forbidden.crl" + const val INTERMEDIATE_CRL = "intermediate.crl" + const val EMPTY_CRL = "empty.crl" + + fun X509Certificate.withCrlDistPoint(issuerKeyPair: KeyPair, crlDistPoint: String?, crlIssuer: X500Principal? = null): X509Certificate { + val signatureScheme = Crypto.findSignatureScheme(issuerKeyPair.private) + val provider = Crypto.findProvider(signatureScheme.providerName) + val issuerSigner = ContentSignerBuilder.build(signatureScheme, issuerKeyPair.private, provider) + val builder = X509Utilities.createPartialCertificate( + CertRole.extract(this)!!.certificateType, + issuerX500Principal, + issuerKeyPair.public, + subjectX500Principal, + publicKey, + Pair(Date(System.currentTimeMillis() - 5.minutes.toMillis()), Date(System.currentTimeMillis() + 10.days.toMillis())), + null + ) + if (crlDistPoint != null) { + val distPointName = DistributionPointName(GeneralNames(GeneralName(GeneralName.uniformResourceIdentifier, crlDistPoint))) + val crlIssuerGeneralNames = crlIssuer?.let { GeneralNames(GeneralName(X500Name.getInstance(it.encoded))) } + val distPoint = DistributionPoint(distPointName, null, crlIssuerGeneralNames) + builder.addExtension(Extension.cRLDistributionPoints, false, CRLDistPoint(arrayOf(distPoint))) + } + return builder.build(issuerSigner).toJca() + } + } + + private val server: Server = Server(InetSocketAddress(hostAndPort.host, hostAndPort.port)).apply { + handler = HandlerCollection().apply { + addHandler(buildServletContextHandler()) + } + } + + val revokedNodeCerts: MutableList = ArrayList() + val revokedIntermediateCerts: MutableList = ArrayList() + + val rootCa: CertificateAndKeyPair = DEV_ROOT_CA + + private lateinit var _intermediateCa: CertificateAndKeyPair + val intermediateCa: CertificateAndKeyPair get() = _intermediateCa + + val hostAndPort: NetworkHostAndPort + get() = server.connectors.mapNotNull { it as? ServerConnector } + .map { NetworkHostAndPort(it.host, it.localPort) } + .first() + + fun start() { + server.start() + _intermediateCa = CertificateAndKeyPair( + DEV_INTERMEDIATE_CA.certificate.withCrlDistPoint(rootCa.keyPair, "http://$hostAndPort/crl/$INTERMEDIATE_CRL"), + DEV_INTERMEDIATE_CA.keyPair + ) + println("Network management web services started on $hostAndPort") + } + + fun replaceNodeCertDistPoint(nodeCaCert: X509Certificate, + nodeCaCrlDistPoint: String? = "http://$hostAndPort/crl/$NODE_CRL", + crlIssuer: X500Principal? = null): X509Certificate { + return nodeCaCert.withCrlDistPoint(intermediateCa.keyPair, nodeCaCrlDistPoint, crlIssuer) + } + + fun createRevocationList(signatureAlgorithm: String, + ca: CertificateAndKeyPair, + endpoint: String, + indirect: Boolean, + serialNumbers: List): X509CRL { + println("Generating CRL for $endpoint") + val builder = JcaX509v2CRLBuilder(ca.certificate.subjectX500Principal, Date(System.currentTimeMillis() - 1.minutes.toMillis())) + val extensionUtils = JcaX509ExtensionUtils() + builder.addExtension(Extension.authorityKeyIdentifier, false, extensionUtils.createAuthorityKeyIdentifier(ca.certificate)) + val issuingDistPointName = GeneralName(GeneralName.uniformResourceIdentifier, "http://$hostAndPort/crl/$endpoint") + // This is required and needs to match the certificate settings with respect to being indirect + val issuingDistPoint = IssuingDistributionPoint(DistributionPointName(GeneralNames(issuingDistPointName)), indirect, false) + builder.addExtension(Extension.issuingDistributionPoint, true, issuingDistPoint) + builder.setNextUpdate(Date(System.currentTimeMillis() + 1.seconds.toMillis())) + serialNumbers.forEach { + builder.addCRLEntry(it, Date(System.currentTimeMillis() - 10.minutes.toMillis()), ReasonFlags.certificateHold) + } + val signer = JcaContentSignerBuilder(signatureAlgorithm).setProvider(Crypto.findProvider("BC")).build(ca.keyPair.private) + return JcaX509CRLConverter().setProvider(Crypto.findProvider("BC")).getCRL(builder.build(signer)) + } + + override fun close() { + println("Shutting down network management web services...") + server.stop() + server.join() + } + + private fun buildServletContextHandler(): ServletContextHandler { + return ServletContextHandler().apply { + contextPath = "/" + val resourceConfig = ResourceConfig().apply { + register(CrlServlet(this@CrlServer)) + } + val jerseyServlet = ServletHolder(ServletContainer(resourceConfig)).apply { initOrder = 0 } + addServlet(jerseyServlet, "/*") + } + } + + @Path("crl") + class CrlServlet(private val crlServer: CrlServer) { + @GET + @Path(NODE_CRL) + @Produces("application/pkcs7-crl") + fun getNodeCRL(): Response { + return Response.ok(crlServer.createRevocationList( + SIGNATURE_ALGORITHM, + crlServer.intermediateCa, + NODE_CRL, + false, + crlServer.revokedNodeCerts + ).encoded).build() + } + + @GET + @Path(FORBIDDEN_CRL) + @Produces("application/pkcs7-crl") + fun getNodeSlowCRL(): Response { + return Response.status(Response.Status.FORBIDDEN).build() + } + + @GET + @Path(INTERMEDIATE_CRL) + @Produces("application/pkcs7-crl") + fun getIntermediateCRL(): Response { + return Response.ok(crlServer.createRevocationList( + SIGNATURE_ALGORITHM, + crlServer.rootCa, + INTERMEDIATE_CRL, + false, + crlServer.revokedIntermediateCerts + ).encoded).build() + } + + @GET + @Path(EMPTY_CRL) + @Produces("application/pkcs7-crl") + fun getEmptyCRL(): Response { + return Response.ok(crlServer.createRevocationList( + SIGNATURE_ALGORITHM, + crlServer.rootCa, + EMPTY_CRL, + true, emptyList() + ).encoded).build() + } + } +}