[CORDA-1659]: Improve handling/logging of failed address binding. (#3498)

core/src/main/kotlin/net/corda/core/internal/errors/AddressBindingException.kt

@@ -0,0 +1,20 @@
+package net.corda.core.internal.errors
+
+import net.corda.core.CordaRuntimeException
+import net.corda.core.utilities.NetworkHostAndPort
+
+class AddressBindingException(val addresses: Set<NetworkHostAndPort>) : CordaRuntimeException(message(addresses)) {
+
+    constructor(address: NetworkHostAndPort) : this(setOf(address))
+
+    private companion object {
+        private fun message(addresses: Set<NetworkHostAndPort>): String {
+            require(addresses.isNotEmpty())
+            return if (addresses.size > 1) {
+                "Failed to bind on an address in ${addresses.joinToString(", ", "[", "]")}."
+            } else {
+                "Failed to bind on address ${addresses.single()}."
+            }
+        }
+    }
+}

node/src/integration-test/kotlin/net/corda/node/AddressBindingFailureTests.kt

@@ -0,0 +1,48 @@
+package net.corda.node
+
+import net.corda.core.utilities.NetworkHostAndPort
+import net.corda.core.utilities.getOrThrow
+import net.corda.core.internal.errors.AddressBindingException
+import net.corda.testing.driver.DriverParameters
+import net.corda.testing.driver.PortAllocation
+import net.corda.testing.driver.driver
+import org.assertj.core.api.Assertions.assertThat
+import org.assertj.core.api.Assertions.assertThatThrownBy
+import org.junit.Test
+import java.net.InetSocketAddress
+import java.net.ServerSocket
+
+class AddressBindingFailureTests {
+
+    companion object {
+        private val portAllocation = PortAllocation.Incremental(20_000)
+    }
+
+    @Test
+    fun `p2p address`() = assertBindExceptionForOverrides { address -> mapOf("p2pAddress" to address.toString()) }
+
+    @Test
+    fun `rpc address`() = assertBindExceptionForOverrides { address -> mapOf("rpcSettings" to mapOf("address" to address.toString())) }
+
+    @Test
+    fun `rpc admin address`() = assertBindExceptionForOverrides { address -> mapOf("rpcSettings" to mapOf("adminAddress" to address.toString())) }
+
+    @Test
+    fun `H2 address`() = assertBindExceptionForOverrides { address -> mapOf("h2Settings" to mapOf("address" to address.toString())) }
+
+    private fun assertBindExceptionForOverrides(overrides: (NetworkHostAndPort) -> Map<String, Any?>) {
+
+        ServerSocket(0).use { socket ->
+
+            val address = InetSocketAddress(socket.localPort).toNetworkHostAndPort()
+            driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList(), inMemoryDB = false, portAllocation = portAllocation)) {
+
+                assertThatThrownBy { startNode(customOverrides = overrides(address)).getOrThrow() }.isInstanceOfSatisfying(AddressBindingException::class.java) { exception ->
+                    assertThat(exception.addresses).contains(address).withFailMessage("Expected addresses to contain $address but was ${exception.addresses}.")
+                }
+            }
+        }
+    }
+
+    private fun InetSocketAddress.toNetworkHostAndPort() = NetworkHostAndPort(hostName, port)
+}

node/src/integration-test/kotlin/net/corda/node/BootTests.kt

@@ -36,7 +36,7 @@ class BootTests {
 
     @Test
     fun `double node start doesn't write into log file`() {
-        driver {
+        driver(DriverParameters(notarySpecs = emptyList())) {
             val alice = startNode(providedName = ALICE_NAME).get()
             val logFolder = alice.baseDirectory / NodeStartup.LOGS_DIRECTORY_NAME
             val logFile = logFolder.list { it.filter { it.fileName.toString().endsWith(".log") }.findAny().get() }

node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt

@@ -11,7 +11,16 @@ import net.corda.core.concurrent.CordaFuture
 import net.corda.core.context.InvocationContext
 import net.corda.core.crypto.newSecureRandom
 import net.corda.core.crypto.sign
-import net.corda.core.flows.*
+import net.corda.core.flows.ContractUpgradeFlow
+import net.corda.core.flows.FinalityFlow
+import net.corda.core.flows.FlowLogic
+import net.corda.core.flows.FlowLogicRefFactory
+import net.corda.core.flows.FlowSession
+import net.corda.core.flows.InitiatedBy
+import net.corda.core.flows.InitiatingFlow
+import net.corda.core.flows.NotaryChangeFlow
+import net.corda.core.flows.NotaryFlow
+import net.corda.core.flows.StartableByService
 import net.corda.core.identity.AbstractParty
 import net.corda.core.identity.CordaX500Name
 import net.corda.core.identity.Party
@@ -23,14 +32,31 @@ import net.corda.core.internal.concurrent.map
 import net.corda.core.internal.concurrent.openFuture
 import net.corda.core.internal.notary.NotaryService
 import net.corda.core.internal.uncheckedCast
-import net.corda.core.messaging.*
+import net.corda.core.messaging.CordaRPCOps
-import net.corda.core.node.*
+import net.corda.core.messaging.FlowHandle
-import net.corda.core.node.services.*
+import net.corda.core.messaging.FlowHandleImpl
+import net.corda.core.messaging.FlowProgressHandle
+import net.corda.core.messaging.FlowProgressHandleImpl
+import net.corda.core.messaging.RPCOps
+import net.corda.core.node.AppServiceHub
+import net.corda.core.node.NetworkParameters
+import net.corda.core.node.NodeInfo
+import net.corda.core.node.ServiceHub
+import net.corda.core.node.ServicesForResolution
+import net.corda.core.node.services.AttachmentStorage
+import net.corda.core.node.services.CordaService
+import net.corda.core.node.services.IdentityService
+import net.corda.core.node.services.KeyManagementService
+import net.corda.core.node.services.TransactionVerifierService
 import net.corda.core.serialization.SerializationWhitelist
 import net.corda.core.serialization.SerializeAsToken
 import net.corda.core.serialization.SingletonSerializeAsToken
 import net.corda.core.serialization.serialize
-import net.corda.core.utilities.*
+import net.corda.core.utilities.NetworkHostAndPort
+import net.corda.core.utilities.days
+import net.corda.core.utilities.debug
+import net.corda.core.utilities.getOrThrow
+import net.corda.core.utilities.minutes
 import net.corda.node.CordaClock
 import net.corda.node.VersionInfo
 import net.corda.node.internal.CheckpointVerifier.verifyCheckpointsCompatible
@@ -45,21 +71,61 @@ import net.corda.node.internal.security.RPCSecurityManager
 import net.corda.node.services.ContractUpgradeHandler
 import net.corda.node.services.FinalityHandler
 import net.corda.node.services.NotaryChangeHandler
-import net.corda.node.services.api.*
+import net.corda.node.services.api.CheckpointStorage
-import net.corda.node.services.config.*
+import net.corda.node.services.api.DummyAuditService
+import net.corda.node.services.api.FlowStarter
+import net.corda.node.services.api.IdentityServiceInternal
+import net.corda.node.services.api.MonitoringService
+import net.corda.node.services.api.NetworkMapCacheBaseInternal
+import net.corda.node.services.api.NetworkMapCacheInternal
+import net.corda.node.services.api.NodePropertiesStore
+import net.corda.node.services.api.SchedulerService
+import net.corda.node.services.api.SchemaService
+import net.corda.node.services.api.ServiceHubInternal
+import net.corda.node.services.api.StartedNodeServices
+import net.corda.node.services.api.VaultServiceInternal
+import net.corda.node.services.api.WritableTransactionStorage
+import net.corda.node.services.config.BFTSMaRtConfiguration
+import net.corda.node.services.config.NodeConfiguration
+import net.corda.node.services.config.NotaryConfig
+import net.corda.node.services.config.configureWithDevSSLCertificate
 import net.corda.node.services.config.shell.toShellConfig
+import net.corda.node.services.config.shouldInitCrashShell
 import net.corda.node.services.events.NodeSchedulerService
 import net.corda.node.services.events.ScheduledActivityObserver
 import net.corda.node.services.identity.PersistentIdentityService
 import net.corda.node.services.keys.PersistentKeyManagementService
 import net.corda.node.services.messaging.DeduplicationHandler
 import net.corda.node.services.messaging.MessagingService
-import net.corda.node.services.network.*
+import net.corda.node.services.network.NetworkMapCacheImpl
-import net.corda.node.services.persistence.*
+import net.corda.node.services.network.NetworkMapClient
+import net.corda.node.services.network.NetworkMapUpdater
+import net.corda.node.services.network.NodeInfoWatcher
+import net.corda.node.services.network.PersistentNetworkMapCache
+import net.corda.node.services.persistence.AbstractPartyDescriptor
+import net.corda.node.services.persistence.AbstractPartyToX500NameAsStringConverter
+import net.corda.node.services.persistence.DBCheckpointStorage
+import net.corda.node.services.persistence.DBTransactionMappingStorage
+import net.corda.node.services.persistence.DBTransactionStorage
+import net.corda.node.services.persistence.NodeAttachmentService
+import net.corda.node.services.persistence.NodePropertiesPersistentStore
 import net.corda.node.services.schema.HibernateObserver
 import net.corda.node.services.schema.NodeSchemaService
-import net.corda.node.services.statemachine.*
+import net.corda.node.services.statemachine.ExternalEvent
-import net.corda.node.services.transactions.*
+import net.corda.node.services.statemachine.FlowLogicRefFactoryImpl
+import net.corda.node.services.statemachine.FlowMonitor
+import net.corda.node.services.statemachine.SingleThreadedStateMachineManager
+import net.corda.node.services.statemachine.StateMachineManager
+import net.corda.node.services.statemachine.StateMachineManagerInternal
+import net.corda.node.services.statemachine.appName
+import net.corda.node.services.statemachine.flowVersionAndInitiatingClass
+import net.corda.node.services.transactions.BFTNonValidatingNotaryService
+import net.corda.node.services.transactions.BFTSMaRt
+import net.corda.node.services.transactions.RaftNonValidatingNotaryService
+import net.corda.node.services.transactions.RaftUniquenessProvider
+import net.corda.node.services.transactions.RaftValidatingNotaryService
+import net.corda.node.services.transactions.SimpleNotaryService
+import net.corda.node.services.transactions.ValidatingNotaryService
 import net.corda.node.services.upgrade.ContractUpgradeServiceImpl
 import net.corda.node.services.vault.NodeVaultService
 import net.corda.node.utilities.AffinityExecutor
@@ -368,7 +434,11 @@ abstract class AbstractNode(val configuration: NodeConfiguration,
 
     open fun startShell() {
         if (configuration.shouldInitCrashShell()) {
-            InteractiveShell.startShellInternal(configuration.toShellConfig(), cordappLoader.appClassLoader)
+            val shellConfiguration = configuration.toShellConfig()
+            shellConfiguration.sshHostKeyDirectory?.let {
+                log.info("Binding Shell SSHD server on port $it.")
+            }
+            InteractiveShell.startShellInternal(shellConfiguration, cordappLoader.appClassLoader)
         }
     }
 

node/src/main/kotlin/net/corda/node/internal/Node.kt

@@ -26,6 +26,7 @@ import net.corda.node.VersionInfo
 import net.corda.node.internal.artemis.ArtemisBroker
 import net.corda.node.internal.artemis.BrokerAddresses
 import net.corda.node.internal.cordapp.CordappLoader
+import net.corda.core.internal.errors.AddressBindingException
 import net.corda.node.internal.security.RPCSecurityManagerImpl
 import net.corda.node.internal.security.RPCSecurityManagerWithAdditionalUser
 import net.corda.node.serialization.amqp.AMQPServerSerializationScheme
@@ -36,7 +37,6 @@ import net.corda.node.services.api.NodePropertiesStore
 import net.corda.node.services.api.SchemaService
 import net.corda.node.services.config.NodeConfiguration
 import net.corda.node.services.config.SecurityConfiguration
-import net.corda.node.services.config.VerifierType
 import net.corda.node.services.config.shouldInitCrashShell
 import net.corda.node.services.config.shouldStartLocalShell
 import net.corda.node.services.messaging.ArtemisMessagingServer
@@ -61,10 +61,12 @@ import net.corda.serialization.internal.AMQP_RPC_CLIENT_CONTEXT
 import net.corda.serialization.internal.AMQP_RPC_SERVER_CONTEXT
 import net.corda.serialization.internal.AMQP_STORAGE_CONTEXT
 import net.corda.serialization.internal.SerializationFactoryImpl
+import org.h2.jdbc.JdbcSQLException
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import rx.Scheduler
 import rx.schedulers.Schedulers
+import java.net.BindException
 import java.nio.file.Path
 import java.security.PublicKey
 import java.time.Clock
@@ -328,7 +330,7 @@ open class Node(configuration: NodeConfiguration,
         if (databaseUrl != null && databaseUrl.startsWith(h2Prefix)) {
             val effectiveH2Settings = configuration.effectiveH2Settings
 
-            if(effectiveH2Settings != null && effectiveH2Settings.address != null) {
+            if (effectiveH2Settings?.address != null) {
                 val databaseName = databaseUrl.removePrefix(h2Prefix).substringBefore(';')
                 val server = org.h2.tools.Server.createTcpServer(
                         "-tcpPort", effectiveH2Settings.address.port.toString(),
@@ -338,7 +340,15 @@ open class Node(configuration: NodeConfiguration,
                 // override interface that createTcpServer listens on (which is always 0.0.0.0)
                 System.setProperty("h2.bindAddress", effectiveH2Settings.address.host)
                 runOnStop += server::stop
-                val url = server.start().url
+                val url = try {
+                    server.start().url
+                } catch (e: JdbcSQLException) {
+                    if (e.cause is BindException) {
+                        throw AddressBindingException(effectiveH2Settings.address)
+                    } else {
+                        throw e
+                    }
+                }
                 printBasicNodeInfo("Database connection url is", "jdbc:h2:$url/node")
             }
         }

node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt

@@ -6,8 +6,14 @@ import com.typesafe.config.ConfigException
 import com.typesafe.config.ConfigRenderOptions
 import io.netty.channel.unix.Errors
 import net.corda.core.crypto.Crypto
-import net.corda.core.internal.*
+import net.corda.core.internal.Emoji
 import net.corda.core.internal.concurrent.thenMatch
+import net.corda.core.internal.createDirectories
+import net.corda.core.internal.div
+import net.corda.core.internal.errors.AddressBindingException
+import net.corda.core.internal.exists
+import net.corda.core.internal.location
+import net.corda.core.internal.randomOrNull
 import net.corda.core.utilities.Try
 import net.corda.core.utilities.loggerFor
 import net.corda.node.*
@@ -149,6 +155,9 @@ open class NodeStartup(val args: Array<String>) {
         } catch (e: CheckpointIncompatibleException) {
             logger.error(e.message)
             return false
+        } catch (e: AddressBindingException) {
+            logger.error(e.message)
+            return false
         } catch (e: NetworkParametersReader.Error) {
             logger.error(e.message)
             return false

node/src/main/kotlin/net/corda/node/internal/artemis/ArtemisBroker.kt

@@ -1,8 +1,10 @@
 package net.corda.node.internal.artemis
 
+import io.netty.channel.unix.Errors
 import net.corda.core.utilities.NetworkHostAndPort
 import net.corda.node.internal.LifecycleSupport
 import org.apache.activemq.artemis.api.core.management.ActiveMQServerControl
+import java.net.BindException
 
 interface ArtemisBroker : LifecycleSupport, AutoCloseable {
     val addresses: BrokerAddresses
@@ -15,3 +17,5 @@ interface ArtemisBroker : LifecycleSupport, AutoCloseable {
 data class BrokerAddresses(val primary: NetworkHostAndPort, private val adminArg: NetworkHostAndPort?) {
     val admin = adminArg ?: primary
 }
+
+fun java.io.IOException.isBindingError() = this is BindException || this is Errors.NativeIoException && message?.contains("Address already in use") == true

node/src/main/kotlin/net/corda/node/services/messaging/ArtemisMessagingServer.kt

@@ -1,5 +1,6 @@
 package net.corda.node.services.messaging
 
+import io.netty.channel.unix.Errors
 import net.corda.core.internal.ThreadBox
 import net.corda.core.internal.div
 import net.corda.core.serialization.SingletonSerializeAsToken
@@ -9,6 +10,7 @@ import net.corda.core.utilities.debug
 import net.corda.node.internal.artemis.*
 import net.corda.node.internal.artemis.BrokerJaasLoginModule.Companion.NODE_P2P_ROLE
 import net.corda.node.internal.artemis.BrokerJaasLoginModule.Companion.PEER_ROLE
+import net.corda.core.internal.errors.AddressBindingException
 import net.corda.node.services.config.NodeConfiguration
 import net.corda.nodeapi.ArtemisTcpTransport.Companion.p2pAcceptorTcpTransport
 import net.corda.nodeapi.internal.AmqpMessageSizeChecksInterceptor
@@ -91,7 +93,7 @@ class ArtemisMessagingServer(private val config: NodeConfiguration,
 
     // TODO: Maybe wrap [IOException] on a key store load error so that it's clearly splitting key store loading from
     // Artemis IO errors
-    @Throws(IOException::class, KeyStoreException::class)
+    @Throws(IOException::class, AddressBindingException::class, KeyStoreException::class)
     private fun configureAndStartServer() {
         val artemisConfig = createArtemisConfig()
         val securityManager = createArtemisSecurityManager()
@@ -104,7 +106,15 @@ class ArtemisMessagingServer(private val config: NodeConfiguration,
             registerPostQueueDeletionCallback { address, qName -> log.debug { "Queue deleted: $qName for $address" } }
         }
 
+        try {
             activeMQServer.start()
+        } catch (e: java.io.IOException) {
+            if (e.isBindingError()) {
+                throw AddressBindingException(config.p2pAddress)
+            } else {
+                throw e
+            }
+        }
         activeMQServer.remotingService.addIncomingInterceptor(ArtemisMessageSizeChecksInterceptor(maxMessageSize))
         activeMQServer.remotingService.addIncomingInterceptor(AmqpMessageSizeChecksInterceptor(maxMessageSize))
         // Config driven switch between legacy CORE bridges and the newer AMQP protocol bridges.

node/src/main/kotlin/net/corda/node/services/rpc/ArtemisRpcBroker.kt

@@ -1,10 +1,12 @@
 package net.corda.node.services.rpc
 
+import io.netty.channel.unix.Errors
 import net.corda.core.utilities.NetworkHostAndPort
 import net.corda.core.utilities.loggerFor
 import net.corda.node.internal.artemis.*
 import net.corda.node.internal.artemis.BrokerJaasLoginModule.Companion.NODE_SECURITY_CONFIG
 import net.corda.node.internal.artemis.BrokerJaasLoginModule.Companion.RPC_SECURITY_CONFIG
+import net.corda.core.internal.errors.AddressBindingException
 import net.corda.node.internal.security.RPCSecurityManager
 import net.corda.nodeapi.BrokerRpcSslOptions
 import net.corda.nodeapi.internal.config.SSLConfiguration
@@ -44,7 +46,15 @@ internal class ArtemisRpcBroker internal constructor(
 
     override fun start() {
         logger.debug("Artemis RPC broker is starting.")
+        try {
             server.start()
+        } catch (e: java.io.IOException) {
+            if (e.isBindingError()) {
+                throw AddressBindingException(adminAddressOptional?.let { setOf(it, addresses.primary) } ?: setOf(addresses.primary))
+            } else {
+                throw e
+            }
+        }
         logger.debug("Artemis RPC broker is started.")
     }
 

webserver/src/main/kotlin/net/corda/webserver/WebServer.kt

@@ -4,6 +4,7 @@ package net.corda.webserver
 
 import com.typesafe.config.ConfigException
 import net.corda.core.internal.div
+import net.corda.core.internal.errors.AddressBindingException
 import net.corda.core.internal.location
 import net.corda.core.internal.rootCause
 import net.corda.webserver.internal.NodeWebServer
@@ -66,6 +67,9 @@ fun main(args: Array<String>) {
         val elapsed = (System.currentTimeMillis() - startTime) / 10 / 100.0
         println("Webserver started up in $elapsed sec")
         server.run()
+    } catch (e: AddressBindingException) {
+        log.error(e.message)
+        exitProcess(1)
     } catch (e: Exception) {
         log.error("Exception during node startup", e)
         exitProcess(1)

webserver/src/main/kotlin/net/corda/webserver/internal/NodeWebServer.kt

@@ -1,26 +1,18 @@
 package net.corda.webserver.internal
 
 import com.google.common.html.HtmlEscapers.htmlEscaper
+import io.netty.channel.unix.Errors
 import net.corda.client.jackson.JacksonSupport
 import net.corda.client.rpc.CordaRPCClient
 import net.corda.client.rpc.RPCException
+import net.corda.core.internal.errors.AddressBindingException
 import net.corda.core.messaging.CordaRPCOps
 import net.corda.core.utilities.contextLogger
 import net.corda.webserver.WebServerConfig
 import net.corda.webserver.converters.CordaConverterProvider
 import net.corda.webserver.services.WebServerPluginRegistry
-import net.corda.webserver.servlets.AttachmentDownloadServlet
+import net.corda.webserver.servlets.*
-import net.corda.webserver.servlets.CorDappInfoServlet
+import org.eclipse.jetty.server.*
-import net.corda.webserver.servlets.DataUploadServlet
-import net.corda.webserver.servlets.ObjectMapperConfig
-import net.corda.webserver.servlets.ResponseFilter
-import org.eclipse.jetty.server.Connector
-import org.eclipse.jetty.server.HttpConfiguration
-import org.eclipse.jetty.server.HttpConnectionFactory
-import org.eclipse.jetty.server.SecureRequestCustomizer
-import org.eclipse.jetty.server.Server
-import org.eclipse.jetty.server.ServerConnector
-import org.eclipse.jetty.server.SslConnectionFactory
 import org.eclipse.jetty.server.handler.ErrorHandler
 import org.eclipse.jetty.server.handler.HandlerCollection
 import org.eclipse.jetty.servlet.DefaultServlet
@@ -34,6 +26,7 @@ import org.slf4j.LoggerFactory
 import java.io.IOException
 import java.io.Writer
 import java.lang.reflect.InvocationTargetException
+import java.net.BindException
 import java.nio.file.NoSuchFileException
 import java.util.*
 import javax.servlet.http.HttpServletRequest
@@ -95,7 +88,15 @@ class NodeWebServer(val config: WebServerConfig) {
         server.connectors = arrayOf<Connector>(connector)
 
         server.handler = handlerCollection
+        try {
             server.start()
+        } catch (e: IOException) {
+            if (e is BindException || e is Errors.NativeIoException && e.message?.contains("Address already in use") == true) {
+                throw AddressBindingException(address)
+            } else {
+                throw e
+            }
+        }
         log.info("Starting webserver on address $address")
         return server
     }