Backports: CORDA-1663, CORDA-1638, CORDA-1542 (#3881)

docs/source/changelog.rst

@@ -6,6 +6,13 @@ release, see :doc:`upgrade-notes`.
 
 Unreleased
 ==========
+* Fixed an issue preventing Shell from returning control to the user when CTRL+C is pressed in the terminal.
+
+* Fixed a problem that sometimes prevented nodes from starting in presence of custom state types in the database without a corresponding type from installed CorDapps.
+
+* Introduced a grace period before the initial node registration fails if the node cannot connect to the Doorman.
+  It retries 10 times with a 1 minute interval in between each try. At the moment this is not configurable.
+
 * Fixed an error thrown by NodeVaultService upon recording a transaction with a number of inputs greater than the default page size.
 
 * Changes to the JSON/YAML serialisation format from ``JacksonSupport``, which also applies to the node shell:

node/src/integration-test/kotlin/net/corda/node/persistence/FailNodeOnNotMigratedAttachmentContractsTableNameTests.kt

@@ -5,6 +5,7 @@ import net.corda.core.internal.packageName
 import net.corda.core.messaging.startFlow
 import net.corda.core.utilities.getOrThrow
 import net.corda.node.services.Permissions
+import net.corda.nodeapi.internal.persistence.DatabaseIncompatibleException
 import net.corda.test.node.Message
 import net.corda.test.node.MessageState
 import net.corda.test.node.SendMessageFlow
@@ -13,6 +14,8 @@ import net.corda.testing.driver.DriverParameters
 import net.corda.testing.driver.driver
 import net.corda.testing.driver.internal.RandomFree
 import net.corda.testing.node.User
+import org.assertj.core.api.Assertions
+import org.assertj.core.api.Assertions.assertThatThrownBy
 import org.junit.Test
 import java.nio.file.Path
 import java.sql.DriverManager
@@ -32,9 +35,9 @@ class FailNodeOnNotMigratedAttachmentContractsTableNameTests {
     fun `node fails when not detecting compatible table name`(tableNameFromMapping: String, tableNameInDB: String) {
         val user = User("mark", "dadada", setOf(Permissions.startFlow<SendMessageFlow>(), Permissions.invokeRpc("vaultQuery")))
         val message = Message("Hello world!")
-        val baseDir: Path = driver(DriverParameters(startNodesInProcess = true,
+        val baseDir: Path = driver(DriverParameters(startNodesInProcess = true, portAllocation = RandomFree, extraCordappPackagesToScan = listOf(MessageState::class.packageName))) {
-                portAllocation = RandomFree, extraCordappPackagesToScan = listOf(MessageState::class.packageName))) {
             val (nodeName, baseDir) = {
+                defaultNotaryNode.getOrThrow()
                 val nodeHandle = startNode(rpcUsers = listOf(user)).getOrThrow()
                 val nodeName = nodeHandle.nodeInfo.singleIdentity().name
                 CordaRPCClient(nodeHandle.rpcAddress).start(user.username, user.password).use {
@@ -49,11 +52,8 @@ class FailNodeOnNotMigratedAttachmentContractsTableNameTests {
                 it.createStatement().execute("ALTER TABLE $tableNameFromMapping RENAME TO $tableNameInDB")
                 it.commit()
             }
-            assertFailsWith(net.corda.nodeapi.internal.persistence.DatabaseIncompatibleException::class) {
+            assertThatThrownBy { startNode(providedName = nodeName, rpcUsers = listOf(user)).getOrThrow() }.isInstanceOf(DatabaseIncompatibleException::class.java)
-                val nodeHandle = startNode(providedName = nodeName, rpcUsers = listOf(user)).getOrThrow()
+            baseDir
-                nodeHandle.stop()
-            }
-             baseDir
         }
 
         // check that the node didn't recreated the correct table matching it's entity mapping

node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt

@@ -16,6 +16,7 @@ import net.corda.node.services.transactions.bftSMaRtSerialFilter
 import net.corda.node.shell.InteractiveShell
 import net.corda.node.utilities.registration.HTTPNetworkRegistrationService
 import net.corda.node.utilities.registration.NetworkRegistrationHelper
+import net.corda.node.utilities.registration.UnableToRegisterNodeWithDoormanException
 import net.corda.nodeapi.internal.addShutdownHook
 import net.corda.nodeapi.internal.persistence.DatabaseIncompatibleException
 import org.fusesource.jansi.Ansi
@@ -106,6 +107,9 @@ open class NodeStartup(val args: Array<String>) {
                 return true
             }
             logStartupInfo(versionInfo, cmdlineOptions, conf)
+        } catch (e: UnableToRegisterNodeWithDoormanException) {
+            logger.warn("Node registration service is unavailable. Perhaps try to perform the initial registration again after a while.")
+            return false
         } catch (e: Exception) {
             logger.error("Exception during node registration", e)
             return false

node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt

@@ -380,8 +380,8 @@ class NodeVaultService(
                 log.trace { "State update of type: $concreteType" }
                 val seen = contractStateTypeMappings.any { it.value.contains(concreteType.name) }
                 if (!seen) {
-                    val contractInterfaces = deriveContractInterfaces(concreteType)
+                    val contractStateTypes = deriveContractTypes(concreteType)
-                    contractInterfaces.map {
+                    contractStateTypes.map {
                         val contractInterface = contractStateTypeMappings.getOrPut(it.name, { mutableSetOf() })
                         contractInterface.add(concreteType.name)
                     }
@@ -490,25 +490,42 @@ class NodeVaultService(
         val distinctTypes = results.map { it }
 
         val contractInterfaceToConcreteTypes = mutableMapOf<String, MutableSet<String>>()
+        val unknownTypes = mutableSetOf<String>()
         distinctTypes.forEach { type ->
-            val concreteType: Class<ContractState> = uncheckedCast(Class.forName(type))
+            val concreteType: Class<ContractState>? = try {
-            val contractInterfaces = deriveContractInterfaces(concreteType)
+                uncheckedCast(Class.forName(type))
-            contractInterfaces.map {
+            } catch (e: ClassNotFoundException) {
-                val contractInterface = contractInterfaceToConcreteTypes.getOrPut(it.name, { mutableSetOf() })
+                unknownTypes += type
-                contractInterface.add(concreteType.name)
+                null
             }
+            concreteType?.let {
+                val contractTypes = deriveContractTypes(it)
+                contractTypes.map {
+                    val contractStateType = contractInterfaceToConcreteTypes.getOrPut(it.name) { mutableSetOf() }
+                    contractStateType.add(concreteType.name)
+                }
+            }
+        }
+        if (unknownTypes.isNotEmpty()) {
+            log.warn("There are unknown contract state types in the vault, which will prevent these states from being used. The relevant CorDapps must be loaded for these states to be used. The types not on the classpath are ${unknownTypes.joinToString(", ", "[", "]")}.")
         }
         return contractInterfaceToConcreteTypes
     }
 
-    private fun <T : ContractState> deriveContractInterfaces(clazz: Class<T>): Set<Class<T>> {
+    private fun <T : ContractState> deriveContractTypes(clazz: Class<T>): Set<Class<T>> {
-        val myInterfaces: MutableSet<Class<T>> = mutableSetOf()
+        val myTypes : MutableSet<Class<T>> = mutableSetOf()
-        clazz.interfaces.forEach {
+        clazz.superclass?.let {
-            if (it != ContractState::class.java) {
+            if (!it.isInstance(Any::class)) {
-                myInterfaces.add(uncheckedCast(it))
+                myTypes.add(uncheckedCast(it))
-                myInterfaces.addAll(deriveContractInterfaces(uncheckedCast(it)))
+                myTypes.addAll(deriveContractTypes(uncheckedCast(it)))
             }
         }
-        return myInterfaces
+        clazz.interfaces.forEach {
+            if (it != ContractState::class.java) {
+                myTypes.add(uncheckedCast(it))
+                myTypes.addAll(deriveContractTypes(uncheckedCast(it)))
+            }
+        }
+        return myTypes
     }
-}
+}

node/src/main/kotlin/net/corda/node/shell/InteractiveShell.kt

@@ -240,12 +240,18 @@ object InteractiveShell {
 
             val latch = CountDownLatch(1)
             ansiProgressRenderer.render(stateObservable, { latch.countDown() })
-            try {
+            while (!Thread.currentThread().isInterrupted) {
-                // Wait for the flow to end and the progress tracker to notice. By the time the latch is released
+                try {
-                // the tracker is done with the screen.
+                    latch.await()
-                latch.await()
+                    break
-            } catch (e: InterruptedException) {
+                } catch (e: InterruptedException) {
-                // TODO: When the flow framework allows us to kill flows mid-flight, do so here.
+                    try {
+                        // TODO: When the flow framework allows us to kill flows mid-flight, do so here.
+                    } finally {
+                        Thread.currentThread().interrupt()
+                        break
+                    }
+                }
             }
             stateObservable.returnValue.get()?.apply {
                 if (this !is Throwable) {

node/src/main/kotlin/net/corda/node/utilities/registration/HTTPNetworkRegistrationService.kt

@@ -15,6 +15,7 @@ import java.net.URL
 import java.security.cert.X509Certificate
 import java.util.*
 import java.util.zip.ZipInputStream
+import javax.naming.ServiceUnavailableException
 
 class HTTPNetworkRegistrationService(compatibilityZoneURL: URL) : NetworkRegistrationService {
     private val registrationURL = URL("$compatibilityZoneURL/certificate")
@@ -22,6 +23,7 @@ class HTTPNetworkRegistrationService(compatibilityZoneURL: URL) : NetworkRegistr
     companion object {
         // TODO: Propagate version information from gradle
         val clientVersion = "1.0"
+        private val TRANSIENT_ERROR_STATUS_CODES = setOf(HTTP_BAD_GATEWAY, HTTP_UNAVAILABLE, HTTP_GATEWAY_TIMEOUT)
     }
 
     @Throws(CertificateRequestException::class)
@@ -44,6 +46,7 @@ class HTTPNetworkRegistrationService(compatibilityZoneURL: URL) : NetworkRegistr
             }
             HTTP_NO_CONTENT -> CertificateResponse(pollInterval, null)
             HTTP_UNAUTHORIZED -> throw CertificateRequestException("Certificate signing request has been rejected: ${conn.errorMessage}")
+            in TRANSIENT_ERROR_STATUS_CODES -> throw ServiceUnavailableException("Could not connect with Doorman. Http response status code was ${conn.responseCode}.")
             else -> throwUnexpectedResponseCode(conn)
         }
     }

node/src/main/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelper.kt

@@ -15,11 +15,14 @@ import net.corda.nodeapi.internal.crypto.X509Utilities.CORDA_CLIENT_TLS
 import net.corda.nodeapi.internal.crypto.X509Utilities.CORDA_ROOT_CA
 import org.bouncycastle.openssl.jcajce.JcaPEMWriter
 import org.bouncycastle.util.io.pem.PemObject
+import java.io.IOException
 import java.io.StringWriter
 import java.nio.file.Path
 import java.security.KeyPair
 import java.security.KeyStore
 import java.security.cert.X509Certificate
+import java.time.Duration
+import javax.naming.ServiceUnavailableException
 
 /**
  * Helper for managing the node registration process, which checks for any existing certificates and requests them if
@@ -31,7 +34,8 @@ class NetworkRegistrationHelper(private val config: SSLConfiguration,
                                 private val certService: NetworkRegistrationService,
                                 private val networkRootTrustStorePath: Path,
                                 networkRootTrustStorePassword: String,
-                                private val certRole: CertRole) {
+                                private val certRole: CertRole,
+                                private val nextIdleDuration: (Duration?) -> Duration? = FixedPeriodLimitedRetrialStrategy(10, Duration.ofMinutes(1))) {
 
     // Constructor for corda node, cert role is restricted to [CertRole.NODE_CA].
     constructor(config: NodeConfiguration, certService: NetworkRegistrationService, regConfig: NodeRegistrationOption) :
@@ -171,12 +175,22 @@ class NetworkRegistrationHelper(private val config: SSLConfiguration,
     private fun pollServerForCertificates(requestId: String): List<X509Certificate> {
         println("Start polling server for certificate signing approval.")
         // Poll server to download the signed certificate once request has been approved.
+        var idlePeriodDuration: Duration? = null
         while (true) {
-            val (pollInterval, certificates) = certService.retrieveCertificates(requestId)
+            try {
-            if (certificates != null) {
+                val (pollInterval, certificates) = certService.retrieveCertificates(requestId)
-                return certificates
+                if (certificates != null) {
+                    return certificates
+                }
+                Thread.sleep(pollInterval.toMillis())
+            } catch (e: ServiceUnavailableException) {
+                idlePeriodDuration = nextIdleDuration(idlePeriodDuration)
+                if (idlePeriodDuration != null) {
+                    Thread.sleep(idlePeriodDuration.toMillis())
+                } else {
+                    throw UnableToRegisterNodeWithDoormanException()
+                }
             }
-            Thread.sleep(pollInterval.toMillis())
         }
     }
 
@@ -216,3 +230,17 @@ class NetworkRegistrationHelper(private val config: SSLConfiguration,
         }
     }
 }
+
+class UnableToRegisterNodeWithDoormanException : IOException()
+
+private class FixedPeriodLimitedRetrialStrategy(times: Int, private val period: Duration) : (Duration?) -> Duration? {
+    init {
+        require(times > 0)
+    }
+    private var counter = times
+    override fun invoke(@Suppress("UNUSED_PARAMETER") previousPeriod: Duration?): Duration? {
+        synchronized(this) {
+            return if (counter-- > 0) period else null
+        }
+    }
+}