INFRA-670 Stop shutting down the database before some services that use it (#6690)

* INFRA-670 Stop shutting down the database before some services that use it.

* Test fixes

node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt

@@ -167,6 +167,8 @@ import net.corda.nodeapi.internal.persistence.OutstandingDatabaseChangesExceptio
 import net.corda.nodeapi.internal.persistence.RestrictedConnection
 import net.corda.nodeapi.internal.persistence.RestrictedEntityManager
 import net.corda.nodeapi.internal.persistence.SchemaMigration
+import net.corda.nodeapi.internal.persistence.contextDatabase
+import net.corda.nodeapi.internal.persistence.withoutDatabaseAccess
 import net.corda.tools.shell.InteractiveShell
 import org.apache.activemq.artemis.utils.ReusableLatch
 import org.jolokia.jvmagent.JolokiaServer
@@ -252,7 +254,7 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
     private val notaryLoader = configuration.notary?.let {
         NotaryLoader(it, versionInfo)
     }
-    val cordappLoader: CordappLoader = makeCordappLoader(configuration, versionInfo).closeOnStop()
+    val cordappLoader: CordappLoader = makeCordappLoader(configuration, versionInfo).closeOnStop(false)
     val schemaService = NodeSchemaService(cordappLoader.cordappSchemas).tokenize()
     val identityService = PersistentIdentityService(cacheFactory).tokenize()
     val database: CordaPersistence = createCordaPersistence(
@@ -387,8 +389,13 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
         return this
     }
 
-    protected fun <T : AutoCloseable> T.closeOnStop(): T {
-        runOnStop += this::close
+    protected fun <T : AutoCloseable> T.closeOnStop(usesDatabase: Boolean = true): T {
+        if (usesDatabase) {
+            contextDatabase // Will throw if no database is available, since this would run after closing the database, yet claims it needs it.
+            runOnStop += this::close
+        } else {
+            runOnStop += { withoutDatabaseAccess { this.close() } }
+        }
         return this
     }
 
@@ -533,7 +540,6 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
         installCoreFlows()
         registerCordappFlows()
         services.rpcFlows += cordappLoader.cordapps.flatMap { it.rpcFlows }
-        val rpcOps = makeRPCOps(cordappLoader, checkpointDumper)
         startShell()
         networkMapClient?.start(trustRoot)
 
@@ -546,6 +552,11 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
         networkMapCache.start(netParams.notaries)
 
         startDatabase()
+        // The following services need to be closed before the database, so need to be registered after it is started.
+        networkMapUpdater.closeOnStop()
+        schedulerService.closeOnStop()
+        val rpcOps = makeRPCOps(cordappLoader, checkpointDumper)
+
         val (identity, identityKeyPair) = obtainIdentity()
         X509Utilities.validateCertPath(trustRoot, identity.certPath)
 
@@ -815,7 +826,7 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
             configuration.baseDirectory,
             configuration.extraNetworkMapKeys,
             networkParametersStorage
-    ).closeOnStop()
+    )
 
     protected open fun makeNodeSchedulerService() = NodeSchedulerService(
             platformClock,
@@ -826,7 +837,7 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
             nodeProperties,
             configuration.drainingModePollPeriod,
             unfinishedSchedules = busyNodeLatch
-    ).tokenize().closeOnStop()
+    ).tokenize()
 
     private fun makeCordappLoader(configuration: NodeConfiguration, versionInfo: VersionInfo): CordappLoader {
         val generatedCordapps = mutableListOf(VirtualCordapp.generateCore(versionInfo))

testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalMockNetwork.kt

@@ -12,8 +12,14 @@ import net.corda.core.flows.InitiatedBy
 import net.corda.core.identity.CordaX500Name
 import net.corda.core.identity.Party
 import net.corda.core.identity.PartyAndCertificate
-import net.corda.core.internal.*
+import net.corda.core.internal.FlowIORequest
+import net.corda.core.internal.NetworkParametersStorage
+import net.corda.core.internal.PLATFORM_VERSION
+import net.corda.core.internal.VisibleForTesting
+import net.corda.core.internal.createDirectories
+import net.corda.core.internal.div
 import net.corda.core.internal.notary.NotaryService
+import net.corda.core.internal.uncheckedCast
 import net.corda.core.messaging.CordaRPCOps
 import net.corda.core.messaging.MessageRecipients
 import net.corda.core.messaging.RPCOps
@@ -26,6 +32,9 @@ import net.corda.core.utilities.NetworkHostAndPort
 import net.corda.core.utilities.contextLogger
 import net.corda.core.utilities.hours
 import net.corda.core.utilities.seconds
+import net.corda.coretesting.internal.rigorousMock
+import net.corda.coretesting.internal.stubs.CertificateStoreStubs
+import net.corda.coretesting.internal.testThreadFactory
 import net.corda.node.VersionInfo
 import net.corda.node.internal.AbstractNode
 import net.corda.node.internal.InitiatedFlowFactory
@@ -33,7 +42,11 @@ import net.corda.node.internal.NodeFlowManager
 import net.corda.node.services.api.FlowStarter
 import net.corda.node.services.api.ServiceHubInternal
 import net.corda.node.services.api.StartedNodeServices
-import net.corda.node.services.config.*
+import net.corda.node.services.config.FlowTimeoutConfiguration
+import net.corda.node.services.config.NetworkParameterAcceptanceSettings
+import net.corda.node.services.config.NodeConfiguration
+import net.corda.node.services.config.NotaryConfig
+import net.corda.node.services.config.VerifierType
 import net.corda.node.services.identity.PersistentIdentityService
 import net.corda.node.services.keys.BasicHSMKeyManagementService
 import net.corda.node.services.keys.KeyManagementServiceInternal
@@ -51,11 +64,12 @@ import net.corda.nodeapi.internal.network.NetworkParametersCopier
 import net.corda.nodeapi.internal.persistence.CordaPersistence
 import net.corda.nodeapi.internal.persistence.DatabaseConfig
 import net.corda.testing.common.internal.testNetworkParameters
-import net.corda.coretesting.internal.rigorousMock
-import net.corda.coretesting.internal.stubs.CertificateStoreStubs
-import net.corda.coretesting.internal.testThreadFactory
-import net.corda.testing.node.*
+import net.corda.testing.node.InMemoryMessagingNetwork
+import net.corda.testing.node.MockNetworkNotarySpec
+import net.corda.testing.node.MockNetworkParameters
+import net.corda.testing.node.MockNodeParameters
 import net.corda.testing.node.MockServices.Companion.makeTestDataSourceProperties
+import net.corda.testing.node.TestClock
 import org.apache.activemq.artemis.utils.ReusableLatch
 import org.apache.sshd.common.util.security.SecurityUtils
 import rx.Observable
@@ -377,7 +391,7 @@ open class InternalMockNetwork(cordappPackages: List<String> = emptyList(),
         }
 
         override fun makeMessagingService(): MockNodeMessagingService {
-            return MockNodeMessagingService(configuration, serverThread).closeOnStop()
+            return MockNodeMessagingService(configuration, serverThread).closeOnStop(usesDatabase = false)
         }
 
         override fun startMessagingService(rpcOps: RPCOps,