ENT-4240 Added general exception handler for Virtual Machine errors. (#5530)

* Added general exception handler for capturing and exiting on all types of Virtual Machine errors.

* Minimize additional processing (use constant string for error message).

* Remove redundant DB specific thread termination handler.

* Incorporating PR review feedback.

* Add println before halt.

* Add println before halt ... to stderr

node/src/main/kotlin/net/corda/node/internal/GeneralExceptionHandler.kt

@@ -5,14 +5,21 @@ import org.slf4j.LoggerFactory
 import java.sql.SQLException
 
 /**
- * If a thread dies because it can't connect to the database, the node ends up in an inconsistent state.
+ *
+ * Cater for all type of unrecoverable [VirtualMachineError] in which the node may end up in an inconsistent state.
  * Fail fast and hard.
  */
-class DbExceptionHandler(private val parentHandler: Thread.UncaughtExceptionHandler? = null) : Thread.UncaughtExceptionHandler {
+class GeneralExceptionHandler(private val parentHandler: Thread.UncaughtExceptionHandler? = null) : Thread.UncaughtExceptionHandler {
+
     override fun uncaughtException(t: Thread?, e: Throwable?) {
 
+        // fail fast with minimal overhead and further processing
+        if (e is VirtualMachineError) {
+            System.err.println("${e.message}")
+            Runtime.getRuntime().halt(1)
+        }
         // the error is a database connection issue - pull the rug
-        if (e is Error && e.cause is SQLException) {
+        else if (e is Error && e.cause is SQLException) {
             errorAndTerminate("Thread ${t!!.name} failed due to database connection error. This is unrecoverable, terminating node.", e)
         }
 

node/src/main/kotlin/net/corda/node/internal/Node.kt

@@ -462,11 +462,10 @@ open class Node(configuration: NodeConfiguration,
     }
 
     /**
-     * Register a default exception handler for all threads that terminates the process if the database connection goes away and
-     * cannot be recovered.
+     * Register a default exception handler for all threads that terminate the process due to an unrecoverable Virtual Machine error.
      */
     private fun registerDefaultExceptionHandler() {
-        Thread.setDefaultUncaughtExceptionHandler(DbExceptionHandler(Thread.getDefaultUncaughtExceptionHandler()))
+        Thread.setDefaultUncaughtExceptionHandler(GeneralExceptionHandler(Thread.getDefaultUncaughtExceptionHandler()))
     }
 
     /**

node/src/main/kotlin/net/corda/node/utilities/errorAndTerminate.kt

@@ -10,6 +10,7 @@ import kotlin.concurrent.thread
  */
 @Synchronized
 fun errorAndTerminate(message: String, e: Throwable?) {
+    try {
         thread {
             val log = LoggerFactory.getLogger("errorAndTerminate")
             log.error(message, e)
@@ -17,5 +18,8 @@ fun errorAndTerminate(message: String, e: Throwable?) {
 
         // give the logger a chance to flush the error message before killing the node
         Thread.sleep(10.seconds.toMillis())
+    }
+    finally {
         Runtime.getRuntime().halt(1)
     }
+}