ENT-4240 Added general exception handler for Virtual Machine errors. (#5530)

* Added general exception handler for capturing and exiting on all types of Virtual Machine errors.

* Minimize additional processing (use constant string for error message).

* Remove redundant DB specific thread termination handler.

* Incorporating PR review feedback.

* Add println before halt.

* Add println before halt ... to stderr
This commit is contained in:
josecoll 2019-10-01 16:12:55 +01:00 committed by GitHub
parent 6ae9151012
commit 1ac1c483ff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 23 additions and 13 deletions

View File

@ -5,14 +5,21 @@ import org.slf4j.LoggerFactory
import java.sql.SQLException
/**
* If a thread dies because it can't connect to the database, the node ends up in an inconsistent state.
*
* Cater for all type of unrecoverable [VirtualMachineError] in which the node may end up in an inconsistent state.
* Fail fast and hard.
*/
class DbExceptionHandler(private val parentHandler: Thread.UncaughtExceptionHandler? = null) : Thread.UncaughtExceptionHandler {
class GeneralExceptionHandler(private val parentHandler: Thread.UncaughtExceptionHandler? = null) : Thread.UncaughtExceptionHandler {
override fun uncaughtException(t: Thread?, e: Throwable?) {
// fail fast with minimal overhead and further processing
if (e is VirtualMachineError) {
System.err.println("${e.message}")
Runtime.getRuntime().halt(1)
}
// the error is a database connection issue - pull the rug
if (e is Error && e.cause is SQLException) {
else if (e is Error && e.cause is SQLException) {
errorAndTerminate("Thread ${t!!.name} failed due to database connection error. This is unrecoverable, terminating node.", e)
}

View File

@ -462,11 +462,10 @@ open class Node(configuration: NodeConfiguration,
}
/**
* Register a default exception handler for all threads that terminates the process if the database connection goes away and
* cannot be recovered.
* Register a default exception handler for all threads that terminate the process due to an unrecoverable Virtual Machine error.
*/
private fun registerDefaultExceptionHandler() {
Thread.setDefaultUncaughtExceptionHandler(DbExceptionHandler(Thread.getDefaultUncaughtExceptionHandler()))
Thread.setDefaultUncaughtExceptionHandler(GeneralExceptionHandler(Thread.getDefaultUncaughtExceptionHandler()))
}
/**

View File

@ -10,12 +10,16 @@ import kotlin.concurrent.thread
*/
@Synchronized
fun errorAndTerminate(message: String, e: Throwable?) {
thread {
val log = LoggerFactory.getLogger("errorAndTerminate")
log.error(message, e)
}
try {
thread {
val log = LoggerFactory.getLogger("errorAndTerminate")
log.error(message, e)
}
// give the logger a chance to flush the error message before killing the node
Thread.sleep(10.seconds.toMillis())
Runtime.getRuntime().halt(1)
// give the logger a chance to flush the error message before killing the node
Thread.sleep(10.seconds.toMillis())
}
finally {
Runtime.getRuntime().halt(1)
}
}