mirror of
https://github.com/corda/corda.git
synced 2024-12-24 07:06:44 +00:00
CORDA-4173 Obfuscated zib bombs used for unit tests, so that antivirus software stop complaining about them (#6989)
This commit is contained in:
parent
65bba87741
commit
efaf1549a9
@ -12,6 +12,10 @@ description 'Corda core'
|
|||||||
// required by DJVM and Avian JVM (for running inside the SGX enclave) which only supports Java 8.
|
// required by DJVM and Avian JVM (for running inside the SGX enclave) which only supports Java 8.
|
||||||
targetCompatibility = VERSION_1_8
|
targetCompatibility = VERSION_1_8
|
||||||
|
|
||||||
|
sourceSets {
|
||||||
|
obfuscator
|
||||||
|
}
|
||||||
|
|
||||||
configurations {
|
configurations {
|
||||||
integrationTestCompile.extendsFrom testCompile
|
integrationTestCompile.extendsFrom testCompile
|
||||||
integrationTestRuntimeOnly.extendsFrom testRuntimeOnly
|
integrationTestRuntimeOnly.extendsFrom testRuntimeOnly
|
||||||
@ -22,6 +26,9 @@ configurations {
|
|||||||
|
|
||||||
dependencies {
|
dependencies {
|
||||||
|
|
||||||
|
obfuscatorImplementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version"
|
||||||
|
|
||||||
|
testImplementation sourceSets.obfuscator.output
|
||||||
testImplementation "org.junit.jupiter:junit-jupiter-api:${junit_jupiter_version}"
|
testImplementation "org.junit.jupiter:junit-jupiter-api:${junit_jupiter_version}"
|
||||||
testImplementation "junit:junit:$junit_version"
|
testImplementation "junit:junit:$junit_version"
|
||||||
testRuntimeOnly "org.junit.vintage:junit-vintage-engine:${junit_vintage_version}"
|
testRuntimeOnly "org.junit.vintage:junit-vintage-engine:${junit_vintage_version}"
|
||||||
@ -172,3 +179,10 @@ scanApi {
|
|||||||
publish {
|
publish {
|
||||||
name jar.baseName
|
name jar.baseName
|
||||||
}
|
}
|
||||||
|
|
||||||
|
tasks.register("writeTestResources", JavaExec) {
|
||||||
|
classpath sourceSets.obfuscator.output
|
||||||
|
classpath sourceSets.obfuscator.runtimeClasspath
|
||||||
|
main 'net.corda.core.internal.utilities.TestResourceWriter'
|
||||||
|
args new File(sourceSets.test.resources.srcDirs.first(), "zip").toString()
|
||||||
|
}
|
||||||
|
@ -0,0 +1,54 @@
|
|||||||
|
package net.corda.core.internal.utilities
|
||||||
|
|
||||||
|
import net.corda.core.obfuscator.XorOutputStream
|
||||||
|
import java.net.URL
|
||||||
|
import java.nio.file.Files
|
||||||
|
import java.nio.file.Paths
|
||||||
|
import java.util.zip.ZipEntry
|
||||||
|
import java.util.zip.ZipOutputStream
|
||||||
|
|
||||||
|
object TestResourceWriter {
|
||||||
|
|
||||||
|
private val externalZipBombUrls = arrayOf(
|
||||||
|
URL("https://www.bamsoftware.com/hacks/zipbomb/zbsm.zip"),
|
||||||
|
URL("https://www.bamsoftware.com/hacks/zipbomb/zblg.zip"),
|
||||||
|
URL("https://www.bamsoftware.com/hacks/zipbomb/zbxl.zip")
|
||||||
|
)
|
||||||
|
|
||||||
|
@JvmStatic
|
||||||
|
@Suppress("NestedBlockDepth", "MagicNumber")
|
||||||
|
fun main(vararg args : String) {
|
||||||
|
for(arg in args) {
|
||||||
|
/**
|
||||||
|
* Download zip bombs
|
||||||
|
*/
|
||||||
|
for(url in externalZipBombUrls) {
|
||||||
|
url.openStream().use { inputStream ->
|
||||||
|
val destination = Paths.get(arg).resolve(Paths.get(url.path + ".xor").fileName)
|
||||||
|
Files.newOutputStream(destination).buffered().let(::XorOutputStream).use { outputStream ->
|
||||||
|
inputStream.copyTo(outputStream)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* Create a jar archive with a huge manifest file, used in unit tests to check that it is also identified as a zip bomb.
|
||||||
|
* This is because {@link java.util.jar.JarInputStream}
|
||||||
|
* <a href="https://github.com/openjdk/jdk/blob/4dedba9ebe11750f4b39c41feb4a4314ccdb3a08/src/java.base/share/classes/java/util/jar/JarInputStream.java#L95">eagerly loads the manifest file in memory</a>
|
||||||
|
* which would make such a jar dangerous if used as an attachment
|
||||||
|
*/
|
||||||
|
val destination = Paths.get(arg).resolve(Paths.get("big-manifest.jar.xor").fileName)
|
||||||
|
ZipOutputStream(XorOutputStream((Files.newOutputStream(destination).buffered()))).use { zos ->
|
||||||
|
val zipEntry = ZipEntry("MANIFEST.MF")
|
||||||
|
zipEntry.method = ZipEntry.DEFLATED
|
||||||
|
zos.putNextEntry(zipEntry)
|
||||||
|
val buffer = ByteArray(0x100000) { 0x0 }
|
||||||
|
var written = 0L
|
||||||
|
while(written < 10_000_000_000) {
|
||||||
|
zos.write(buffer)
|
||||||
|
written += buffer.size
|
||||||
|
}
|
||||||
|
zos.closeEntry()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,30 @@
|
|||||||
|
package net.corda.core.obfuscator
|
||||||
|
|
||||||
|
import java.io.FilterInputStream
|
||||||
|
import java.io.InputStream
|
||||||
|
|
||||||
|
@Suppress("MagicNumber")
|
||||||
|
class XorInputStream(private val source : InputStream) : FilterInputStream(source) {
|
||||||
|
var prev : Int = 0
|
||||||
|
|
||||||
|
override fun read(): Int {
|
||||||
|
prev = source.read() xor prev
|
||||||
|
return prev - 0x80
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun read(buffer: ByteArray): Int {
|
||||||
|
return read(buffer, 0, buffer.size)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun read(buffer: ByteArray, off: Int, len: Int): Int {
|
||||||
|
var read = 0
|
||||||
|
while(true) {
|
||||||
|
val b = source.read()
|
||||||
|
if(b < 0) break
|
||||||
|
buffer[off + read++] = ((b xor prev) - 0x80).toByte()
|
||||||
|
prev = b
|
||||||
|
if(read == len) break
|
||||||
|
}
|
||||||
|
return read
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,30 @@
|
|||||||
|
package net.corda.core.obfuscator
|
||||||
|
|
||||||
|
import java.io.FilterOutputStream
|
||||||
|
import java.io.OutputStream
|
||||||
|
|
||||||
|
@Suppress("MagicNumber")
|
||||||
|
class XorOutputStream(private val destination : OutputStream) : FilterOutputStream(destination) {
|
||||||
|
var prev : Int = 0
|
||||||
|
|
||||||
|
override fun write(byte: Int) {
|
||||||
|
val b = (byte + 0x80) xor prev
|
||||||
|
destination.write(b)
|
||||||
|
prev = b
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun write(buffer: ByteArray) {
|
||||||
|
write(buffer, 0, buffer.size)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun write(buffer: ByteArray, off: Int, len: Int) {
|
||||||
|
var written = 0
|
||||||
|
while(true) {
|
||||||
|
val b = (buffer[written] + 0x80) xor prev
|
||||||
|
destination.write(b)
|
||||||
|
prev = b
|
||||||
|
++written
|
||||||
|
if(written == len) break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -7,6 +7,18 @@ The Corda core module defines a lot of types and helpers that can only be exerci
|
|||||||
the context of a node. However, as everything else depends on the core module, we cannot pull the node into
|
the context of a node. However, as everything else depends on the core module, we cannot pull the node into
|
||||||
this module. Therefore, any tests that require further Corda dependencies need to be defined in the module
|
this module. Therefore, any tests that require further Corda dependencies need to be defined in the module
|
||||||
`core-tests`, which has the full set of dependencies including `node-driver`.
|
`core-tests`, which has the full set of dependencies including `node-driver`.
|
||||||
|
|
||||||
|
|
||||||
|
# ZipBomb tests
|
||||||
|
|
||||||
|
There is a unit test that checks the zip bomb detector in `net.corda.core.internal.utilities.ZipBombDetector` works correctly.
|
||||||
|
This test (`core/src/test/kotlin/net/corda/core/internal/utilities/ZipBombDetectorTest.kt`) uses real zip bombs, provided by `https://www.bamsoftware.com/hacks/zipbomb/`.
|
||||||
|
As it is undesirable to have unit test depends on external internet resources we do not control, those files are included as resources in
|
||||||
|
`core/src/test/resources/zip/`, however some Windows antivirus software correctly identifies those files as zip bombs,
|
||||||
|
raising an alert to the user. To mitigate this, those files have been obfuscated using `net.corda.core.obfuscator.XorOutputStream`
|
||||||
|
(which simply XORs every byte of the file with the previous one, except for the first byte that is XORed with zero)
|
||||||
|
to prevent antivirus software from detecting them as zip bombs and are de-obfuscated on the fly in unit tests using
|
||||||
|
`net.corda.core.obfuscator.XorInputStream`.
|
||||||
|
|
||||||
|
There is a dedicated Gradle task to re-download and re-obfuscate all the test resource files named `writeTestResources`,
|
||||||
|
its source code is in `core/src/obfuscator/kotlin/net/corda/core/internal/utilities/TestResourceWriter.kt`
|
||||||
|
|
@ -1,5 +1,6 @@
|
|||||||
package net.corda.core.internal.utilities
|
package net.corda.core.internal.utilities
|
||||||
|
|
||||||
|
import net.corda.core.obfuscator.XorInputStream
|
||||||
import org.junit.Assert
|
import org.junit.Assert
|
||||||
import org.junit.Test
|
import org.junit.Test
|
||||||
import org.junit.runner.RunWith
|
import org.junit.runner.RunWith
|
||||||
@ -22,28 +23,28 @@ class ZipBombDetectorTest(private val case : TestCase) {
|
|||||||
// so the total uncompressed size doesn't exceed maxUncompressedSize
|
// so the total uncompressed size doesn't exceed maxUncompressedSize
|
||||||
SMALL_BOMB(
|
SMALL_BOMB(
|
||||||
"A large (5.5 GB) zip archive",
|
"A large (5.5 GB) zip archive",
|
||||||
"zip/zbsm.zip", 64_000_000, 10f, false),
|
"zip/zbsm.zip.xor", 64_000_000, 10f, false),
|
||||||
|
|
||||||
// Decreasing maxUncompressedSize leads to a successful detection
|
// Decreasing maxUncompressedSize leads to a successful detection
|
||||||
SMALL_BOMB2(
|
SMALL_BOMB2(
|
||||||
"A large (5.5 GB) zip archive, with 1MB maxUncompressedSize",
|
"A large (5.5 GB) zip archive, with 1MB maxUncompressedSize",
|
||||||
"zip/zbsm.zip", 1_000_000, 10f, true),
|
"zip/zbsm.zip.xor", 1_000_000, 10f, true),
|
||||||
|
|
||||||
// ZipInputStream is also unable to read all entries of zblg.zip, but since the first one is already bigger than 4GB,
|
// ZipInputStream is also unable to read all entries of zblg.zip, but since the first one is already bigger than 4GB,
|
||||||
// that is enough to exceed maxUncompressedSize
|
// that is enough to exceed maxUncompressedSize
|
||||||
LARGE_BOMB(
|
LARGE_BOMB(
|
||||||
"A huge (281 TB) Zip bomb, this is the biggest possible non-recursive non-Zip64 archive",
|
"A huge (281 TB) Zip bomb, this is the biggest possible non-recursive non-Zip64 archive",
|
||||||
"zip/zblg.zip", 64_000_000, 10f, true),
|
"zip/zblg.zip.xor", 64_000_000, 10f, true),
|
||||||
|
|
||||||
//Same for this, but its entries are 22GB each
|
//Same for this, but its entries are 22GB each
|
||||||
EXTRA_LARGE_BOMB(
|
EXTRA_LARGE_BOMB(
|
||||||
"A humongous (4.5 PB) Zip64 bomb",
|
"A humongous (4.5 PB) Zip64 bomb",
|
||||||
"zip/zbxl.zip", 64_000_000, 10f, true),
|
"zip/zbxl.zip.xor", 64_000_000, 10f, true),
|
||||||
|
|
||||||
//This is a jar file containing a single 10GB manifest
|
//This is a jar file containing a single 10GB manifest
|
||||||
BIG_MANIFEST(
|
BIG_MANIFEST(
|
||||||
"A jar file with a huge manifest",
|
"A jar file with a huge manifest",
|
||||||
"zip/big-manifest.jar", 64_000_000, 10f, true);
|
"zip/big-manifest.jar.xor", 64_000_000, 10f, true);
|
||||||
|
|
||||||
override fun toString() = description
|
override fun toString() = description
|
||||||
}
|
}
|
||||||
@ -51,7 +52,7 @@ class ZipBombDetectorTest(private val case : TestCase) {
|
|||||||
companion object {
|
companion object {
|
||||||
@JvmStatic
|
@JvmStatic
|
||||||
@Parameterized.Parameters(name = "{0}")
|
@Parameterized.Parameters(name = "{0}")
|
||||||
fun primeNumbers(): Collection<*> {
|
fun generateTestCases(): Collection<*> {
|
||||||
return TestCase.values().toList()
|
return TestCase.values().toList()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -59,7 +60,10 @@ class ZipBombDetectorTest(private val case : TestCase) {
|
|||||||
@Test(timeout=10_000)
|
@Test(timeout=10_000)
|
||||||
fun test() {
|
fun test() {
|
||||||
(javaClass.classLoader.getResourceAsStream(case.zipResource) ?:
|
(javaClass.classLoader.getResourceAsStream(case.zipResource) ?:
|
||||||
throw IllegalStateException("Missing test resource file ${case.zipResource}")).let {
|
throw IllegalStateException("Missing test resource file ${case.zipResource}"))
|
||||||
|
.buffered()
|
||||||
|
.let(::XorInputStream)
|
||||||
|
.let {
|
||||||
Assert.assertEquals(case.expectedOutcome, ZipBombDetector.scanZip(it, case.maxUncompressedSize, case.maxCompressionRatio))
|
Assert.assertEquals(case.expectedOutcome, ZipBombDetector.scanZip(it, case.maxUncompressedSize, case.maxCompressionRatio))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,50 @@
|
|||||||
|
package net.corda.core.obfuscator
|
||||||
|
|
||||||
|
import org.junit.Assert
|
||||||
|
import org.junit.Test
|
||||||
|
import org.junit.runner.RunWith
|
||||||
|
import org.junit.runners.Parameterized
|
||||||
|
import java.io.ByteArrayInputStream
|
||||||
|
import java.io.ByteArrayOutputStream
|
||||||
|
import java.security.DigestInputStream
|
||||||
|
import java.security.DigestOutputStream
|
||||||
|
import java.security.MessageDigest
|
||||||
|
import java.util.*
|
||||||
|
|
||||||
|
@RunWith(Parameterized::class)
|
||||||
|
class XorStreamTest(private val size : Int) {
|
||||||
|
private val random = Random(0)
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
@JvmStatic
|
||||||
|
@Parameterized.Parameters(name = "{0}")
|
||||||
|
fun generateTestCases(): Collection<*> {
|
||||||
|
return listOf(0, 16, 31, 127, 1000, 1024)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout = 5000)
|
||||||
|
fun test() {
|
||||||
|
val baos = ByteArrayOutputStream(size)
|
||||||
|
val md = MessageDigest.getInstance("MD5")
|
||||||
|
val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
|
||||||
|
DigestOutputStream(XorOutputStream(baos), md).use { outputStream ->
|
||||||
|
var written = 0
|
||||||
|
while(written < size) {
|
||||||
|
random.nextBytes(buffer)
|
||||||
|
val bytesToWrite = (size - written).coerceAtMost(buffer.size)
|
||||||
|
outputStream.write(buffer, 0, bytesToWrite)
|
||||||
|
written += bytesToWrite
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val digest = md.digest()
|
||||||
|
md.reset()
|
||||||
|
DigestInputStream(XorInputStream(ByteArrayInputStream(baos.toByteArray())), md).use { inputStream ->
|
||||||
|
while(true) {
|
||||||
|
val read = inputStream.read(buffer)
|
||||||
|
if(read <= 0) break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Assert.assertArrayEquals(digest, md.digest())
|
||||||
|
}
|
||||||
|
}
|
Binary file not shown.
BIN
core/src/test/resources/zip/big-manifest.jar.xor
Normal file
BIN
core/src/test/resources/zip/big-manifest.jar.xor
Normal file
Binary file not shown.
Binary file not shown.
BIN
core/src/test/resources/zip/zblg.zip.xor
Normal file
BIN
core/src/test/resources/zip/zblg.zip.xor
Normal file
Binary file not shown.
Binary file not shown.
BIN
core/src/test/resources/zip/zbsm.zip.xor
Normal file
BIN
core/src/test/resources/zip/zbsm.zip.xor
Normal file
Binary file not shown.
Binary file not shown.
BIN
core/src/test/resources/zip/zbxl.zip.xor
Normal file
BIN
core/src/test/resources/zip/zbxl.zip.xor
Normal file
Binary file not shown.
Loading…
Reference in New Issue
Block a user