CORDA-4173 Obfuscated zib bombs used for unit tests, so that antivirus software stop complaining about them (#6989)

core/build.gradle

@@ -12,6 +12,10 @@ description 'Corda core'
 // required by DJVM and Avian JVM (for running inside the SGX enclave) which only supports Java 8.
 targetCompatibility = VERSION_1_8
 
+sourceSets {
+    obfuscator
+}
+
 configurations {
     integrationTestCompile.extendsFrom testCompile
     integrationTestRuntimeOnly.extendsFrom testRuntimeOnly
@@ -22,6 +26,9 @@ configurations {
 
 dependencies {
 
+    obfuscatorImplementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version"
+
+    testImplementation sourceSets.obfuscator.output
     testImplementation "org.junit.jupiter:junit-jupiter-api:${junit_jupiter_version}"
     testImplementation "junit:junit:$junit_version"
     testRuntimeOnly "org.junit.vintage:junit-vintage-engine:${junit_vintage_version}"
@@ -172,3 +179,10 @@ scanApi {
 publish {
     name jar.baseName
 }
+
+tasks.register("writeTestResources", JavaExec) {
+    classpath sourceSets.obfuscator.output
+    classpath sourceSets.obfuscator.runtimeClasspath
+    main 'net.corda.core.internal.utilities.TestResourceWriter'
+    args new File(sourceSets.test.resources.srcDirs.first(), "zip").toString()
+}

core/src/obfuscator/kotlin/net/corda/core/internal/utilities/TestResourceWriter.kt

@@ -0,0 +1,54 @@
+package net.corda.core.internal.utilities
+
+import net.corda.core.obfuscator.XorOutputStream
+import java.net.URL
+import java.nio.file.Files
+import java.nio.file.Paths
+import java.util.zip.ZipEntry
+import java.util.zip.ZipOutputStream
+
+object TestResourceWriter {
+
+    private val externalZipBombUrls = arrayOf(
+        URL("https://www.bamsoftware.com/hacks/zipbomb/zbsm.zip"),
+        URL("https://www.bamsoftware.com/hacks/zipbomb/zblg.zip"),
+        URL("https://www.bamsoftware.com/hacks/zipbomb/zbxl.zip")
+    )
+
+    @JvmStatic
+    @Suppress("NestedBlockDepth", "MagicNumber")
+    fun main(vararg args : String) {
+        for(arg in args) {
+            /**
+             * Download zip bombs
+             */
+            for(url in externalZipBombUrls) {
+                url.openStream().use { inputStream ->
+                    val destination = Paths.get(arg).resolve(Paths.get(url.path +  ".xor").fileName)
+                    Files.newOutputStream(destination).buffered().let(::XorOutputStream).use { outputStream ->
+                        inputStream.copyTo(outputStream)
+                    }
+                }
+            }
+            /**
+             * Create a jar archive with a huge manifest file, used in unit tests to check that it is also identified as a zip bomb.
+             * This is because {@link java.util.jar.JarInputStream}
+             * <a href="https://github.com/openjdk/jdk/blob/4dedba9ebe11750f4b39c41feb4a4314ccdb3a08/src/java.base/share/classes/java/util/jar/JarInputStream.java#L95">eagerly loads the manifest file in memory</a>
+             * which would make such a jar dangerous if used as an attachment
+             */
+            val destination = Paths.get(arg).resolve(Paths.get("big-manifest.jar.xor").fileName)
+            ZipOutputStream(XorOutputStream((Files.newOutputStream(destination).buffered()))).use { zos ->
+                val zipEntry = ZipEntry("MANIFEST.MF")
+                zipEntry.method = ZipEntry.DEFLATED
+                zos.putNextEntry(zipEntry)
+                val buffer = ByteArray(0x100000) { 0x0 }
+                var written = 0L
+                while(written < 10_000_000_000) {
+                    zos.write(buffer)
+                    written += buffer.size
+                }
+                zos.closeEntry()
+            }
+        }
+    }
+}

core/src/obfuscator/kotlin/net/corda/core/obfuscator/XorInputStream.kt

@@ -0,0 +1,30 @@
+package net.corda.core.obfuscator
+
+import java.io.FilterInputStream
+import java.io.InputStream
+
+@Suppress("MagicNumber")
+class XorInputStream(private val source : InputStream) : FilterInputStream(source) {
+    var prev : Int = 0
+
+    override fun read(): Int {
+        prev = source.read() xor prev
+        return prev - 0x80
+    }
+
+    override fun read(buffer: ByteArray): Int {
+        return read(buffer, 0, buffer.size)
+    }
+
+    override fun read(buffer: ByteArray, off: Int, len: Int): Int {
+        var read = 0
+        while(true) {
+            val b = source.read()
+            if(b < 0) break
+            buffer[off + read++] = ((b xor prev) - 0x80).toByte()
+            prev = b
+            if(read == len) break
+        }
+        return read
+    }
+}

core/src/obfuscator/kotlin/net/corda/core/obfuscator/XorOutputStream.kt

@@ -0,0 +1,30 @@
+package net.corda.core.obfuscator
+
+import java.io.FilterOutputStream
+import java.io.OutputStream
+
+@Suppress("MagicNumber")
+class XorOutputStream(private val destination : OutputStream) : FilterOutputStream(destination) {
+    var prev : Int = 0
+
+    override fun write(byte: Int) {
+        val b = (byte + 0x80) xor prev
+        destination.write(b)
+        prev = b
+    }
+
+    override fun write(buffer: ByteArray) {
+        write(buffer, 0, buffer.size)
+    }
+
+    override fun write(buffer: ByteArray, off: Int, len: Int) {
+        var written = 0
+        while(true) {
+            val b = (buffer[written] + 0x80) xor prev
+            destination.write(b)
+            prev = b
+            ++written
+            if(written == len) break
+        }
+    }
+}

core/src/test/README.md

@@ -7,6 +7,18 @@ The Corda core module defines a lot of types and helpers that can only be exerci
 the context of a node. However, as everything else depends on the core module, we cannot pull the node into
 this module. Therefore, any tests that require further Corda dependencies need to be defined in the module
  `core-tests`, which has the full set of dependencies including `node-driver`.
- 
 
+# ZipBomb tests
+
+There is a unit test that checks the zip bomb detector in `net.corda.core.internal.utilities.ZipBombDetector` works correctly. 
+This test (`core/src/test/kotlin/net/corda/core/internal/utilities/ZipBombDetectorTest.kt`) uses real zip bombs, provided by `https://www.bamsoftware.com/hacks/zipbomb/`.
+As it is undesirable to have unit test depends on external internet resources we do not control, those files are included as resources in
+`core/src/test/resources/zip/`, however some Windows antivirus software correctly identifies those files as zip bombs, 
+raising an alert to the user. To mitigate this, those files have been obfuscated using `net.corda.core.obfuscator.XorOutputStream` 
+(which simply XORs every byte of the file with the previous one, except for the first byte that is XORed with zero) 
+to prevent antivirus software from detecting them as zip bombs and are de-obfuscated on the fly in unit tests using 
+`net.corda.core.obfuscator.XorInputStream`. 
+
+There is a dedicated Gradle task to re-download and re-obfuscate all the test resource files named `writeTestResources`, 
+its source code is in `core/src/obfuscator/kotlin/net/corda/core/internal/utilities/TestResourceWriter.kt` 
  

core/src/test/kotlin/net/corda/core/internal/utilities/ZipBombDetectorTest.kt

@@ -1,5 +1,6 @@
 package net.corda.core.internal.utilities
 
+import net.corda.core.obfuscator.XorInputStream
 import org.junit.Assert
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -22,28 +23,28 @@ class ZipBombDetectorTest(private val case : TestCase) {
         // so the total uncompressed size doesn't exceed maxUncompressedSize
         SMALL_BOMB(
                 "A large (5.5 GB) zip archive",
-                "zip/zbsm.zip", 64_000_000, 10f, false),
+                "zip/zbsm.zip.xor", 64_000_000, 10f, false),
 
         // Decreasing maxUncompressedSize leads to a successful detection
         SMALL_BOMB2(
                 "A large (5.5 GB) zip archive, with 1MB maxUncompressedSize",
-                "zip/zbsm.zip", 1_000_000, 10f, true),
+                "zip/zbsm.zip.xor", 1_000_000, 10f, true),
 
         // ZipInputStream is also unable to read all entries of zblg.zip, but since the first one is already bigger than 4GB,
         // that is enough to exceed maxUncompressedSize
         LARGE_BOMB(
             "A huge (281 TB) Zip bomb, this is the biggest possible non-recursive non-Zip64 archive",
-            "zip/zblg.zip", 64_000_000, 10f, true),
+            "zip/zblg.zip.xor", 64_000_000, 10f, true),
 
         //Same for this, but its entries are 22GB each
         EXTRA_LARGE_BOMB(
             "A humongous (4.5 PB) Zip64 bomb",
-            "zip/zbxl.zip", 64_000_000, 10f, true),
+            "zip/zbxl.zip.xor", 64_000_000, 10f, true),
 
         //This is a jar file containing a single 10GB manifest
         BIG_MANIFEST(
                 "A jar file with a huge manifest",
-                "zip/big-manifest.jar", 64_000_000, 10f, true);
+                "zip/big-manifest.jar.xor", 64_000_000, 10f, true);
 
         override fun toString() = description
     }
@@ -51,7 +52,7 @@ class ZipBombDetectorTest(private val case : TestCase) {
     companion object {
         @JvmStatic
         @Parameterized.Parameters(name = "{0}")
-        fun primeNumbers(): Collection<*> {
+        fun generateTestCases(): Collection<*> {
             return TestCase.values().toList()
         }
     }
@@ -59,7 +60,10 @@ class ZipBombDetectorTest(private val case : TestCase) {
     @Test(timeout=10_000)
     fun test() {
         (javaClass.classLoader.getResourceAsStream(case.zipResource) ?:
-            throw IllegalStateException("Missing test resource file ${case.zipResource}")).let {
+            throw IllegalStateException("Missing test resource file ${case.zipResource}"))
+                .buffered()
+                .let(::XorInputStream)
+                .let {
             Assert.assertEquals(case.expectedOutcome, ZipBombDetector.scanZip(it, case.maxUncompressedSize, case.maxCompressionRatio))
         }
     }

core/src/test/kotlin/net/corda/core/obfuscator/XorStreamTest.kt

@@ -0,0 +1,50 @@
+package net.corda.core.obfuscator
+
+import org.junit.Assert
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
+import java.security.DigestInputStream
+import java.security.DigestOutputStream
+import java.security.MessageDigest
+import java.util.*
+
+@RunWith(Parameterized::class)
+class XorStreamTest(private val size : Int) {
+    private val random = Random(0)
+
+    companion object {
+        @JvmStatic
+        @Parameterized.Parameters(name = "{0}")
+        fun generateTestCases(): Collection<*> {
+            return listOf(0, 16, 31, 127, 1000, 1024)
+        }
+    }
+
+    @Test(timeout = 5000)
+    fun test() {
+        val baos = ByteArrayOutputStream(size)
+        val md = MessageDigest.getInstance("MD5")
+        val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
+        DigestOutputStream(XorOutputStream(baos), md).use { outputStream ->
+            var written = 0
+            while(written < size) {
+                random.nextBytes(buffer)
+                val bytesToWrite = (size - written).coerceAtMost(buffer.size)
+                outputStream.write(buffer, 0, bytesToWrite)
+                written += bytesToWrite
+            }
+        }
+        val digest = md.digest()
+        md.reset()
+        DigestInputStream(XorInputStream(ByteArrayInputStream(baos.toByteArray())), md).use { inputStream ->
+            while(true) {
+                val read = inputStream.read(buffer)
+                if(read <= 0) break
+            }
+        }
+        Assert.assertArrayEquals(digest, md.digest())
+    }
+}