Generalise support for file uploads over HTTP to allow reuse of the upload servlet.

docs/source/node-administration.rst

@@ -15,7 +15,7 @@ you can upload it by running this command from a UNIX terminal:
 
 .. sourcecode:: shell
 
-   curl -F myfile=@path/to/my/file.zip http://localhost:31338/attachments/upload
+   curl -F myfile=@path/to/my/file.zip http://localhost:31338/upload/attachment
 
 The attachment will be identified by the SHA-256 hash of the contents, which you can get by doing:
 
@@ -23,8 +23,8 @@ The attachment will be identified by the SHA-256 hash of the contents, which you
 
    shasum -a 256 file.zip
 
-on a Mac or by using ``sha256sum`` on Linux. Alternatively, check the node logs. There is presently no way to manage
+on a Mac or by using ``sha256sum`` on Linux. Alternatively, the hash will be returned to you when you upload the
-attachments from a GUI.
+attachment.
 
 An attachment may be downloaded by fetching:
 

src/main/kotlin/core/node/AbstractNode.kt

@@ -48,6 +48,11 @@ abstract class AbstractNode(val dir: Path, val configuration: NodeConfiguration,
     // low-performance prototyping period.
     protected open val serverThread = Executors.newSingleThreadExecutor()
 
+    // Objects in this list will be scanned by the DataUploadServlet and can be handed new data via HTTP.
+    // Don't mutate this after startup.
+    protected val _servicesThatAcceptUploads = ArrayList<AcceptsFileUpload>()
+    val servicesThatAcceptUploads: List<AcceptsFileUpload> = _servicesThatAcceptUploads
+
     val services = object : ServiceHub {
         override val networkService: MessagingService get() = net
         override val networkMapService: NetworkMap = MockNetworkMap()
@@ -85,7 +90,9 @@ abstract class AbstractNode(val dir: Path, val configuration: NodeConfiguration,
 
     open fun start(): AbstractNode {
         log.info("Node starting up ...")
+
         storage = initialiseStorageService(dir)
+
         net = makeMessagingService()
         smm = StateMachineManager(services, serverThread)
         wallet = NodeWalletService(services)
@@ -130,6 +137,7 @@ abstract class AbstractNode(val dir: Path, val configuration: NodeConfiguration,
 
     protected open fun initialiseStorageService(dir: Path): StorageService {
         val attachments = makeAttachmentStorage(dir)
+        _servicesThatAcceptUploads += attachments
         val (identity, keypair) = obtainKeyPair(dir)
         return constructStorageService(attachments, identity, keypair)
     }
@@ -195,7 +203,6 @@ abstract class AbstractNode(val dir: Path, val configuration: NodeConfiguration,
             Files.createDirectory(attachmentsDir)
         } catch (e: FileAlreadyExistsException) {
         }
-        val attachments = NodeAttachmentService(attachmentsDir)
+        return NodeAttachmentService(attachmentsDir)
-        return attachments
     }
 }

src/main/kotlin/core/node/AcceptsFileUpload.kt

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2015 Distributed Ledger Group LLC.  Distributed as Licensed Company IP to DLG Group Members
+ * pursuant to the August 7, 2015 Advisory Services Agreement and subject to the Company IP License terms
+ * set forth therein.
+ *
+ * All other rights reserved.
+ */
+
+package core.node
+
+import java.io.InputStream
+
+/**
+ * A service that implements AcceptsFileUpload can have new binary data provided to it via an HTTP upload.
+ *
+ * TODO: In future, also accept uploads over the MQ interface too.
+ */
+interface AcceptsFileUpload {
+    /** A string that prefixes the URLs, e.g. "attachments" or "interest-rates". Should be OK for URLs. */
+    val dataTypePrefix: String
+
+    /** What file extensions are acceptable for the file to be handed to upload() */
+    val acceptableFileExtensions: List<String>
+
+    /**
+     * Accepts the data in the given input stream, and returns some sort of useful return message that will be sent
+     * back to the user in the response.
+     */
+    fun upload(data: InputStream): String
+}

src/main/kotlin/core/node/Node.kt

@@ -13,7 +13,7 @@ import core.messaging.LegallyIdentifiableNode
 import core.messaging.MessagingService
 import core.node.services.ArtemisMessagingService
 import core.node.servlets.AttachmentDownloadServlet
-import core.node.servlets.AttachmentUploadServlet
+import core.node.servlets.DataUploadServlet
 import core.utilities.loggerFor
 import org.eclipse.jetty.server.Server
 import org.eclipse.jetty.servlet.ServletContextHandler
@@ -59,8 +59,8 @@ class Node(dir: Path, val p2pAddr: HostAndPort, configuration: NodeConfiguration
         val port = p2pAddr.port + 1   // TODO: Move this into the node config file.
         val server = Server(port)
         val handler = ServletContextHandler()
-        handler.setAttribute("storage", storage)
+        handler.setAttribute("node", this)
-        handler.addServlet(AttachmentUploadServlet::class.java, "/attachments/upload")
+        handler.addServlet(DataUploadServlet::class.java, "/upload/*")
         handler.addServlet(AttachmentDownloadServlet::class.java, "/attachments/*")
         server.handler = handler
         server.start()

src/main/kotlin/core/node/services/NodeAttachmentService.kt

@@ -15,6 +15,7 @@ import com.google.common.io.CountingInputStream
 import core.Attachment
 import core.crypto.SecureHash
 import core.extractZipFile
+import core.node.AcceptsFileUpload
 import core.utilities.loggerFor
 import java.io.FilterInputStream
 import java.io.InputStream
@@ -30,7 +31,7 @@ import javax.annotation.concurrent.ThreadSafe
  * Stores attachments in the specified local directory, which must exist. Doesn't allow new attachments to be uploaded.
  */
 @ThreadSafe
-class NodeAttachmentService(val storePath: Path) : AttachmentStorage {
+class NodeAttachmentService(val storePath: Path) : AttachmentStorage, AcceptsFileUpload {
     private val log = loggerFor<NodeAttachmentService>()
 
     @VisibleForTesting
@@ -140,4 +141,9 @@ class NodeAttachmentService(val storePath: Path) : AttachmentStorage {
             }
         }
     }
+
+    // Implementations for AcceptsFileUpload
+    override val dataTypePrefix = "attachment"
+    override val acceptableFileExtensions = listOf(".jar", ".zip")
+    override fun upload(data: InputStream) = importAttachment(data).toString()
 }

src/main/kotlin/core/node/servlets/DataUploadServlet.kt

@@ -8,8 +8,8 @@
 
 package core.node.servlets
 
-import core.crypto.SecureHash
+import core.node.AcceptsFileUpload
-import core.node.services.StorageService
+import core.node.Node
 import core.utilities.loggerFor
 import org.apache.commons.fileupload.servlet.ServletFileUpload
 import java.util.*
@@ -17,43 +17,55 @@ import javax.servlet.http.HttpServlet
 import javax.servlet.http.HttpServletRequest
 import javax.servlet.http.HttpServletResponse
 
-class AttachmentUploadServlet : HttpServlet() {
+/**
-    private val log = loggerFor<AttachmentUploadServlet>()
+ * Accepts binary streams, finds the right [AcceptsFileUpload] implementor and hands the stream off to it.
+ */
+class DataUploadServlet : HttpServlet() {
+    private val log = loggerFor<DataUploadServlet>()
 
     override fun doPost(req: HttpServletRequest, resp: HttpServletResponse) {
+        val node = servletContext.getAttribute("node") as Node
+
         @Suppress("DEPRECATION")    // Bogus warning due to superclass static method being deprecated.
         val isMultipart = ServletFileUpload.isMultipartContent(req)
 
         if (!isMultipart) {
-            log.error("Got a non-file upload request to the attachments servlet")
+            resp.sendError(HttpServletResponse.SC_BAD_REQUEST, "This end point is for data uploads only.")
-            resp.sendError(HttpServletResponse.SC_BAD_REQUEST, "This end point is for file uploads only.")
+            return
+        }
+
+        val acceptor: AcceptsFileUpload? = findAcceptor(node, req)
+        if (acceptor == null) {
+            resp.sendError(HttpServletResponse.SC_BAD_REQUEST, "Got a file upload request for an unknown data type")
             return
         }
 
         val upload = ServletFileUpload()
         val iterator = upload.getItemIterator(req)
-        val ids = ArrayList<SecureHash>()
+        val messages = ArrayList<String>()
         while (iterator.hasNext()) {
             val item = iterator.next()
-            if (!item.name.endsWith(".jar")) {
+            if (!acceptor.acceptableFileExtensions.any { item.name.endsWith(it) }) {
-                log.error("Attempted upload of a non-JAR attachment: mime=${item.contentType} filename=${item.name}")
                 resp.sendError(HttpServletResponse.SC_BAD_REQUEST,
-                        "${item.name}: Must be have a MIME type of application/java-archive and a filename ending in .jar")
+                        "${item.name}: Must be have a filename ending in one of: ${acceptor.acceptableFileExtensions}")
                 return
             }
 
             log.info("Receiving ${item.name}")
 
-            val storage = servletContext.getAttribute("storage") as StorageService
             item.openStream().use {
-                val id = storage.attachments.importAttachment(it)
+                val message = acceptor.upload(it)
-                log.info("${item.name} successfully inserted into the attachment store with id $id")
+                log.info("${item.name} successfully accepted: $message")
-                ids += id
+                messages += message
             }
         }
 
         // Send back the hashes as a convenience for the user.
         val writer = resp.writer
-        ids.forEach { writer.println(it) }
+        messages.forEach { writer.println(it) }
+    }
+
+    private fun findAcceptor(node: Node, req: HttpServletRequest): AcceptsFileUpload? {
+        return node.servicesThatAcceptUploads.firstOrNull { req.pathInfo.substring(1).substringBefore('/') == it.dataTypePrefix }
     }
 }