NOTICK - Add C++ Serialiser Support for Maps and Arrays (#5775)

Adds support for understanding both Maps and Arrays

Irritatingly, whilst arrays are mostly serialized as lists, we cannot simply use a restricted List reader to deserialize them because there are subtle differences in the way we need to work out if its actually a restricted type or not. Rather than add a bunch of random logic into the factory method I've isolated it in the class hierarchy. So a little bit more code makes the implementations a lot neater. We also need to deal with the fact arras of unboxed primitives exist, which whilst Java really gets excited about, we don't need to care about. An int, is an int, is an int!.

Map support required we add a slightly better Value dumper, essentially the "key" component of the KV pair needs to be more flexible than a simple string when we're dumping out param:value pairs.

Testing
Added a lot more unit tests to both the ordered type notation code to ensure we build up the schema dependency struct in the correct order. Quite important as we rely on that in the composite factory to be strictly ordered to ensure we're not constructing a reader for a type we don't yet understand... and there were some small bugs in the version that predates this PR

Also added a lot higher level tests to ensure actual reading out of the blob works

experimental/cpp-serializer/bin/blob-inspector/.gitignore

@@ -1 +1,3 @@
 blob-inspector
+
+*.a

experimental/cpp-serializer/bin/blob-inspector/BlobInspector.cxx

@@ -0,0 +1,73 @@
+#include "BlobInspector.h"
+#include "CordaBytes.h"
+
+#include <iostream>
+#include <sstream>
+
+#include "proton/codec.h"
+#include "proton/proton_wrapper.h"
+
+#include "amqp/schema/descriptors/AMQPDescriptorRegistory.h"
+
+#include "amqp/CompositeFactory.h"
+#include "amqp/schema/described-types/Envelope.h"
+
+/******************************************************************************/
+
+BlobInspector::BlobInspector (CordaBytes & cb_)
+    : m_data { pn_data (cb_.size()) }
+{
+    // returns how many bytes we processed which right now we don't care
+    // about but I assume there is a case where it doesn't process the
+    // entire file
+    auto rtn = pn_data_decode (m_data, cb_.bytes(), cb_.size());
+    assert (rtn == cb_.size());
+}
+
+/******************************************************************************/
+
+std::string
+BlobInspector::dump() {
+    std::unique_ptr<amqp::internal::schema::Envelope> envelope;
+
+    if (pn_data_is_described (m_data)) {
+        proton::auto_enter p (m_data);
+
+        auto a = pn_data_get_ulong(m_data);
+
+        envelope.reset (
+                dynamic_cast<amqp::internal::schema::Envelope *> (
+                        amqp::internal::AMQPDescriptorRegistory[a]->build(m_data).release()));
+    }
+
+    amqp::internal::CompositeFactory cf;
+
+    cf.process (envelope->schema());
+
+    auto reader = cf.byDescriptor (envelope->descriptor());
+    assert (reader);
+
+    {
+        // move to the actual blob entry in the tree - ideally we'd have
+        // saved this on the Envelope but that's not easily doable as we
+        // can't grab an actual copy of our data pointer
+        proton::auto_enter p (m_data);
+        pn_data_next (m_data);
+        proton::is_list (m_data);
+        assert (pn_data_get_list (m_data) == 3);
+        {
+            proton::auto_enter p (m_data);
+
+            std::stringstream ss;
+
+            // We wrap our output like this to make sure it's valid JSON to
+            // facilitate easy pretty printing
+            ss << reader->dump ("{ Parsed", m_data, envelope->schema())->dump()
+               << " }";
+
+            return ss.str();
+        }
+    }
+}
+
+/******************************************************************************/

experimental/cpp-serializer/bin/blob-inspector/BlobInspector.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <iosfwd>
+#include "CordaBytes.h"
+
+/******************************************************************************/
+
+struct pn_data_t;
+
+/******************************************************************************/
+
+class BlobInspector {
+    private :
+        pn_data_t * m_data;
+
+    public :
+        BlobInspector (CordaBytes &);
+
+        std::string dump();
+
+};
+
+/******************************************************************************/

experimental/cpp-serializer/bin/blob-inspector/CMakeLists.txt

@@ -4,6 +4,18 @@ include_directories (${BLOB-INSPECTOR_SOURCE_DIR}/src/amqp)
 link_directories (${BLOB-INSPECTOR_BINARY_DIR}/src/amqp)
 link_directories (${BLOB-INSPECTOR_BINARY_DIR}/src/proton)
 
-add_executable (blob-inspector main)
+set (blob-inspector-sources
+        BlobInspector.cxx
+        CordaBytes.cxx)
+
+
+add_executable (blob-inspector main.cxx ${blob-inspector-sources})
 
 target_link_libraries (blob-inspector amqp proton qpid-proton)
+
+#
+# Unit tests for the blob inspector. For this to work we also need to create
+# a linkable library from the code here to link into our test.
+#
+add_library (blob-inspector-lib ${blob-inspector-sources} )
+ADD_SUBDIRECTORY (test)

experimental/cpp-serializer/bin/blob-inspector/CordaBytes.cxx

@@ -0,0 +1,38 @@
+#include "CordaBytes.h"
+
+#include <array>
+#include <sys/stat.h>
+#include "amqp/AMQPHeader.h"
+
+/******************************************************************************/
+
+CordaBytes::CordaBytes (const std::string & file_)
+    : m_blob { nullptr }
+{
+    std::ifstream file { file_, std::ios::in | std::ios::binary };
+    struct stat results { };
+
+    if (::stat(file_.c_str(), &results) != 0) {
+        throw std::runtime_error ("Not a file");
+    }
+
+    // Disregard the Corda header
+    m_size = results.st_size - (amqp::AMQP_HEADER.size() + 1);
+
+    std::array<char, 7> header { };
+    file.read (header.data(), 7);
+
+    if (header != amqp::AMQP_HEADER) {
+        throw std::runtime_error ("Not a Corda stream");
+    }
+
+    file.read (reinterpret_cast<char *>(&m_encoding), 1);
+
+    m_blob = new char[m_size];
+
+    memset (m_blob, 0, m_size);
+    file.read (m_blob, m_size);
+}
+
+/******************************************************************************/
+

experimental/cpp-serializer/bin/blob-inspector/CordaBytes.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include "string"
+#include <fstream>
+#include "amqp/AMQPSectionId.h"
+
+/******************************************************************************/
+
+class CordaBytes {
+    private :
+        amqp::amqp_section_id_t m_encoding;
+        size_t m_size;
+        char * m_blob;
+
+    public :
+        explicit CordaBytes (const std::string &);
+
+        ~CordaBytes() {
+            delete [] m_blob;
+        }
+
+        const decltype (m_encoding) & encoding() const {
+            return m_encoding;
+        }
+
+        decltype (m_size) size() const { return m_size; }
+
+        const char * const bytes() const { return m_blob; }
+};
+
+/******************************************************************************/

experimental/cpp-serializer/bin/blob-inspector/main.cxx

@@ -9,74 +9,18 @@
 #include <proton/codec.h>
 #include <sys/stat.h>
 
-#import "debug.h"
+#include "debug.h"
 
 #include "proton/proton_wrapper.h"
 
 #include "amqp/AMQPHeader.h"
 #include "amqp/AMQPSectionId.h"
-#include "amqp/descriptors/AMQPDescriptorRegistory.h"
+#include "amqp/schema/descriptors/AMQPDescriptorRegistory.h"
 
-#include "amqp/schema/Envelope.h"
+#include "amqp/schema/described-types/Envelope.h"
 #include "amqp/CompositeFactory.h"
-
-/******************************************************************************/
-
-void
-data_and_stop(std::ifstream & f_, ssize_t sz) {
-    char * blob = new char[sz];
-    memset (blob, 0, sz);
-    f_.read(blob, sz);
-
-    pn_data_t * d = pn_data(sz);
-
-    // returns how many bytes we processed which right now we don't care
-    // about but I assume there is a case where it doesn't process the
-    // entire file
-    auto rtn = pn_data_decode (d, blob, sz);
-    assert (rtn == sz);
-
-    std::unique_ptr<amqp::internal::schema::Envelope> envelope;
-
-    if (pn_data_is_described(d)) {
-        proton::auto_enter p (d);
-
-        auto a = pn_data_get_ulong(d);
-
-        envelope.reset (
-            dynamic_cast<amqp::internal::schema::Envelope *> (
-                amqp::AMQPDescriptorRegistory[a]->build(d).release()));
-
-        DBG (std::endl << "Types in schema: " << std::endl
-            << *envelope << std::endl); // NOLINT
-    }
-
-    amqp::internal::CompositeFactory cf;
-
-    cf.process (envelope->schema());
-
-    auto reader = cf.byDescriptor (envelope->descriptor());
-    assert (reader);
-
-    {
-        // move to the actual blob entry in the tree - ideally we'd have
-        // saved this on the Envelope but that's not easily doable as we
-        // can't grab an actual copy of our data pointer
-        proton::auto_enter p (d);
-        pn_data_next (d);
-        proton::is_list (d);
-        assert (pn_data_get_list (d) == 3);
-        {
-            proton::auto_enter p (d);
-
-            // We wrap our output like this to make sure it's valid JSON to
-            // facilitate easy pretty printing
-            std::cout
-                << reader->dump ("{ Parsed", d, envelope->schema())->dump()
-                << " }" << std::endl;
-        }
-    }
-}
+#include "CordaBytes.h"
+#include "BlobInspector.h"
 
 /******************************************************************************/
 
@@ -88,22 +32,14 @@ main (int argc, char **argv) {
         return EXIT_FAILURE;
     }
 
-    std::ifstream f (argv[1], std::ios::in | std::ios::binary);
-    std::array<char, 7> header { };
-    f.read(header.data(), 7);
-
-    if (header != amqp::AMQP_HEADER) {
-        std::cerr << "Bad Header in blob" << std::endl;
-        return EXIT_FAILURE;
-    }
-
-    amqp::amqp_section_id_t encoding;
-    f.read((char *)&encoding, 1);
-
-    if (encoding == amqp::DATA_AND_STOP) {
-        data_and_stop(f, results.st_size - 8);
+    CordaBytes cb (argv[1]);
+    
+    if (cb.encoding() == amqp::DATA_AND_STOP) {
+        BlobInspector blobInspector (cb);
+        auto val = blobInspector.dump();
+        std::cout << val << std::endl;
     } else {
-        std::cerr << "BAD ENCODING " << encoding << " != "
+        std::cerr << "BAD ENCODING " << cb.encoding() << " != "
             << amqp::DATA_AND_STOP << std::endl;
 
         return EXIT_FAILURE;

experimental/cpp-serializer/bin/blob-inspector/test/.gitignore

@@ -0,0 +1 @@
+blob-inspector-test

experimental/cpp-serializer/bin/blob-inspector/test/CMakeLists.txt

@@ -0,0 +1,17 @@
+set (EXE "blob-inspector-test")
+
+set (blob-inspector-test-sources
+        main.cxx
+        blob-inspector-test.cxx
+)
+
+link_directories (${BLOB-INSPECTOR_BINARY_DIR}/bin/blob-inspector)
+include_directories (${BLOB-INSPECTOR_BINARY_DIR}/bin/blob-inspector)
+
+add_executable (${EXE} ${blob-inspector-test-sources})
+
+target_link_libraries (${EXE} gtest amqp blob-inspector-lib)
+
+if (UNIX)
+    target_link_libraries (${EXE} pthread qpid-proton proton)
+endif (UNIX)

experimental/cpp-serializer/bin/blob-inspector/test/blob-inspector-test.cxx

@@ -0,0 +1,171 @@
+#include <gtest/gtest.h>
+#include "CordaBytes.h"
+#include "BlobInspector.h"
+
+const std::string filepath ("../../test-files/"); // NOLINT
+
+/******************************************************************************
+ *
+ * mapType Tests
+ *
+ ******************************************************************************/
+
+void
+test (const std::string & file_, const std::string & result_) {
+    auto path { filepath + file_ } ;
+    CordaBytes cb (path);
+    auto val = BlobInspector (cb).dump();
+    ASSERT_EQ(result_, val);
+}
+
+/******************************************************************************/
+
+/**
+ * int
+ */
+TEST (BlobInspector, _i_) { // NOLINT
+    test ("_i_", "{ Parsed : { a : 69 } }");
+}
+
+/******************************************************************************/
+
+/**
+ * long
+ */
+TEST (BlobInspector, _l_) { // NOLINT
+    test ("_l_", "{ Parsed : { x : 100000000000 } }");
+}
+
+/******************************************************************************/
+
+/**
+ * int
+ */
+TEST (BlobInspector, _Oi_) { // NOLINT
+    test ("_Oi_", "{ Parsed : { a : 1 } }");
+}
+
+/******************************************************************************/
+
+/**
+ * int
+ */
+TEST (BlobInspector, _Ai_) { // NOLINT
+    test ("_Ai_", "{ Parsed : { z : [ 1, 2, 3, 4, 5, 6 ] } }");
+}
+
+/******************************************************************************/
+
+/**
+ * List of ints
+ */
+TEST (BlobInspector, _Li_) { // NOLINT
+    test ("_Li_", "{ Parsed : { a : [ 1, 2, 3, 4, 5, 6 ] } }");
+}
+
+/******************************************************************************/
+
+/**
+ * List of a class with a single int property
+ */
+TEST (BlobInspector, _L_i__) { // NOLINT
+    test (
+        "_L_i__",
+        "{ Parsed : { listy : [ { a : 1 }, { a : 2 }, { a : 3 } ] } }");
+}
+
+/******************************************************************************/
+
+TEST (BlobInspector, _Le_) { // NOLINT
+    test ("_Le_", "{ Parsed : { listy : [ A, B, C ] } }");
+}
+
+/******************************************************************************/
+
+TEST (BlobInspector,_Le_2) { // NOLINT
+    EXPECT_THROW (
+        {
+            test ("_Le_2", "");
+        },
+        std::runtime_error);
+}
+
+/******************************************************************************/
+
+/**
+ * A map of ints to strings
+ */
+TEST (BlobInspector, _Mis_) { // NOLINT
+    test ("_Mis_",
+        R"({ Parsed : { a : { 1 : "two", 3 : "four", 5 : "six" } } })");
+}
+
+/******************************************************************************/
+
+/**
+ * A map of ints to lists of Strings
+ */
+TEST (BlobInspector, _MiLs_) { // NOLINT
+    test ("_MiLs_",
+        R"({ Parsed : { a : { 1 : [ "two", "three", "four" ], 5 : [ "six" ], 7 : [  ] } } })");
+}
+
+/******************************************************************************/
+
+/**
+ * a map of ints to a composite with a n int and string property
+ */
+TEST (BlobInspector, _Mi_is__) { // NOLINT
+    test ("_Mi_is__",
+        R"({ Parsed : { a : { 1 : { a : 2, b : "three" }, 4 : { a : 5, b : "six" }, 7 : { a : 8, b : "nine" } } } })");
+}
+
+/******************************************************************************/
+
+TEST (BlobInspector,_Pls_) { // NOLINT
+    test ("_Pls_",
+            R"({ Parsed : { a : { first : 1, second : "two" } } })");
+}
+
+/******************************************************************************/
+
+TEST (BlobInspector, _e_) { // NOLINT
+    test ("_e_", "{ Parsed : { e : A } }");
+}
+
+/******************************************************************************/
+
+TEST (BlobInspector, _i_is__) { // NOLINT
+    test ("_i_is__",
+            R"({ Parsed : { a : 1, b : { a : 2, b : "three" } } })");
+}
+
+/******************************************************************************/
+
+// Array of unboxed integers
+TEST (BlobInspector, _Ci_) { // NOLINT
+    test ("_Ci_",
+        R"({ Parsed : { z : [ 1, 2, 3 ] } })");
+}
+
+/******************************************************************************/
+
+/**
+ * Composite with
+ *   * one int property
+ *   * one long property
+ *   * one list property that is a list of Maps of int to strings
+ */
+TEST (BlobInspector, __i_LMis_l__) { // NOLINT
+    test ("__i_LMis_l__",
+        R"({ Parsed : { x : [ { 1 : "two", 3 : "four", 5 : "six" }, { 7 : "eight", 9 : "ten" } ], y : { x : 1000000 }, z : { a : 666 } } })");
+}
+
+/******************************************************************************/
+
+TEST (BlobInspector, _ALd_) { // NOLINT
+    test ("_ALd_",
+            R"({ Parsed : { a : [ [ 10.100000, 11.200000, 12.300000 ], [  ], [ 13.400000 ] ] } })");
+}
+
+/******************************************************************************/

experimental/cpp-serializer/bin/blob-inspector/test/main.cxx

@@ -0,0 +1,7 @@
+#include <gtest/gtest.h>
+
+int
+main (int argc, char ** argv){
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}