From 468c0c7404f4f1ae551a03743ef2d772088eca4b Mon Sep 17 00:00:00 2001 From: Mike Hearn Date: Wed, 6 Jun 2018 13:57:25 +0200 Subject: [PATCH 1/4] CORDA-1349: Docs: improve docs on permissioning, doorman and network map. (#3277) * Docs: improve docs on permissioning, doorman and network map. * Add a convenience serialization API for Java users, marked as internal for now with a TODO to make it public after we start work on Corda 4.0. Otherwise serializing arbitrary objects to AMQP is awkward. --- .../net/corda/core/node/NetworkParameters.kt | 2 +- .../core/serialization/SerializationAPI.kt | 19 + docs/source/permissioning.rst | 381 +++++++++++------- .../network/NetworkParametersCopier.kt | 4 +- 4 files changed, 267 insertions(+), 139 deletions(-) diff --git a/core/src/main/kotlin/net/corda/core/node/NetworkParameters.kt b/core/src/main/kotlin/net/corda/core/node/NetworkParameters.kt index 4747c9aece..11970f62f4 100644 --- a/core/src/main/kotlin/net/corda/core/node/NetworkParameters.kt +++ b/core/src/main/kotlin/net/corda/core/node/NetworkParameters.kt @@ -19,7 +19,7 @@ import java.time.Instant * @property epoch Version number of the network parameters. Starting from 1, this will always increment on each new set * of parameters. * @property whitelistedContractImplementations List of whitelisted jars containing contract code for each contract class. - * This will be used by [net.corda.core.contracts.WhitelistedByZoneAttachmentConstraint]. Read more about contract constraints here: + * This will be used by [net.corda.core.contracts.WhitelistedByZoneAttachmentConstraint]. [You can learn more about contract constraints here](https://docs.corda.net/api-contract-constraints.html). * @property eventHorizon Time after which nodes will be removed from the network map if they have not been seen * during this period */ diff --git a/core/src/main/kotlin/net/corda/core/serialization/SerializationAPI.kt b/core/src/main/kotlin/net/corda/core/serialization/SerializationAPI.kt index 02447e9a2b..82755a9efd 100644 --- a/core/src/main/kotlin/net/corda/core/serialization/SerializationAPI.kt +++ b/core/src/main/kotlin/net/corda/core/serialization/SerializationAPI.kt @@ -1,5 +1,6 @@ package net.corda.core.serialization +import net.corda.core.CordaInternal import net.corda.core.DoNotImplement import net.corda.core.crypto.SecureHash import net.corda.core.crypto.sha256 @@ -262,6 +263,24 @@ fun T.serialize(serializationFactory: SerializationFactory = Serializa */ @Suppress("unused") class SerializedBytes(bytes: ByteArray) : OpaqueBytes(bytes) { + companion object { + /** + * Serializes the given object and returns a [SerializedBytes] wrapper for it. An alias for [Any.serialize] + * intended to make the calling smoother for Java users. + * + * TODO: Take out the @CordaInternal annotation post-Enterprise GA when we can add API again. + * + * @suppress + */ + @JvmStatic + @CordaInternal + @JvmOverloads + fun from(obj: T, serializationFactory: SerializationFactory = SerializationFactory.defaultFactory, + context: SerializationContext = serializationFactory.defaultContext): SerializedBytes { + return obj.serialize(serializationFactory, context) + } + } + // It's OK to use lazy here because SerializedBytes is configured to use the ImmutableClassSerializer. val hash: SecureHash by lazy { bytes.sha256() } } diff --git a/docs/source/permissioning.rst b/docs/source/permissioning.rst index 31ed5b28ce..8752f7ac6b 100644 --- a/docs/source/permissioning.rst +++ b/docs/source/permissioning.rst @@ -1,57 +1,55 @@ +.. highlight:: kotlin +.. raw:: html + + + + Network permissioning ===================== .. contents:: -Corda networks are *permissioned*. To connect to a network, a node needs three keystores in its -``/certificates/`` folder: +Every Corda node is a part of a network (also called a zone), and networks are *permissioned*. To connect to a +zone, a node needs a signed X.509 certificate from the network operator. Production deployments require a secure certificate authority. +The issued certificates take the form of three keystores in a node's ``/certificates/`` folder: -* ``truststore.jks``, which stores trusted public keys and certificates (in our case, those of the network root CA) +* ``network-root-truststore.jks``, which stores the network/zone operator's public keys and certificates * ``nodekeystore.jks``, which stores the node’s identity keypairs and certificates * ``sslkeystore.jks``, which stores the node’s TLS keypairs and certificates -Production deployments require a secure certificate authority. -Most production deployments will use an existing certificate authority or construct one using software that will be -made available in the coming months. Until then, the documentation below can be used to create your own certificate -authority. - -.. note:: If you are looking for information on how to connect to the existing compatibility zone go to the section: `Connecting to a compatibility zone`_ +Most users will join an existing network such as the main Corda network or the Corda TestNet. You can also build your +own networks. During development, no network is required because you can use the included tools to pre-create +and pre-distribute the certificates and map files that would normally be provided dynamically by the network. Effectively +the bootstrapper tool creates a private semi-static network for you. Certificate hierarchy --------------------- -A Corda network has four types of certificate authorities (CAs): -* The **root network CA** -* The **doorman CA** +A Corda network has three types of certificate authorities (CAs): - * The doorman CA is used instead of the root network CA for day-to-day - key signing to reduce the risk of the root network CA's private key being compromised +* The **root network CA**, that defines the extent of a compatibility zone. +* The **doorman CA**. The doorman CA is used instead of the root network CA for day-to-day key signing to reduce the + risk of the root network CA's private key being compromised. This is equivalent to an intermediate certificate + in the web PKI. +* Each node also serves as its own CA in issuing the child certificates that it uses to sign its identity keys and TLS + certificates. -* The **node CAs** +Each certificate has an X.509 extension in it that defines the certificate/key's role in the system (see below for details). +They also use X.509 name constraints to ensure that the X.500 names that encode a human meaningful identity are propagated +to all the child certificates properly. The following constraints are imposed: - * Each node serves as its own CA in issuing the child certificates that it uses to sign its identity - keys and TLS certificates +* Doorman certificates are issued by a network root. Network root certs do not contain a role extension. +* Node certificates are signed by a doorman certificate (as defined by the extension). +* Legal identity/TLS certificates are issued by a certificate marked as node CA. +* Confidential identity certificates are issued by a certificate marked as well known legal identity. +* Party certificates are marked as either a well known identity or a confidential identity. -* The **legal identity CAs** - - * Node's well-known legal identity, apart from signing transactions, can also issue certificates for confidential legal identities - -The following constraints are also imposed: - -* Doorman certificates are issued by a network root which certificate doesn't contain the extension -* Well-known service identity certificates are issued by an entity with a Doorman certificate -* Node CA certificates are issued by an entity with a Doorman certificate -* Well known legal identity/TLS certificates are issued by a certificate marked as node CA -* Confidential legal identity certificates are issued by a certificate marked as well known legal identity -* Party certificates are marked as either a well known identity or a confidential identity -* The structure of certificates above Doorman/Network map is intentionally left untouched, as they are not relevant to - the identity service and therefore there is no advantage in enforcing a specific structure on those certificates. The - certificate hierarchy consistency checks are required because nodes can issue their own certificates and can set - their own role flags on certificates, and it's important to verify that these are set consistently with the - certificate hierarchy design. As as side-effect this also acts as a secondary depth restriction on issued - certificates - -All the certificates must be issued with the custom role extension (see below). +The structure of certificates above Doorman/Network map is intentionally left untouched, as they are not relevant to +the identity service and therefore there is no advantage in enforcing a specific structure on those certificates. The +certificate hierarchy consistency checks are required because nodes can issue their own certificates and can set +their own role flags on certificates, and it's important to verify that these are set consistently with the +certificate hierarchy design. As a side-effect this also acts as a secondary depth restriction on issued +certificates. We can visualise the permissioning structure as follows: @@ -61,31 +59,27 @@ We can visualise the permissioning structure as follows: Keypair and certificate formats ------------------------------- + You can use any standard key tools to create the required public/private keypairs and certificates. The keypairs and certificates must obey the following restrictions: -* The certificates must follow the `X.509 standard `_ - - * We recommend X.509 v3 for forward compatibility - -* The TLS certificates must follow the `TLS v1.2 standard `_ - -* The root network CA, doorman CA and node CA keys, as well as the node TLS - keys, must follow one of the following schemes: - +1. The certificates must follow the `X.509v3 standard `__ +2. The TLS certificates must follow the `TLS v1.2 standard `__ +3. The root network CA, doorman CA, and node CA keys, as well as the node TLS keys, must follow one of the following schemes: * ECDSA using the NIST P-256 curve (secp256r1) - * ECDSA using the Koblitz k1 curve (secp256k1) + * RSA with 3072-bit key size or higher. - * RSA with 3072-bit key size - -.. note:: Corda's ``X509Utilities`` show how to generate the required public/private keypairs and certificates using - Bouncy Castle. You can find the ``X509Utilities`` in the `Corda repository `_, under - ``/node-api/src/main/kotlin/net/corda/nodeapi/internal/crypto/X509Utilities.kt``. +The required identity and TLS keys/certificates will be automatically generated for you by the node on first run. +However, you can also generate them manually for more control. The ``X509Utilities`` class shows how to generate the +required public/private keypairs and certificates using Bouncy Castle. You can find the ``X509Utilities`` in the `Corda +repository `__, under +``/node-api/src/main/kotlin/net/corda/nodeapi/internal/crypto/X509Utilities.kt``. Certificate role extension -------------------------- -Corda certificates have a custom X.509 v3 extension that specifies the role the certificate relates to. This extension + +Corda certificates have a custom X.509v3 extension that specifies the role the certificate relates to. This extension has the OID ``1.3.6.1.4.1.50530.1.1`` and is non-critical, so implementations outside of Corda nodes can safely ignore it. The extension contains a single ASN.1 integer identifying the identity type the certificate is for: @@ -97,98 +91,36 @@ The extension contains a single ASN.1 integer identifying the identity type the 6. Well-known legal identity 7. Confidential legal identity -In a typical installation, node administrators needn't be aware of these. However, when node certificates are managed -by external tools (such as an existing PKI solution deployed within an organisation), it is important to understand -these constraints. +In a typical installation, node administrators need not be aware of these. However, if node certificates are to be +managed by external tools, such as those provided as part of an existing PKI solution deployed within an organisation, +it is important to recognise these extensions and the constraints noted above. Certificate path validation is extended so that a certificate must contain the extension if the extension was present in the certificate of the issuer. -Creating the root and doorman CAs ---------------------------------- -Creating the root network CA's keystore and truststore -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Manually creating the node keys +------------------------------- -1. Create a new keypair +The node expects a Java-style key store (this may change in future to support PKCS#12 keystores) called ``nodekeystore.jks``, +with the private key and certificate having an alias of "cordaclientca". This certificate should be signed by the +doorman CA for your network. The basic constraints extension must be set to true. - * This will be used as the root network CA's keypair +For the TLS keys, the basic constraints extension must be set to false. The keystore name is ``sslkeystore.jks`` and +the key alias must be ``cordaclienttls``. -2. Create a self-signed certificate for the keypair. The basic constraints extension must be set to ``true`` - - * This will be used as the root network CA's certificate - -3. Create a new keystore and store the root network CA's keypair and certificate in it for later use - - * This keystore will be used by the root network CA to sign the doorman CA's certificate - -4. Create a new Java keystore named ``truststore.jks`` and store the root network CA's certificate in it using the - alias ``cordarootca`` - - * This keystore must then be provisioned to the individual nodes later so they can store it in their ``certificates`` folder - -.. warning:: The root network CA's private key should be protected and kept safe. - -Creating the doorman CA's keystore -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -1. Create a new keypair - - * This will be used as the doorman CA's keypair - -2. Obtain a certificate for the keypair signed with the root network CA key. The basic constraints extension must be - set to ``true`` - - * This will be used as the doorman CA's certificate - -3. Create a new keystore and store the doorman CA's keypair and certificate chain - (i.e. the doorman CA certificate *and* the root network CA certificate) in it for later use - - * This keystore will be used by the doorman CA to sign the nodes' identity certificates - -Creating the node CA keystores and TLS keystores ------------------------------------------------- - -Creating the node CA keystores -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -1. For each node, create a new keypair - -2. Obtain a certificate for the keypair signed with the doorman CA key. The basic constraints extension must be - set to ``true`` - -3. Create a new Java keystore named ``nodekeystore.jks`` and store the keypair in it using the alias ``cordaclientca`` - - * The node will store this keystore locally to sign its identity keys and anonymous keys - -Creating the node TLS keystores -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -1. For each node, create a new keypair - -2. Create a certificate for the keypair signed with the node CA key. The basic constraints extension must be set to - ``false`` - -3. Create a new Java keystore named ``sslkeystore.jks`` and store the key and certificates in it using the alias - ``cordaclienttls`` - - * The node will store this keystore locally to sign its TLS certificates - -Installing the certificates on the nodes ----------------------------------------- -For each node, copy the following files to the node's certificate directory (``/certificates/``): - -1. The node's ``nodekeystore.jks`` keystore -2. The node's ``sslkeystore.jks`` keystore -3. The root network CA's ``truststore.jks`` keystore +These two files should be in the node's certificate directory (``/certificates/``), along with the network's +own root certificates in a ``network-root-truststore.jks`` file. Connecting to a compatibility zone ---------------------------------- + To connect to a compatibility zone you need to register with their certificate signing authority (doorman) by submitting -a certificate signing request (CSR) to obtain a valid identity for the zone. +a certificate signing request (CSR) to obtain a valid identity for the zone. You could do this out of band, for instance +via email or a web form, but there's also a simple request/response protocol built into Corda. Before you can register, you must first have received the trust store file containing the root certificate from the zone -operator. Then run the following command: +operator. For high security zones this might be delivered physically. Then run the following command: ``java -jar corda.jar --initial-registration --network-root-truststore-password `` @@ -199,9 +131,10 @@ The certificate signing request will be created based on node information obtain The following information from the node configuration file is needed to generate the request. * **myLegalName** Your company's legal name as an X.500 string. X.500 allows differentiation between entities with the same - name as the legal name needs to be unique on the network. If another node has already been permissioned with this + name, as the legal name needs to be unique on the network. If another node has already been permissioned with this name then the permissioning server will automatically reject the request. The request will also be rejected if it - violates legal name rules, see :ref:`node_naming` for more information. + violates legal name rules, see :ref:`node_naming` for more information. You can use the X.500 schema to disambiguate + entities that have the same or similar brand names. * **emailAddress** e.g. "admin@company.com" @@ -210,14 +143,188 @@ The following information from the node configuration file is needed to generate * **networkServices or compatibilityZoneURL** The Corda compatibility zone services must be configured. This must be either: * **compatibilityZoneURL** The Corda compatibility zone network management service root URL. - * **networkServices** Replaces the ``compatibilityZoneURL`` when the Doorman and Network Map services + * **networkServices** Replaces the ``compatibilityZoneURL`` when the doorman and network map services are configured to operate on different URL endpoints. The ``doorman`` entry is used for registration. A new pair of private and public keys generated by the Corda node will be used to create the request. -The utility will submit the request to the doorman server and poll for a result periodically to retrieve the certificates. -Once the request has been approved and the certificates downloaded from the server, the node will create the keystore and trust store using the certificates and the generated private key. +The utility will submit the request to the doorman server and poll for a result periodically to retrieve the +certificates. Once the request has been approved and the certificates downloaded from the server, the node will create +the keystore and trust store using the certificates and the generated private key. -.. note:: You can exit the utility at any time if the approval process is taking longer than expected. The request process will resume on restart. +.. note:: You can exit the utility at any time if the approval process is taking longer than expected. The request + process will resume on restart as long as the ``--initial-registration`` flag is specified. This process only is needed when the node connects to the network for the first time, or when the certificate expires. + +Creating your own compatibility zone +------------------------------------ + +This section documents how to implement your own doorman and network map servers, which is the basic process required to +create a dedicated zone. At this time we do not provide tooling to do this, because the needs of each zone are different +and no generic, configurable doorman codebase has been written. + +Do you need a zone? +^^^^^^^^^^^^^^^^^^^ + +Think twice before going down this route: + +1. It isn't necessary for testing. +2. It isn't necessary for adding another layer of permissioning or 'know your customer' requirements onto your app. + +**Testing.** Creating a production-ready zone isn't necessary for testing as you can use the *network bootstrapper* +tool to create all the certificates, keys, and distribute the needed map files to run many nodes. The bootstrapper can +create a network locally on your desktop/laptop but it also knows how to automate cloud providers via their APIs and +using Docker. In this way you can bring up a simulation of a real Corda network with different nodes on different +machines in the cloud for your own testing. Testing this way has several advantages, most obviously that you avoid +race conditions in your tests caused by nodes/tests starting before all map data has propagated to all nodes. +You can read more about the reasons for the creation of the bootstrapper tool +`in a blog post on the design thinking behind Corda's network map infrastructure `__. + +**Permissioning.** And creating a zone is also unnecessary for imposing permissioning requirements beyond that of the +base Corda network. You can control who can use your app by creating a *business network*. A business network is what we +call a coalition of nodes that have chosen to run a particular app within a given commercial context. Business networks +aren't represented in the Corda API at this time, partly because the technical side is so simple. You can create one +via a simple three step process: + +1. Distribute a list of X.500 names that are members of your business network, e.g. a simple way to do this is by + hosting a text file with one name per line on your website at a fixed HTTPS URL. You could also write a simple + request/response flow that serves the list over the Corda protocol itself, although this requires the business + network to have a node for itself. +2. Write a bit of code that downloads and caches the contents of this file on disk, and which loads it into memory in + the node. A good place to do this is in a class annotated with ``@CordaService``, because this class can expose + a ``Set`` field representing the membership of your service. +3. In your flows use ``serviceHub.findService`` to get a reference to your ``@CordaService`` class, read the list of + members and at the start of each flow, throw a FlowException if the counterparty isn't in the membership list. + +In this way you can impose a centrally controlled ACL that all members will collectively enforce. + +.. note:: A production-ready Corda network and a new iteration of the testnet will be available soon. + +Why create your own zone? +^^^^^^^^^^^^^^^^^^^^^^^^^ + +The primary reason to create a zone and provide the associated infrastructure is control over *network parameters*. These +are settings that control Corda's operation, and on which all users in a network must agree. Failure to agree would create +the Corda equivalent of a blockchain "hard fork". Parameters control things like the root of identity, +how quickly users should upgrade, how long nodes can be offline before they are evicted from the system and so on. + +Creating a zone involves the following steps: + +1. Create the zone private keys and certificates. This procedure is conventional and no special knowledge is required: + any self-signed set of certificates can be used. A professional quality zone will probably keep the keys inside a + hardware security module (as the main Corda network and test networks do). +2. Write a network map server. +3. Optionally, create a doorman server. +4. Finally, you would select and generate your network parameter file. + +Writing a network map server +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This server implements a simple HTTP based protocol described in the ":doc:`network-map`" page. +The map server is responsible for gathering NodeInfo files from nodes, storing them, and distributing them back to the +nodes in the zone. By doing this it is also responsible for choosing who is in and who is out: having a signed +identity certificate is not enough to be a part of a Corda zone, you also need to be listed in the network map. +It can be thought of as a DNS equivalent. If you want to de-list a user, you would do it here. + +It is very likely that your map server won't be entirely standalone, but rather, integrated with whatever your master +user database is. + +The network map server also distributes signed network parameter files and controls the rollout schedule for when they +become available for download and opt-in, and when they become enforced. This is again a policy decision you will +probably choose to place some simple UI or workflow tooling around, in particular to enforce restrictions on who can +edit the map or the parameters. + +Writing a doorman server +^^^^^^^^^^^^^^^^^^^^^^^^ + +This step is optional because your users can obtain a signed certificate in many different ways. The doorman protocol +is again a very simple HTTP based approach in which a node creates keys and requests a certificate, polling until it +gets back what it expects. However, you could also integrate this process with the rest of your signup process. For example, +by building a tool that's integrated with your payment flow (if payment is required to take part in your zone at all). +Alternatively you may wish to distribute USB smartcard tokens that generate the private key on first use, as is typically +seen in national PKIs. There are many options. + +If you do choose to make a doorman server, the bulk of the code you write will be workflow related. For instance, +related to keeping track of an applicant as they proceed through approval. You should also impose any naming policies +you have in the doorman process. If names are meant to match identities registered in government databases then that +should be enforced here, alternatively, if names can be self-selected or anonymous, you would only bother with a +deduplication check. Again it will likely be integrated with a master user database. + +Corda does not currently provide a doorman or network map service out of the box, partly because when stripped of the +zone specific policy there isn't much to them: just a basic HTTP server that most programmers will have favourite +frameworks for anyway. + +The protocol is: + +* If $URL = ``https://some.server.com/some/path`` +* Node submits a PKCS#10 certificate signing request using HTTP POST to ``$URL/certificate``. It will have a MIME + type of ``application/octet-stream``. The ``Client-Version`` header is set to be "1.0". +* The server returns an opaque string that references this request (let's call it ``$requestid``, or an HTTP error if something went wrong. +* The returned request ID should be persisted to disk, to handle zones where approval may take a long time due to manual + intervention being required. +* The node starts polling ``$URL/$requestid`` using HTTP GET. The poll interval can be controlled by the server returning + a response with a ``Cache-Control`` header. +* If the request is answered with a ``200 OK`` response, the body is expected to be a zip file. Each file is expected to + be a binary X.509 certificate, and the certs are expected to be in order. +* If the request is answered with a ``204 No Content`` response, the node will try again later. +* If the request is answered with a ``403 Not Authorized`` response, the node will treat that as request rejection and give up. +* Other response codes will cause the node to abort with an exception. + +Setting zone parameters +^^^^^^^^^^^^^^^^^^^^^^^ + +Zone parameters are stored in a file containing a Corda AMQP serialised ``SignedDataWithCert`` +object. It is easy to create such a file with a small Java or Kotlin program. The ``NetworkParameters`` object is a +simple data holder that could be read from e.g. a config file, or settings from a database. Signing and saving the +resulting file is just a few lines of code. A full example can be found in ``NetworkParametersCopier.kt`` in the source +tree, but a flavour of it looks like this: + +.. container:: codeset + + .. sourcecode:: java + + NetworkParameters networkParameters = new NetworkParameters( + 4, // minPlatformVersion + Collections.emptyList(), // notaries + 1024 * 1024 * 20, // maxMessageSize + 1024 * 1024 * 15, // maxTransactionSize + Instant.now(), // modifiedTime + 2, // epoch + Collections.emptyMap() // whitelist + ); + CertificateAndKeyPair signingCertAndKeyPair = loadNetworkMapCA(); + SerializedBytes> bytes = SerializedBytes.from(netMapCA.sign(networkParameters)); + Files.copy(bytes.open(), Paths.get("params-file")); + + .. sourcecode:: kotlin + + val networkParameters = NetworkParameters( + minimumPlatformVersion = 4, + notaries = listOf(...), + maxMessageSize = 1024 * 1024 * 20 // 20mb, for example. + maxTransactionSize = 1024 * 1024 * 15, + modifiedTime = Instant.now(), + epoch = 2, + ... etc ... + ) + val signingCertAndKeyPair: CertificateAndKeyPair = loadNetworkMapCA() + val signedParams: SerializedBytes = signingCertAndKeyPair.sign(networkParameters).serialize() + signedParams.open().copyTo(Paths.get("/some/path")) + +Each individual parameter is documented in `the JavaDocs/KDocs for the NetworkParameters class +`__. The network map +certificate is usually chained off the root certificate, and can be created according to the instructions above. Each +time the zone parameters are changed, the epoch should be incremented. Epochs are essentially version numbers for the +parameters, and they therefore cannot go backwards. Once saved, the new parameters can be served by the network map server. + +Selecting parameter values +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +How to choose the parameters? This is the most complex question facing you as a new zone operator. Some settings may seem +straightforward and others may involve cost/benefit tradeoffs specific to your business. For example, you could choose +to run a validating notary yourself, in which case you would (in the absence of SGX) see all the users' data. Or you could +run a non-validating notary, with BFT fault tolerance, which implies recruiting others to take part in the cluster. + +New network parameters will be added over time as Corda evolves. You will need to ensure that when your users upgrade, +all the new network parameters are being served. You can ask for advice on the `corda-dev mailing list `__. \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkParametersCopier.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkParametersCopier.kt index df2e325605..ed8241153f 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkParametersCopier.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkParametersCopier.kt @@ -1,9 +1,11 @@ package net.corda.nodeapi.internal.network +import net.corda.core.internal.SignedDataWithCert import net.corda.core.internal.VisibleForTesting import net.corda.core.internal.copyTo import net.corda.core.internal.div import net.corda.core.node.NetworkParameters +import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.serialize import net.corda.nodeapi.internal.createDevNetworkMapCa import net.corda.nodeapi.internal.crypto.CertificateAndKeyPair @@ -19,7 +21,7 @@ class NetworkParametersCopier( val update: Boolean = false ) { private val copyOptions = if (overwriteFile) arrayOf(StandardCopyOption.REPLACE_EXISTING) else emptyArray() - private val serialisedSignedNetParams = signingCertAndKeyPair.sign(networkParameters).serialize() + private val serialisedSignedNetParams: SerializedBytes> = signingCertAndKeyPair.sign(networkParameters).serialize() fun install(nodeDir: Path) { val fileName = if (update) NETWORK_PARAMS_UPDATE_FILE_NAME else NETWORK_PARAMS_FILE_NAME From e2b4943bbb46bfb9c7ceba9c60e4dab4699602c5 Mon Sep 17 00:00:00 2001 From: Andras Slemmer <0slemi0@gmail.com> Date: Wed, 6 Jun 2018 12:58:23 +0100 Subject: [PATCH 2/4] Observable toposort for transactions (#3027) --- .../core/internal/ResolveTransactionsFlow.kt | 28 +---- .../corda/core/internal/TopologicalSort.kt | 117 ++++++++++++++++++ .../core/internal/TopologicalSortTest.kt | 106 ++++++++++++++++ 3 files changed, 227 insertions(+), 24 deletions(-) create mode 100644 core/src/main/kotlin/net/corda/core/internal/TopologicalSort.kt create mode 100644 core/src/test/kotlin/net/corda/core/internal/TopologicalSortTest.kt diff --git a/core/src/main/kotlin/net/corda/core/internal/ResolveTransactionsFlow.kt b/core/src/main/kotlin/net/corda/core/internal/ResolveTransactionsFlow.kt index d9a6f2c715..00c5a56d70 100644 --- a/core/src/main/kotlin/net/corda/core/internal/ResolveTransactionsFlow.kt +++ b/core/src/main/kotlin/net/corda/core/internal/ResolveTransactionsFlow.kt @@ -47,31 +47,11 @@ class ResolveTransactionsFlow(txHashesArg: Set, * Topologically sorts the given transactions such that dependencies are listed before dependers. */ @JvmStatic fun topologicalSort(transactions: Collection): List { - // Construct txhash -> dependent-txs map - val forwardGraph = HashMap>() - transactions.forEach { stx -> - stx.inputs.forEach { (txhash) -> - // Note that we use a LinkedHashSet here to make the traversal deterministic (as long as the input list is) - forwardGraph.getOrPut(txhash) { LinkedHashSet() }.add(stx) - } + val sort = TopologicalSort() + for (tx in transactions) { + sort.add(tx) } - - val visited = HashSet(transactions.size) - val result = ArrayList(transactions.size) - - fun visit(transaction: SignedTransaction) { - if (transaction.id !in visited) { - visited.add(transaction.id) - forwardGraph[transaction.id]?.forEach(::visit) - result.add(transaction) - } - } - - transactions.forEach(::visit) - - result.reverse() - require(result.size == transactions.size) - return result + return sort.complete() } } diff --git a/core/src/main/kotlin/net/corda/core/internal/TopologicalSort.kt b/core/src/main/kotlin/net/corda/core/internal/TopologicalSort.kt new file mode 100644 index 0000000000..b0fd2d0abf --- /dev/null +++ b/core/src/main/kotlin/net/corda/core/internal/TopologicalSort.kt @@ -0,0 +1,117 @@ +package net.corda.core.internal + +import net.corda.core.contracts.StateRef +import net.corda.core.crypto.SecureHash +import net.corda.core.transactions.SignedTransaction +import rx.Observable + +/** + * Provides a way to topologically sort SignedTransactions. This means that given any two transactions T1 and T2 in the + * list returned by [complete] if T1 is a dependency of T2 then T1 will occur earlier than T2. + */ +class TopologicalSort { + private val forwardGraph = HashMap>() + private val transactions = ArrayList() + + /** + * Add a transaction to the to-be-sorted set of transactions. + */ + fun add(stx: SignedTransaction) { + for (input in stx.inputs) { + // Note that we use a LinkedHashSet here to make the traversal deterministic (as long as the input list is) + forwardGraph.getOrPut(input.txhash) { LinkedHashSet() }.add(stx) + } + transactions.add(stx) + } + + /** + * Return the sorted list of signed transactions. + */ + fun complete(): List { + val visited = HashSet(transactions.size) + val result = ArrayList(transactions.size) + + fun visit(transaction: SignedTransaction) { + if (transaction.id !in visited) { + visited.add(transaction.id) + forwardGraph[transaction.id]?.forEach(::visit) + result.add(transaction) + } + } + + transactions.forEach(::visit) + return result.reversed() + } +} + +private fun getOutputStateRefs(stx: SignedTransaction): List { + return stx.coreTransaction.outputs.mapIndexed { i, _ -> StateRef(stx.id, i) } +} + +/** + * Topologically sort a SignedTransaction Observable on the fly by buffering transactions until all dependencies are met. + * @param initialUnspentRefs the list of unspent references that may be spent by transactions in the observable. This is + * the initial set of references the sort uses to decide whether to buffer transactions or not. For example if this + * is empty then the Observable should start with issue transactions that don't have inputs. + */ +fun Observable.topologicalSort(initialUnspentRefs: Collection): Observable { + data class State( + val unspentRefs: HashSet, + val bufferedTopologicalSort: TopologicalSort, + val bufferedInputs: HashSet, + val bufferedOutputs: HashSet + ) + + var state = State( + unspentRefs = HashSet(initialUnspentRefs), + bufferedTopologicalSort = TopologicalSort(), + bufferedInputs = HashSet(), + bufferedOutputs = HashSet() + ) + + return concatMapIterable { stx -> + val results = ArrayList() + if (state.unspentRefs.containsAll(stx.inputs)) { + // Dependencies are satisfied + state.unspentRefs.removeAll(stx.inputs) + state.unspentRefs.addAll(getOutputStateRefs(stx)) + results.add(stx) + } else { + // Dependencies are not satisfied, buffer + state.bufferedTopologicalSort.add(stx) + state.bufferedInputs.addAll(stx.inputs) + for (outputRef in getOutputStateRefs(stx)) { + if (!state.bufferedInputs.remove(outputRef)) { + state.bufferedOutputs.add(outputRef) + } + } + for (inputRef in stx.inputs) { + if (!state.bufferedOutputs.remove(inputRef)) { + state.bufferedInputs.add(inputRef) + } + } + } + if (state.unspentRefs.containsAll(state.bufferedInputs)) { + // Buffer satisfied + results.addAll(state.bufferedTopologicalSort.complete()) + state.unspentRefs.removeAll(state.bufferedInputs) + state.unspentRefs.addAll(state.bufferedOutputs) + state = State( + unspentRefs = state.unspentRefs, + bufferedTopologicalSort = TopologicalSort(), + bufferedInputs = HashSet(), + bufferedOutputs = HashSet() + ) + results + } else { + // Buffer not satisfied + state = State( + unspentRefs = state.unspentRefs, + bufferedTopologicalSort = state.bufferedTopologicalSort, + bufferedInputs = state.bufferedInputs, + bufferedOutputs = state.bufferedOutputs + ) + results + } + } +} diff --git a/core/src/test/kotlin/net/corda/core/internal/TopologicalSortTest.kt b/core/src/test/kotlin/net/corda/core/internal/TopologicalSortTest.kt new file mode 100644 index 0000000000..1c4f76cad9 --- /dev/null +++ b/core/src/test/kotlin/net/corda/core/internal/TopologicalSortTest.kt @@ -0,0 +1,106 @@ +package net.corda.core.internal + +import net.corda.client.mock.Generator +import net.corda.core.contracts.ContractState +import net.corda.core.contracts.StateRef +import net.corda.core.contracts.TransactionState +import net.corda.core.crypto.SecureHash +import net.corda.core.crypto.SignatureMetadata +import net.corda.core.crypto.TransactionSignature +import net.corda.core.crypto.sign +import net.corda.core.identity.AbstractParty +import net.corda.core.identity.Party +import net.corda.core.serialization.serialize +import net.corda.core.transactions.CoreTransaction +import net.corda.core.transactions.SignedTransaction +import net.corda.testing.core.SerializationEnvironmentRule +import net.corda.testing.core.TestIdentity +import org.junit.Rule +import org.junit.Test +import rx.Observable +import java.util.* + +class TopologicalSortTest { + class DummyTransaction( + override val id: SecureHash, + override val inputs: List, + val numberOfOutputs: Int, + override val notary: Party + ) : CoreTransaction() { + override val outputs: List> = (1..numberOfOutputs).map { + TransactionState(DummyState(), "", notary) + } + } + + class DummyState : ContractState { + override val participants: List = emptyList() + } + + @Rule + @JvmField + val testSerialization = SerializationEnvironmentRule() + + @Test + fun topologicalObservableSort() { + val testIdentity = TestIdentity.fresh("asd") + + val N = 10 + // generate random tx DAG + val ids = (1..N).map { SecureHash.sha256("$it") } + val forwardsGenerators = (0 until ids.size).map { i -> + Generator.sampleBernoulli(ids.subList(i + 1, ids.size), 0.8).map { outputs -> ids[i] to outputs } + } + val transactions = Generator.sequence(forwardsGenerators).map { forwardGraph -> + val backGraph = forwardGraph.flatMap { it.second.map { output -> it.first to output } }.fold(HashMap>()) { backGraph, edge -> + backGraph.getOrPut(edge.second) { HashSet() }.add(edge.first) + backGraph + } + val outrefCounts = HashMap() + val transactions = ArrayList() + for ((id, outputs) in forwardGraph) { + val inputs = (backGraph[id]?.toList() ?: emptyList()).map { inputTxId -> + val ref = outrefCounts.compute(inputTxId) { _, count -> + if (count == null) { + 0 + } else { + count + 1 + } + }!! + StateRef(inputTxId, ref) + } + val tx = DummyTransaction(id, inputs, outputs.size, testIdentity.party) + val bits = tx.serialize().bytes + val sig = TransactionSignature(testIdentity.keyPair.private.sign(bits).bytes, testIdentity.publicKey, SignatureMetadata(0, 0)) + val stx = SignedTransaction(tx, listOf(sig)) + transactions.add(stx) + } + transactions + } + + // Swap two random items + transactions.combine(Generator.intRange(0, N - 1), Generator.intRange(0, N - 2)) { txs, i, j -> + val k = 0 // if (i == j) i + 1 else j + val tmp = txs[i] + txs[i] = txs[k] + txs[k] = tmp + txs + } + + val random = SplittableRandom() + for (i in 1..100) { + val txs = transactions.generateOrFail(random) + val ordered = Observable.from(txs).topologicalSort(emptyList()).toList().toBlocking().first() + checkTopologicallyOrdered(ordered) + } + } + + fun checkTopologicallyOrdered(txs: List) { + val outputs = HashSet() + for (tx in txs) { + if (!outputs.containsAll(tx.inputs)) { + throw IllegalStateException("Transaction $tx's inputs ${tx.inputs} are not satisfied by $outputs") + } + outputs.addAll(tx.coreTransaction.outputs.mapIndexed { i, _ -> StateRef(tx.id, i) }) + } + } +} \ No newline at end of file From 5f2c3d175dd1a5dac505b3b54cd623b7e6ee876b Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Wed, 6 Jun 2018 13:08:33 +0100 Subject: [PATCH 3/4] ENT-2036 Handle ClosedChannelException during SSL handshake (#3314) --- .../internal/protonwrapper/netty/AMQPChannelHandler.kt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPChannelHandler.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPChannelHandler.kt index 0c54eed36c..d7d1375c0c 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPChannelHandler.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPChannelHandler.kt @@ -21,6 +21,7 @@ import org.apache.qpid.proton.engine.impl.ProtocolTracer import org.apache.qpid.proton.framing.TransportFrame import org.slf4j.LoggerFactory import java.net.InetSocketAddress +import java.nio.channels.ClosedChannelException import java.security.cert.X509Certificate /** @@ -102,7 +103,12 @@ internal class AMQPChannelHandler(private val serverMode: Boolean, createAMQPEngine(ctx) onOpen(Pair(ctx.channel() as SocketChannel, ConnectionChange(remoteAddress, remoteCert, true, false))) } else { - badCert = true + // This happens when the peer node is closed during SSL establishment. + if (evt.cause() is ClosedChannelException) { + log.warn("SSL Handshake closed early.") + } else { + badCert = true + } log.error("Handshake failure ${evt.cause().message}") if (log.isTraceEnabled) { log.trace("Handshake failure", evt.cause()) From 6a2e50b730df86e9b2edef1e2157ef1513539c68 Mon Sep 17 00:00:00 2001 From: Shams Asari Date: Wed, 6 Jun 2018 13:19:18 +0100 Subject: [PATCH 4/4] Blobinspector: trace level logging with --verbose (#3313) --- tools/blobinspector/build.gradle | 3 ++- .../src/main/kotlin/net/corda/blobinspector/Main.kt | 9 ++++++++- tools/blobinspector/src/main/resources/log4j2.xml | 13 +++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 tools/blobinspector/src/main/resources/log4j2.xml diff --git a/tools/blobinspector/build.gradle b/tools/blobinspector/build.gradle index 5d49461034..df0ca18fa6 100644 --- a/tools/blobinspector/build.gradle +++ b/tools/blobinspector/build.gradle @@ -4,7 +4,8 @@ apply plugin: 'kotlin' dependencies { compile project(':client:jackson') compile 'info.picocli:picocli:3.0.0' - compile "org.slf4j:slf4j-nop:$slf4j_version" + compile "org.slf4j:jul-to-slf4j:$slf4j_version" + compile "org.apache.logging.log4j:log4j-slf4j-impl:$log4j_version" compile "com.jcabi:jcabi-manifests:$jcabi_manifests_version" testCompile project(':test-utils') diff --git a/tools/blobinspector/src/main/kotlin/net/corda/blobinspector/Main.kt b/tools/blobinspector/src/main/kotlin/net/corda/blobinspector/Main.kt index e2bde0639e..c65b02c6f8 100644 --- a/tools/blobinspector/src/main/kotlin/net/corda/blobinspector/Main.kt +++ b/tools/blobinspector/src/main/kotlin/net/corda/blobinspector/Main.kt @@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.JsonFactory import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import com.jcabi.manifests.Manifests import net.corda.client.jackson.JacksonSupport +import net.corda.core.internal.isRegularFile import net.corda.core.internal.rootMessage import net.corda.core.serialization.SerializationContext import net.corda.core.serialization.deserialize @@ -63,6 +64,8 @@ class Main : Runnable { var verbose: Boolean = false override fun run() { + System.setProperty("logLevel", if (verbose) "trace" else "off") + val bytes = source!!.readBytes().run { require(size > amqpMagic.size) { "Insufficient bytes for AMQP blob" } sequence() @@ -74,6 +77,8 @@ class Main : Runnable { val envelope = DeserializationInput.getEnvelope(bytes) println(envelope.schema) println() + println(envelope.transformsSchema) + println() } initialiseSerialization() @@ -112,7 +117,9 @@ private class SourceConverter : ITypeConverter { return try { URL(value) } catch (e: MalformedURLException) { - Paths.get(value).toUri().toURL() + val path = Paths.get(value) + require(path.isRegularFile()) { "$path is not a file" } + path.toUri().toURL() } } } diff --git a/tools/blobinspector/src/main/resources/log4j2.xml b/tools/blobinspector/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..a9885efca9 --- /dev/null +++ b/tools/blobinspector/src/main/resources/log4j2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file