Merge remote-tracking branch 'remotes/open/master' into merges/sollecitom-03-10-2018-10-22

# Conflicts:
#	docs/source/index.rst

core/src/main/kotlin/net/corda/core/internal/cordapp/CordappInfoResolver.kt

@@ -1,5 +1,6 @@
 package net.corda.core.internal.cordapp
 
+import net.corda.core.internal.VisibleForTesting
 import net.corda.core.utilities.loggerFor
 import java.util.concurrent.ConcurrentHashMap
 
@@ -49,6 +50,7 @@ object CordappInfoResolver {
      * Temporarily switch out the internal resolver for another one. For use in testing.
      */
     @Synchronized
+    @VisibleForTesting
     fun withCordappInfoResolution(tempResolver: () -> CordappImpl.Info?, block: () -> Unit) {
         val resolver = cordappInfoResolver
         cordappInfoResolver = tempResolver
@@ -59,6 +61,7 @@ object CordappInfoResolver {
         }
     }
 
+    @VisibleForTesting
     internal fun clear() {
         cordappClasses.clear()
     }

docs/source/api-states.rst

@@ -151,4 +151,46 @@ Where:
 * ``notary`` is the notary service for this state
 * ``encumbrance`` points to another state that must also appear as an input to any transaction consuming this
   state
-* ``constraint`` is a constraint on which contract-code attachments can be used with this state
+* ``constraint`` is a constraint on which contract-code attachments can be used with this state
+
+Reference States
+----------------
+
+A reference input state is a ``ContractState`` which can be referred to in a transaction by the contracts of input and
+output states but whose contract is not executed as part of the transaction verification process. Furthermore,
+reference states are not consumed when the transaction is committed to the ledger but they are checked for
+"current-ness". In other words, the contract logic isn't run for the referencing transaction only. It's still a normal
+state when it occurs in an input or output position.
+
+Reference data states enable many parties to reuse the same state in their transactions as reference data whilst
+still allowing the reference data state owner the capability to update the state. A standard example would be the
+creation of financial instrument reference data and the use of such reference data by parties holding the related
+financial instruments.
+
+Just like regular input states, the chain of provenance for reference states is resolved and all dependency transactions
+verified. This is because users of reference data must be satisfied that the data they are referring to is valid as per
+the rules of the contract which governs it and that all previous participants of teh state assented to updates of it.
+
+**Known limitations:**
+
+*Notary change:* It is likely the case that users of reference states do not have permission to change the notary
+assigned to a reference state. Even if users *did* have this permission the result would likely be a bunch of
+notary change races. As such, if a reference state is added to a transaction which is assigned to a
+different notary to the input and output states then all those inputs and outputs must be moved to the
+notary which the reference state uses.
+
+If two or more reference states assigned to different notaries are added to a transaction then it follows that this
+transaction cannot be committed to the ledger. This would also be the case for transactions not containing reference
+states. There is an additional complication for transaction including reference states, however. It is unlikely that the
+party using the reference states has the authority to change the notary for the state (in other words, the party using the
+reference state would not be listed as a participant on it). Therefore, it is likely that a transaction containing
+reference states with two different notaries cannot be committed to the ledger.
+
+As such, if reference states assigned to multiple different notaries are added to a transaction builder
+then the check below will fail.
+
+        .. warning:: Currently, encumbrances should not be used with reference states. In the case where a state is
+                     encumbered by an encumbrance state, the encumbrance state should also be referenced in the same
+                     transaction that references the encumbered state. This is because the data contained within the
+                     encumbered state may take on a different meaning, and likely would do, once the encumbrance state
+                     is taken into account.

docs/source/api-transactions.rst

@@ -94,15 +94,6 @@ to "walk the chain" and verify that each input was generated through a valid seq
 Reference input states
 ~~~~~~~~~~~~~~~~~~~~~~
 
-A reference input state is a ``ContractState`` which can be referred to in a transaction by the contracts of input and
-output states but whose contract is not executed as part of the transaction verification process. Furthermore,
-reference states are not consumed when the transaction is committed to the ledger but they are checked for
-"current-ness". In other words, the contract logic isn't run for the referencing transaction only. It's still a normal
-state when it occurs in an input or output position.
-
-Reference data states enable many parties to "reuse" the same state in their transactions as reference data whilst
-still allowing the reference data state owner the capability to update the state.
-
 A reference input state is added to a transaction as a ``ReferencedStateAndRef``. A ``ReferencedStateAndRef`` can be
 obtained from a ``StateAndRef`` by calling the ``StateAndRef.referenced()`` method which returns a
 ``ReferencedStateAndRef``.
@@ -123,20 +114,32 @@ obtained from a ``StateAndRef`` by calling the ``StateAndRef.referenced()`` meth
             :end-before: DOCEND 55
             :dedent: 12
 
-**Known limitations:**
+**Handling of update races:**
 
-*Notary change:* It is likely the case that users of reference states do not have permission to change the notary assigned
-to a reference state. Even if users *did* have this permission the result would likely be a bunch of
-notary change races. As such, if a reference state is added to a transaction which is assigned to a
-different notary to the input and output states then all those inputs and outputs must be moved to the
-notary which the reference state uses.
+When using reference states in a transaction, it may be the case that a notarisation failure occurs. This is most likely
+because the creator of the state (being used as a reference state in your transaction), has just updated it.
 
-If two or more reference states assigned to different notaries are added to a transaction then it follows
-that this transaction likely *cannot* be committed to the ledger as it unlikely that the party using the
-reference state can change the assigned notary for one of the reference states.
+Typically, the creator of such reference data will have implemented flows for syndicating the updates out to users.
+However it is inevitable that there will be a delay between the state being used as a reference being consumed, and the
+nodes using it receiving the update.
 
-As such, if reference states assigned to multiple different notaries are added to a transaction builder
-then the check below will fail.
+This is where the ``WithReferencedStatesFlow`` comes in. Given a flow which uses reference states, the
+``WithReferencedStatesFlow`` will execute the the flow as a subFlow. If the flow fails due to a ``NotaryError.Conflict``
+for a reference state, then it will be suspended until the state refs for the reference states are consumed. In this
+case, a consumption means that:
+
+1. the owner of the reference state has updated the state with a valid, notarised transaction
+2. the owner of the reference state has shared the update with the node attempting to run the flow which uses the
+   reference state
+3. The node has successfully committed the transaction updating the reference state (and all the dependencies), and
+   added the updated reference state to the vault.
+
+At the point where the transaction updating the state being used as a reference is committed to storage and the vault
+update occurs, then the ``WithReferencedStatesFlow`` will wake up and re-execute the provided flow.
+
+.. warning:: Caution should be taken when using this flow as it facilitates automated re-running of flows which use
+             reference states. The flow using reference states should include checks to ensure that the reference data is
+             reasonable, especially if the economics of the transaction depends upon the data contained within a reference state.
 
 Output states
 ^^^^^^^^^^^^^

docs/source/key-concepts-states.rst

@@ -59,4 +59,13 @@ is aware of, and which it considers to be relevant to itself:
    :align: center
 
 We can think of the ledger from each node's point of view as the set of all the current (i.e. non-historic) states that
-it is aware of.
+it is aware of.
+
+Reference states
+----------------
+
+Not all states need to be updated by the parties which use them. In the case of reference data, there is a common pattern
+where one party creates reference data, which is then used (but not updated) by other parties. For this use-case, the
+states containing reference data are referred to as "reference states". Syntactically, reference states are no different
+to regular states. However, they are treated different by Corda transactions. See :doc:`key-concepts-transactions` for
+more details.

docs/source/key-concepts-transactions.rst

@@ -33,7 +33,7 @@ Here is an example of an update transaction, with two inputs and two outputs:
    :scale: 25%
    :align: center
 
-A transaction can contain any number of inputs and outputs of any type:
+A transaction can contain any number of inputs, outputs and references of any type:
 
 * They can include many different state types (e.g. both cash and bonds)
 * They can be issuances (have zero inputs) or exits (have zero outputs)
@@ -108,6 +108,18 @@ Each required signers should only sign the transaction if the following two cond
 If the transaction gathers all the required signatures but these conditions do not hold, the transaction's outputs
 will not be valid, and will not be accepted as inputs to subsequent transactions.
 
+Reference states
+----------------
+
+As mentioned in :doc:`key-concepts-states`, some states need to be referred to by the contracts of other input or output
+states but not updated/consumed. This is where reference states come in. When a state is added to the references list of
+a transaction, instead of the inputs or outputs list, then it is treated as a *reference state*. There are two important
+differences between regular states and reference states:
+
+* The specified notary for the transaction **does** check whether the reference states are current. However, reference
+  states are not consumed when the transaction containing them is committed to the ledger.
+* The contracts for reference states are not executed for the transaction containing them.
+
 Other transaction components
 ----------------------------
 As well as input states and output states, transactions contain:

docs/source/network-bootstrapper.rst

@@ -87,9 +87,11 @@ Any CorDapps provided when bootstrapping a network will be scanned for contracts
 The CorDapp JARs will be hashed and scanned for ``Contract`` classes. These contract class implementations will become part
 of the whitelisted contracts in the network parameters (see ``NetworkParameters.whitelistedContractImplementations`` :doc:`network-map`).
 
-By default the bootstrapper will whitelist all the contracts found in all the CorDapp JARs. To prevent certain
-contracts from being whitelisted, add their fully qualified class name in the ``exclude_whitelist.txt``. These will instead
-use the more restrictive ``HashAttachmentConstraint``.
+By default the bootstrapper will whitelist all the contracts found in the unsigned CorDapp JARs (a JAR file not signed by jarSigner tool).
+Whitelisted contracts are checked by `Zone constraints`, while contract classes from signed JARs will be checked by `Signature constraints`.
+To prevent certain contracts from unsigned JARs from being whitelisted, add their fully qualified class name in the ``exclude_whitelist.txt``.
+These will instead use the more restrictive ``HashAttachmentConstraint``.
+Refer to :doc:`api-contract-constraints` to understand the implication of different constraint types before adding ``exclude_whitelist.txt`` files.
 
 For example:
 

node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkBootstrapper.kt

@@ -34,6 +34,7 @@ import java.time.Instant
 import java.util.*
 import java.util.concurrent.Executors
 import java.util.concurrent.TimeUnit
+import java.util.jar.JarInputStream
 import kotlin.collections.component1
 import kotlin.collections.component2
 import kotlin.collections.set
@@ -208,7 +209,7 @@ internal constructor(private val initSerEnv: Boolean,
             println("Gathering notary identities")
             val notaryInfos = gatherNotaryInfos(nodeInfoFiles, configs)
             println("Generating contract implementations whitelist")
-            val newWhitelist = generateWhitelist(existingNetParams, readExcludeWhitelist(directory), cordappJars.map(contractsJarConverter))
+            val newWhitelist = generateWhitelist(existingNetParams, readExcludeWhitelist(directory), cordappJars.filter { !isSigned(it) }.map(contractsJarConverter))
             val newNetParams = installNetworkParameters(notaryInfos, newWhitelist, existingNetParams, nodeDirs)
             if (newNetParams != existingNetParams) {
                 println("${if (existingNetParams == null) "New" else "Updated"} $newNetParams")
@@ -398,4 +399,10 @@ internal constructor(private val initSerEnv: Boolean,
             return magic == amqpMagic && target == SerializationContext.UseCase.P2P
         }
     }
+
+    private fun isSigned(file: Path): Boolean = file.read {
+        JarInputStream(it).use {
+            JarSignatureCollector.collectSigningParties(it).isNotEmpty()
+        }
+    }
 }

node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelper.kt

@@ -199,5 +199,5 @@ internal fun x500toHostName(x500Name: CordaX500Name): String {
     val secureHash = SecureHash.sha256(x500Name.toString())
     // RFC 1035 specifies a limit 255 bytes for hostnames with each label being 63 bytes or less. Due to this, the string
     // representation of the SHA256 hash is truncated to 32 characters.
-    return String.format(HOSTNAME_FORMAT, secureHash.toString().substring(0..32).toLowerCase())
+    return String.format(HOSTNAME_FORMAT, secureHash.toString().take(32).toLowerCase())
 }

node-api/src/test/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/SSLHelperTest.kt

@@ -0,0 +1,35 @@
+package net.corda.nodeapi.internal.protonwrapper.netty
+
+import net.corda.core.crypto.SecureHash
+import net.corda.core.identity.CordaX500Name
+import net.corda.core.utilities.NetworkHostAndPort
+import net.corda.nodeapi.internal.config.CertificateStore
+import net.corda.testing.internal.configureTestSSL
+import org.junit.Test
+import javax.net.ssl.KeyManagerFactory
+import javax.net.ssl.SNIHostName
+import javax.net.ssl.TrustManagerFactory
+import kotlin.test.assertEquals
+
+class SSLHelperTest {
+    @Test
+    fun `ensure SNI header in correct format`() {
+        val legalName = CordaX500Name("Test", "London", "GB")
+        val sslConfig = configureTestSSL(legalName)
+
+        val keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm())
+        val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm())
+
+        keyManagerFactory.init(CertificateStore.fromFile(sslConfig.keyStore.path, sslConfig.keyStore.password, false))
+        trustManagerFactory.init(initialiseTrustStoreAndEnableCrlChecking(CertificateStore.fromFile(sslConfig.trustStore.path, sslConfig.trustStore.password, false), false))
+
+        val sslHandler = createClientSslHelper(NetworkHostAndPort("localhost", 1234), setOf(legalName), keyManagerFactory, trustManagerFactory)
+        val legalNameHash = SecureHash.sha256(legalName.toString()).toString().take(32).toLowerCase()
+
+        // These hardcoded values must not be changed, something is broken if you have to change these hardcoded values.
+        assertEquals("O=Test, L=London, C=GB", legalName.toString())
+        assertEquals("f3df3c01a5f5aa5b9d394680cde3a414", legalNameHash)
+        assertEquals(1, sslHandler.engine().sslParameters.serverNames.size)
+        assertEquals("$legalNameHash.corda.net", (sslHandler.engine().sslParameters.serverNames.first() as SNIHostName).asciiName)
+    }
+}

node/src/main/kotlin/net/corda/node/services/CoreFlowHandlers.kt

@@ -20,6 +20,8 @@ class FinalityHandler(private val sender: FlowSession) : FlowLogic<Unit>() {
     override fun call() {
         subFlow(ReceiveTransactionFlow(sender, true, StatesToRecord.ONLY_RELEVANT))
     }
+
+    internal fun sender(): Party = sender.counterparty
 }
 
 class NotaryChangeHandler(otherSideSession: FlowSession) : AbstractStateReplacementFlow.Acceptor<Party>(otherSideSession) {

node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt

@@ -222,7 +222,7 @@ class FlowStateMachineImpl<R>(override val id: StateMachineRunId,
             suspend(FlowIORequest.WaitForSessionConfirmations, maySkipCheckpoint = true)
             Try.Success(result)
         } catch (throwable: Throwable) {
-            logger.info("Flow threw exception... sending to flow hospital", throwable)
+            logger.info("Flow threw exception... sending it to flow hospital", throwable)
             Try.Failure<R>(throwable)
         }
         val softLocksId = if (hasSoftLockedStates) logic.runId.uuid else null

node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt

@@ -217,7 +217,11 @@ class StaffedFlowHospital {
      */
     object FinalityDoctor : Staff {
         override fun consult(flowFiber: FlowFiber, currentState: StateMachineState, newError: Throwable, history: MedicalHistory): Diagnosis {
-            return if (currentState.flowLogic is FinalityHandler) Diagnosis.OVERNIGHT_OBSERVATION else Diagnosis.NOT_MY_SPECIALTY
+            return (currentState.flowLogic as? FinalityHandler)?.let { logic -> Diagnosis.OVERNIGHT_OBSERVATION.also { warn(logic, flowFiber, currentState) } } ?: Diagnosis.NOT_MY_SPECIALTY
+        }
+
+        private fun warn(flowLogic: FinalityHandler, flowFiber: FlowFiber, currentState: StateMachineState) {
+            log.warn("Flow ${flowFiber.id} failed to be finalised. Manual intervention may be required before retrying the flow by re-starting the node. State machine state: $currentState, initiating party was: ${flowLogic.sender().name}")
         }
     }
 }

release-tools/testing/.gitignore

@@ -0,0 +1 @@
+*.pyc

release-tools/testing/README.md

@@ -0,0 +1,83 @@
+# Release Tools - Test Tracker and Generator
+
+## Introduction
+
+This command-line tool lets the user create and track tests in the [R3T](https://r3-cev.atlassian.net/projects/R3T) JIRA project. All generic test cases are captured as tickets of type **Platform Test Template** with a label "OS" for tests pertaining to **Corda Open Source**, "ENT" for **Corda Enterprise**, and "NS" for **Corda Network Services**. These tickets can be set to **Active** or **Inactive** status based on their relevance for a particular release.
+
+The tool creates a set of new release tests by cloning the current set of active test templates into a set of **Platform Test** tickets. These will each get assigned to the appropriate target version. Further, the tool lets the user create sub-tasks for each of the release tests, one for each release candidate. These steps are described in more detail further down.
+
+## List Test Cases
+
+To list the active test cases for a product, run the following command:
+
+```bash
+$ ./test-manager list-tests <PRODUCT>
+```
+
+Where `<PRODUCT>` is either `OS`, `ENT` or `NS`. This will list the test cases that are currently applicable to Corda Open Source, Corda Enterprise and Corda Network Services, respectively.
+
+## Show Test Status
+
+To show the status of all test runs for a specific release or release candidate, run:
+
+```bash
+$ ./test-manager status <PRODUCT> <VERSION> <CANDIDATE>
+```
+
+Here, `<VERSION>` represents the target version, e.g., product `OS` and version `3.3` would represent Corda Open Source 3.3. `<CANDIDATE>` is optional and will narrow down the report to only show the provided candidate version, e.g., `1` for `RC01`.
+
+## Create JIRA Version
+
+To create a new release version in JIRA, you can run the following command:
+
+```bash
+$ ./test-manager create-version <PRODUCT> <VERSION> <CANDIDATE>
+```
+
+Note that `<CANDIDATE>` is optional. This command will create new versions in the following JIRA projects: `CORDA`, `ENT`, `ENM`, `CID` and `R3T`.
+
+## Create Release Tests
+
+To create the set of parent tests for a new release, you can run:
+
+```bash
+$ ./test-manager create-release-tests <PRODUCT> <VERSION>
+```
+
+This will create the test cases, but none of the test run tickets for respective release candidates. Note also that "blocks"-links between active test templates will be carried across to the created test tickets.
+
+## Create Release Candidate Tests
+
+To create a set of test run tickets for a new release candidate, you can run:
+
+```bash
+$ ./test-manager create-release-candidate-tests <PRODUCT> <VERSION> <CANDIDATE>
+```
+
+This will create a new sub-task under each of the test tickets for `<PRODUCT>` `<VERSION>`, for release candidate `<CANDIDATE>`.
+
+## Options
+
+Each command described above has a set of additional options. More specifically, if you want to use a particular JIRA user instead of being prompted for a user name every time, you can specify `--user <USER>`. For verbose logging, you can supply `--verbose` or `-v`. And to auto-reply to the prompt of whether to proceed or not, provide `--yes` or `-y`.
+
+There is also a useful dry-run option, `--dry-run` or `-d`, that lets you run through the command without creating any tickets or applying any changes to JIRA.
+
+## Example
+
+As an example, say you want to create test cases for Corda Network Services 1.0 RC01. You would then follow the following steps:
+
+```bash
+$ ./test-manager create-version NS 1.0   # Create "Corda Network Services 1.0" - if it doesn't exist
+$ ./test-manager create-version NS 1.0 1 # Create "Corda Network Services 1.0 RC01" - if it doesn't exist
+$ ./test-manager create-release-tests NS 1.0 # Create test cases
+$ ./test-manager create-release-candidate-tests NS 1.0 1 # Create test run for release candidate
+```
+
+Later, when it's time to test RC02, you simply run the following:
+
+```bash
+$ ./test-manager create-version NS 1.0 2
+$ ./test-manager create-release-candidate-tests NS 1.0 2
+```
+
+That's it. Voila, you've got yourself a whole new set of JIRA tickets :-)

release-tools/testing/args.py

@@ -0,0 +1,71 @@
+from __future__ import print_function
+from argparse import Action, ArgumentParser
+import sys, traceback
+
+# {{{ Representation of a command-line program
+class Program:
+
+    # Create a new command-line program represenation, provided an optional name and description
+    def __init__(self, name=None, description=None):
+        self.parser = ArgumentParser(name, description=description)
+        self.subparsers = self.parser.add_subparsers(title='commands')
+        self.arguments = []
+
+    # Enter program definition block
+    def __enter__(self):
+        return self
+
+    # Add argument to the top-level command-line interface and all registered sub-commands
+    def add(self, name, *args, **kwargs):
+        self.parser.add_argument(name, *args, **kwargs)
+        self.arguments.append(([name] + list(args), kwargs))
+
+    # Add sub-command to the set of command-line options
+    def command(self, name, description, handler):
+        return Command(self, self.subparsers, name, description, handler)
+
+    # Parse arguments from the command line, and run the associated command handler
+    def __exit__(self, type, value, tb):
+        args = self.parser.parse_args()
+        try:
+            if 'func' in args:
+                args.func(args)
+            else:
+                self.parser.print_help()
+        except KeyboardInterrupt:
+            print()
+        except Exception as error:
+            if args.verbose:
+                t, exception, tb = sys.exc_info()
+                self.parser.error('{}\n\n{}'.format(error.message, '\n'.join(traceback.format_tb(tb))))
+            else:
+                self.parser.error(error.message)
+# }}}
+
+# {{{ Representation of a sub-command of a command-line program
+class Command:
+
+    # Create a sub-command, provided a name, description and command handler
+    def __init__(self, program, subparsers, name, description, handler):
+        self.program = program
+        self.subparsers = subparsers
+        self.name = name
+        self.description = description
+        self.handler = handler
+
+    # Enter sub-command definition block
+    def __enter__(self):
+        self.parser = self.subparsers.add_parser(self.name, description=self.description)
+        return self
+
+    # Add argument to the CLI command
+    def add(self, name, *args, **kwargs):
+        self.parser.add_argument(name, *args, **kwargs)
+
+    # Exit sub-command definition block and register default handler
+    def __exit__(self, type, value, traceback):
+        for (args, kwargs) in self.program.arguments:
+            self.parser.add_argument(*args, **kwargs)
+        self.parser.set_defaults(func=self.handler)
+
+# }}}

release-tools/testing/jira_manager.py

@@ -0,0 +1,187 @@
+# {{{ JIRA dependency
+from jira import JIRA
+from jira.exceptions import JIRAError
+# }}}
+
+# {{{ Class for interacting with a hosted JIRA system
+class Jira:
+
+    # {{{ Constants
+    BLOCKS = 'Blocks'
+    DUPLICATE = 'Duplicate'
+    RELATES = 'Relates'
+    # }}}
+
+    # {{{ init(address) - Initialise JIRA class, pointing it to the JIRA endpoint
+    def __init__(self, address='https://r3-cev.atlassian.net'):
+        self.address = address
+        self.jira = None
+        self.mock_key = 1
+        self.custom_fields_by_name, self.custom_fields_by_key = {}, {}
+    # }}}
+
+    # {{{ login(user, password) - Log in as a specific JIRA user
+    def login(self, user, password):
+        try:
+            self.jira = JIRA(self.address, auth=(user, password))
+            for x in self.jira.fields():
+                if x['custom']:
+                    self.custom_fields_by_name[x['name']] = x['key']
+                    self.custom_fields_by_key[x['key']] = x['name']
+            return self
+        except Exception as error:
+            message = error.message
+            if isinstance(error, JIRAError):
+                message = error.text if error.text and len(error.text) > 0 and not error.text.startswith('<!') else message
+            raise Exception('failed to log in to JIRA{}{}'.format(': ' if message else '', message))
+    # }}}
+
+    # {{{ search(query) - Search for issues and manually traverse pages if multiple pages are returned
+    def search(self, query, *args):
+        max_count = 50
+        index, offset, count = 0, 0, max_count
+        query = query.format(*args) if len(args) > 0 else query
+        while count == max_count:
+            try:
+                issues = self.jira.search_issues(query, maxResults=max_count, startAt=offset)
+                count = len(issues)
+                offset += count
+                for issue in issues:
+                    index += 1
+                    yield Issue(self, index=index, issue=issue)
+            except JIRAError as error:
+                raise Exception('failed to run query "{}": {}'.format(query, error.text))
+    # }}}
+
+    # {{{ find(key) - Look up issue by key
+    def find(self, key):
+        try:
+            issue = self.jira.issue(key)
+            return Issue(self, issue=issue)
+        except JIRAError as error:
+            raise Exception('failed to look up issue "{}": {}'.format(key, error.text))
+    # }}}
+
+    # {{{ create(fields, dry_run) - Create a new issue
+    def create(self, fields, dry_run=False):
+        if dry_run:
+            return Issue(self, fields=fields)
+        try:
+            issue = self.jira.create_issue(fields)
+            return Issue(self, issue=issue)
+        except JIRAError as error:
+            raise Exception('failed to create issue: {}'.format(error.text))
+    # }}}
+
+    # {{{ link(issue_key, other_issue_key, relationship, dry_run) - Link one issue to another
+    def link(self, issue_key, other_issue_key, relationship=RELATES, dry_run=False):
+        if dry_run:
+            return
+        try:
+            self.jira.create_issue_link(
+                type=relationship,
+                inwardIssue=issue_key,
+                outwardIssue=other_issue_key,
+                comment={
+                    'body': 'Linked {} to {}'.format(issue_key, other_issue_key),
+                }
+            )
+        except JIRAError as error:
+            raise Exception('failed to link {} and {}: {}'.format(issue_key, other_issue_key, error.text))
+    # }}}
+
+# }}}
+
+# {{{ Representation of a JIRA issue
+class Issue:
+
+    mock_index = 1
+
+    # {{{ init(jira, index, issue, key, fields) - Instantiate an abstract representation of an issue
+    def __init__(self, jira, index=0, issue=None, key=None, fields=None):
+        self._jira = jira
+        self._index = index
+        self._issue = issue
+        self._fields = fields
+        self._key = key if key else u'DRY-{:03d}'.format(Issue.mock_index)
+        if not key and not issue:
+            Issue.mock_index += 1
+    # }}}
+
+    # {{{ getattr(key) - Get attribute from issue
+    def __getattr__(self, key):
+        if key == 'index':
+            return self._index
+        if self._issue:
+            value = self._issue.__getattr__(key)
+            return WrappedDictionary(value) if key == 'fields' else value
+        elif self._fields:
+            if key == 'key':
+                return self._key
+            elif key == 'fields':
+                return WrappedDictionary(self._fields)
+        return None
+    # }}}
+
+    # {{{ getitem(key) - Get item from issue
+    def __getitem__(self, key):
+        return self.__getattr__(key)
+    # }}}
+
+    # {{{ str() - Get a string representation of the issue
+    def __str__(self):
+        summary = self.fields.summary.strip()
+        labels = self.fields.labels
+        if len(labels) > 0:
+            return u'[{}] {} ({})'.format(self.key, summary, ', '.join(labels))
+        else:
+            return u'[{}] {}'.format(self.key, summary)
+    # }}}
+
+    # {{{ clone(..., dry_run) - Create a clone of the issue, resetting any provided fields)
+    def clone(self, **kwargs):
+        dry_run = kwargs['dry_run'] if 'dry_run' in kwargs else False
+        fields = self.fields.to_dict()
+        whitelisted_fields = [
+            'project', 'summary', 'description', 'issuetype', 'labels', 'parent', 'priority',
+            self._jira.custom_fields_by_name['Target Version/s'],
+        ]
+        if 'parent' not in kwargs:
+            whitelisted_fields += [
+                self._jira.custom_fields_by_name['Epic Link'],
+                self._jira.custom_fields_by_name['Preconditions'],
+                self._jira.custom_fields_by_name['Test Steps'],
+                self._jira.custom_fields_by_name['Acceptance Criteria'],
+                self._jira.custom_fields_by_name['Primary Test Environment'],
+            ]
+        for key in kwargs:
+            value = kwargs[key]
+            if key == 'parent' and type(value) in (str, unicode):
+                fields[key] = { 'key' : value }
+            elif key == 'version':
+                fields[self._jira.custom_fields_by_name['Target Version/s']] = [{ 'name' : value }]
+            else:
+                fields[key] = value
+        for key in [key for key in fields if key not in whitelisted_fields]:
+            del fields[key]
+        new_issue = self._jira.create(fields, dry_run=dry_run)
+        return new_issue
+    # }}}
+
+# }}}
+
+# {{{ Dictionary with attribute getters
+class WrappedDictionary:
+    def __init__(self, dictionary):
+        self.dictionary = dictionary
+
+    def __getattr__(self, key):
+        return self.__getitem__(key)
+
+    def __getitem__(self, key):
+        return self.dictionary[key]
+
+    def to_dict(self):
+        return dict(self.dictionary)
+
+# }}}

release-tools/testing/login_manager.py

@@ -0,0 +1,60 @@
+# {{{ Dependencies
+
+from __future__ import print_function
+import sys
+
+try:
+    from getpass import getpass
+except:
+    def getpass(message): return raw_input(message)
+
+try:
+    from keyring import get_password, set_password
+except:
+    def get_password(account, user): return None
+    def set_password(account, user, password): pass
+
+# Python 2.x fix; raw_input was renamed to input in Python 3
+try: input = raw_input
+except NameError: pass
+
+# }}}
+
+# {{{ prompt(message, secret) - Get input from user; if secret is true, hide the input
+def prompt(message, secret=False):
+    try:
+        return getpass(message) if secret else input(message)
+    except:
+        print()
+        return ''
+# }}}
+
+# {{{ confirm(message, auto_yes) - Request confirmation from user and proceed if the response is 'yes'
+def confirm(message, auto_yes=False):
+    if auto_yes:
+        print(message.replace('?', '.'))
+        return
+    if not prompt(u'{} (y/N) '.format(message)).lower().strip().startswith('y'):
+        sys.exit(1)
+# }}}
+
+# {{{ login(account, user, password, use_keyring) - Present user with login prompt and return the provided username and password. If use_keyring is true, use previously provided password (if any)
+def login(account, user=None, password=None, use_keyring=True):
+    if not user:
+        user = prompt('Username: ')
+        user = u'{}@r3.com'.format(user) if '@' not in user else user
+        if not user: return (None, None)
+    else:
+        user = u'{}@r3.com'.format(user) if '@' not in user else user
+        print('Username: {}'.format(user))
+    password = get_password(account, user) if password is None and use_keyring else password
+    if not password:
+        password = prompt('Password: ', secret=True)
+        if not password: return (None, None)
+    else:
+        print('Password: ********')
+    if use_keyring:
+        set_password(account, user, password)
+    print()
+    return (user, password)
+# }}}

release-tools/testing/requirements.txt

@@ -0,0 +1,3 @@
+jira==2.0.0
+keyring==13.1.0
+termcolor==1.1.0

release-tools/testing/test-manager

@@ -0,0 +1,312 @@
+#!/usr/bin/env python
+
+# {{{ Dependencies
+from __future__ import print_function
+import sys
+from args import Program
+from login_manager import login, confirm
+from jira_manager import Jira
+# }}}
+
+# {{{ Dependencies for printing coloured content to the console
+try:
+    from termcolor import colored
+    def blue(message): return colored(message, 'blue')
+    def green(message): return colored(message, 'green')
+    def red(message): return colored(message, 'red')
+    def yellow(message): return colored(message, 'yellow')
+    def faint(message): return colored(message, 'white', attrs=['dark'])
+    def on_green(message): return colored(message, 'white', 'on_green')
+    def on_red(message): return colored(message, 'white', 'on_red')
+    def blue_on_white(message): return colored(message, 'blue', 'on_white')
+    def yellow_on_white(message): return colored(message, 'yellow', 'on_white')
+except:
+    def blue(message): return u'[{}]'.format(message)
+    def green(message): return message
+    def red(message): return message
+    def yellow(message): return message
+    def faint(message): return message
+    def on_green(message): return message
+    def on_red(message): return message
+    def blue_on_white(message): return message
+    def yellow_on_white(message): return message
+# }}}
+
+# {{{ Mapping from product code to product name
+product_map = {
+    'OS'   : 'Corda',
+    'ENT'  : 'Corda Enterprise',
+    'NS'   : 'Corda Network Services',
+    'TEST' : 'Corda', # for demo and test purposes
+}
+# }}}
+
+# {{{ JIRA queries
+QUERY_LIST_TEST_CASES = \
+    u'project = R3T AND type = "Platform Test Template" AND status = Active AND labels = "{}" ORDER BY key'
+QUERY_LIST_TEST_INSTANCES = \
+    u'project = R3T AND type = "Platform Test" AND labels = "{}" AND "Target Version/s" = "{}" ORDER BY key'
+QUERY_LIST_TEST_INSTANCE_FOR_TICKET = \
+    u'project = R3T AND type = "Platform Test" AND labels = "{}" AND "Target Version/s" = "{}" AND issue IN linkedIssues({})'
+QUERY_LIST_ALL_TEST_RUNS_FOR_TICKET = \
+    u'project = R3T AND type = "Platform Test Run" AND labels = "{}" AND parent = {} ORDER BY "Target Version/S"'
+QUERY_LIST_TEST_RUN_FOR_TICKET = \
+    u'project = R3T AND type = "Platform Test Run" AND labels = "{}" AND "Target Version/s" = "{}" AND parent = {}'
+QUERY_LIST_BLOCKING_TEST_CASES = \
+    u'project = R3T AND type = "Platform Test Template" AND labels = "{}" AND issue IN linkedIssues({}, "Blocks")'
+# }}}
+
+# {{{ list_test_cases() - List active test cases for a specific product
+def list_test_cases(args):
+    user, password = login('jira', args.user)
+    if not user or not password: sys.exit(1)
+    jira = Jira().login(user, password)
+    print(u'List of active test cases for {}:'.format(yellow(product_map[args.PRODUCT])))
+    if args.verbose:
+        print(faint('[{}]'.format(QUERY_LIST_TEST_CASES.format(args.PRODUCT))))
+    print()
+    has_tests = False
+    for issue in jira.search(QUERY_LIST_TEST_CASES, args.PRODUCT):
+        print(u' - {} {}'.format(blue(issue.key), issue.fields.summary))
+        has_tests = True
+    if not has_tests:
+        print(u' - No active test cases found')
+    print()
+# }}}
+
+# {{{ show_status() - Show the status of all test runs for a specific release or release candidate
+def show_status(args):
+    user, password = login('jira', args.user)
+    if not user or not password: sys.exit(1)
+    jira = Jira().login(user, password)
+    version = '{} {}'.format(product_map[args.PRODUCT], args.VERSION).replace('.0', '')
+    candidate = '{} RC{:02d}'.format(version, args.CANDIDATE) if args.CANDIDATE else version
+    if args.CANDIDATE:
+        print(u'Status of test runs for {} version {} release candidate {}:'.format(yellow(product_map[args.PRODUCT]), yellow(args.VERSION), yellow('RC{:02d}'.format(args.CANDIDATE))))
+    else:
+        print(u'Status of test runs for {} version {}:'.format(yellow(product_map[args.PRODUCT]), yellow(args.VERSION)))
+    if args.verbose:
+        print(faint('[{}]'.format(QUERY_LIST_TEST_INSTANCES.format(args.PRODUCT, version))))
+    print()
+    has_tests = False
+    for issue in jira.search(QUERY_LIST_TEST_INSTANCES, args.PRODUCT, version):
+        status = issue.fields.status['name'].lower()
+        if status == 'pass':
+            status = on_green('Pass')
+        elif status == 'fail':
+            status = on_red('Fail')
+        elif status == 'descope':
+            status = on_green('Descoped')
+        else:
+            status = ''
+        print(u' - {} {} {}'.format(blue(issue.key), issue.fields.summary, status))
+        has_test_runs = False
+        if args.CANDIDATE:
+            if args.verbose:
+                print(faint('   [{}]'.format(QUERY_LIST_TEST_RUN_FOR_TICKET.format(args.PRODUCT, candidate, issue.key))))
+            run_list = jira.search(QUERY_LIST_TEST_RUN_FOR_TICKET, args.PRODUCT, candidate, issue.key)
+        else:
+            if args.verbose:
+                print(faint('   [{}]'.format(QUERY_LIST_ALL_TEST_RUNS_FOR_TICKET.format(args.PRODUCT, issue.key))))
+            run_list = jira.search(QUERY_LIST_ALL_TEST_RUNS_FOR_TICKET, args.PRODUCT, issue.key)
+        for run in run_list:
+            has_test_runs = True
+            print()
+            status = run.fields.status['name'].lower()
+            if status == 'pass':
+                status = on_green('Pass    ')
+            elif status == 'fail':
+                status = on_red('Fail    ')
+            elif status == 'descope':
+                status = on_green('Descoped')
+            elif status == 'in progress':
+                status = yellow_on_white('Active  ')
+            else:
+                status = blue_on_white('Open    ')
+            print(u'    {} {} ({})'.format(status, faint(run.fields[jira.custom_fields_by_name['Target Version/s']][0]['name']), blue(run.key)))
+        if not has_test_runs:
+            print()
+            print(u'    - No release candidate tests found')
+        print()
+        has_tests = True
+    if not has_tests:
+        print(u' - No test cases found for the specified release')
+    print()
+# }}}
+
+# {{{ create_version() - Create a new JIRA version
+def create_version(args):
+    user, password = login('jira', args.user)
+    if not user or not password: sys.exit(1)
+    jira = Jira().login(user, password)
+    version = '{} {}'.format(product_map[args.PRODUCT], args.VERSION).replace('.0', '')
+    version = '{} RC{:02d}'.format(version, args.CANDIDATE) if args.CANDIDATE else version
+    confirm(u'Create new version {}?'.format(yellow(version)), auto_yes=args.yes or args.dry_run)
+    print()
+    if not args.dry_run:
+        for project in ['CORDA', 'ENT', 'ENM', 'R3T', 'CID']:
+            print(u' - Creating version {} for project {} ...'.format(yellow(version), blue(project)))
+            try:
+                jira.jira.create_version(name=version, project=project, description=version)
+                print(u'   {} - Created version for project {}'.format(green('SUCCESS'), blue(project)))
+            except Exception as error:
+                print(u'   {} - Failed to version: {}'.format(red('FAIL'), error))
+            print()
+
+
+# }}}
+
+# {{{ create_release() - Create test cases for a specific version of a product
+def create_release(args):
+    user, password = login('jira', args.user)
+    if not user or not password: sys.exit(1)
+    jira = Jira().login(user, password)
+    version = '{} {}'.format(product_map[args.PRODUCT], args.VERSION).replace('.0', '')
+    confirm(u'Create test cases for {} version {}?'.format(yellow(product_map[args.PRODUCT]), yellow(args.VERSION)), auto_yes=args.yes or args.dry_run)
+    if args.verbose:
+        print(faint('[{}]'.format(QUERY_LIST_TEST_CASES.format(args.PRODUCT))))
+    print()
+    has_tests = False
+    for issue in jira.search(QUERY_LIST_TEST_CASES, args.PRODUCT):
+        print(u' - {} {}'.format(blue(issue.key), issue.fields.summary))
+        print()
+        has_tests = True
+        print(u'    - Creating test case for version {} ...'.format(yellow(args.VERSION)))
+        if args.verbose:
+            print(faint(u'      [{}]'.format(QUERY_LIST_TEST_INSTANCE_FOR_TICKET.format(args.PRODUCT, version, issue.key))))
+        has_test_case_for_version = len(list(jira.search(QUERY_LIST_TEST_INSTANCE_FOR_TICKET.format(args.PRODUCT, version, issue.key))))
+        if has_test_case_for_version:
+            print(u'      {} - Test case for version already exists'.format(yellow('SKIPPED')))
+        else:
+            try:
+                test_case = issue.clone(issuetype='Platform Test', version=version, dry_run=args.dry_run)
+                print(u'      {} - Created ticket {}'.format(green('SUCCESS'), blue(test_case.key)))
+            except Exception as error:
+                print(u'      {} - Failed to create ticket: {}'.format(red('FAIL'), error))
+            print()
+            print(u'    - Linking test case to template ...')
+            try:
+                jira.link(issue.key, test_case.key, dry_run=args.dry_run)
+                print(u'      {} - Linked {} to {}'.format(green('SUCCESS'), blue(issue.key), blue(test_case.key)))
+            except Exception as error:
+                print(u'      {} - Failed to link tickets: {}'.format(red('FAIL'), error))
+        print()
+    print(u'Copying links from test templates for {} version {}?'.format(yellow(product_map[args.PRODUCT]), yellow(args.VERSION)))
+    print()
+    for issue in jira.search(QUERY_LIST_TEST_CASES, args.PRODUCT):
+        print(u' - {} {}'.format(blue(issue.key), issue.fields.summary))
+        print()
+        print(u'    - Copying links for test case {} ...'.format(blue(issue.key)))
+        has_links = False
+        if args.verbose:
+            print(faint(u'      [{}]'.format(QUERY_LIST_BLOCKING_TEST_CASES.format(args.PRODUCT, issue.key))))
+        for blocking_issue in jira.search(QUERY_LIST_BLOCKING_TEST_CASES, args.PRODUCT, issue.key):
+            from_ticket = list(jira.search(QUERY_LIST_TEST_INSTANCE_FOR_TICKET.format(args.PRODUCT, version, issue.key)))
+            to_ticket = list(jira.search(QUERY_LIST_TEST_INSTANCE_FOR_TICKET.format(args.PRODUCT, version, blocking_issue.key)))
+            if len(from_ticket) == 0 or len(to_ticket) == 0:
+                continue
+            has_links = True
+            from_key = from_ticket[0].key
+            to_key = to_ticket[0].key
+            try:
+                jira.link(from_key, to_key, Jira.BLOCKS, dry_run=args.dry_run)
+                print(u'      {} - Linked {} to {}'.format(green('SUCCESS'), blue(from_key), blue(to_key)))
+            except Exception as error:
+                print(u'      {} - Failed to link tickets {} and {}: {}'.format(red('FAIL'), blue(from_key), blue(to_key), error))
+        if not has_links:
+            print(u'      {} - No relevant links found for ticket {}'.format(yellow('SKIPPED'), blue(issue.key)))
+        print()
+    if not has_tests:
+        print(u' - No active test cases found')
+        print()
+# }}}
+
+# {{{ create_release_candidate() - Create test run tickets for a specific release candidate of a product
+def create_release_candidate(args):
+    user, password = login('jira', args.user)
+    if not user or not password: sys.exit(1)
+    jira = Jira().login(user, password)
+    version = '{} {}'.format(product_map[args.PRODUCT], args.VERSION).replace('.0', '')
+    CANDIDATE = args.CANDIDATE[0]
+    candidate = '{} RC{:02d}'.format(version, CANDIDATE)
+    confirm(u'Create test run tickets for {} version {} release candidate {}?'.format(yellow(product_map[args.PRODUCT]), yellow(args.VERSION), yellow('RC{:02d}'.format(CANDIDATE))), auto_yes=args.yes or args.dry_run)
+    if args.verbose:
+        print(faint('[{}]'.format(QUERY_LIST_TEST_INSTANCES.format(args.PRODUCT, version))))
+    print()
+    has_tests = False
+    for issue in jira.search(QUERY_LIST_TEST_INSTANCES, args.PRODUCT, version):
+        print(u' - {} {}'.format(blue(issue.key), issue.fields.summary))
+        epic_field = jira.custom_fields_by_name['Epic Link']
+        epic = issue.fields[epic_field] if epic_field in issue.fields.to_dict() else ''
+        labels = issue.fields.labels + [epic]
+        print()
+        has_tests = True
+        print(u'    - Creating test run ticket for release candidate {} ...'.format(yellow('RC{:02d}'.format(CANDIDATE))))
+        if args.verbose:
+            print(faint(u'      [{}]'.format(QUERY_LIST_TEST_RUN_FOR_TICKET.format(args.PRODUCT, candidate, issue.key))))
+        has_test_instance_for_version = len(list(jira.search(QUERY_LIST_TEST_RUN_FOR_TICKET.format(args.PRODUCT, candidate, issue.key))))
+        if has_test_instance_for_version:
+            print(u'      {} - Ticket for release candidate already exists'.format(yellow('SKIPPED')))
+        else:
+            try:
+                test_case = issue.clone(issuetype='Platform Test Run', version=candidate, parent=issue.key, labels=labels, dry_run=args.dry_run)
+                print(u'      {} - Created ticket {}'.format(green('SUCCESS'), blue(test_case.key)))
+            except Exception as error:
+                print(u'      {} - Failed to create ticket: {}'.format(red('FAIL'), error))
+        print()
+    if not has_tests:
+        print(u' - No active test cases found')
+        print()
+# }}}
+
+# {{{ main() - Entry point
+def main():
+    with Program(description='tool for managing test cases and test runs in JIRA') as program:
+
+        PRODUCTS = ['OS', 'ENT', 'NS', 'TEST']
+
+        program.add('--verbose', '-v', help='turn on verbose logging', action='store_true')
+        program.add('--yes', '-y', help='automatically answer "yes" to all prompts', action='store_true')
+        program.add('--user', '-u', help='the user name or email address used to log in to JIRA', type=str, metavar='USER')
+        program.add('--no-keyring', help='do not retrieve passwords persisted in the keyring', action='store_true')
+
+        def mixin_dry_run(command):
+            command.add('--dry-run', '-d', help='run action without applying any changes to JIRA', action='store_true')
+
+        def mixin_product(command):
+            command.add('PRODUCT', help='the product under test (OS, ENT, NS)', choices=PRODUCTS, metavar='PRODUCT')
+
+        def mixin_version_and_product(command):
+            mixin_product(command)
+            command.add('VERSION', help='the target version of the release, e.g., 4.0', type=float)
+
+        def mixin_candidate(command, optional=False):
+            if optional:
+                nargs = '?'
+            else:
+                nargs = 1
+            command.add('CANDIDATE', help='the number of the release candidate, e.g., 1 for RC01', type=int, nargs=nargs)
+
+        with program.command('list-tests', 'list test cases applicable to the provided specification', list_test_cases) as command:
+            mixin_product(command)
+
+        with program.command('status', 'show the status of all test runs for a specific release or release candidate', show_status) as command:
+            mixin_version_and_product(command)
+            mixin_candidate(command, True)
+
+        with program.command('create-version', 'create a new version in JIRA', create_version) as command:
+            mixin_dry_run(command)
+            mixin_version_and_product(command)
+            mixin_candidate(command, True)
+
+        with program.command('create-release-tests', 'create test cases for a new release in JIRA', create_release) as command:
+            mixin_dry_run(command)
+            mixin_version_and_product(command)
+
+        with program.command('create-release-candidate-tests', 'create test runs for a new release candidate in JIRA', create_release_candidate) as command:
+            mixin_dry_run(command)
+            mixin_version_and_product(command)
+            mixin_candidate(command)
+# }}}
+
+if __name__ == '__main__': main()