Enable encryption of rhizome payloads

commandline.c

@@ -1080,7 +1080,6 @@ int app_rhizome_add_file(int argc, const char *const *argv, const struct command
   if (!m)
     return WHY("Manifest struct could not be allocated -- not added to rhizome");
   
-  
   if (manifestpath[0] && access(manifestpath, R_OK) == 0) {
     if (config.debug.rhizome) DEBUGF("reading manifest from %s", manifestpath);
     /* Don't verify the manifest, because it will fail if it is incomplete.
@@ -1100,11 +1099,6 @@ int app_rhizome_add_file(int argc, const char *const *argv, const struct command
   if (rhizome_fill_manifest(m, filepath, *authorSidHex?&authorSid:NULL, bskhex?&bsk:NULL))
     return -1;
   
-  /* Keep note as to whether we are supposed to be encrypting this file or not */
-  // TODO should we encrypt??
-  m->payloadEncryption=0;
-  rhizome_manifest_set_ll(m,"crypt",m->payloadEncryption?1:0); 
-  
   if (m->fileLength){
     if (rhizome_add_file(m, filepath))
       return -1;
@@ -1222,7 +1216,7 @@ int app_rhizome_extract_manifest(int argc, const char *const *argv, const struct
   if (config.debug.verbose) DEBUG_argv("command", argc, argv);
   const char *pins, *manifestid, *manifestpath;
   cli_arg(argc, argv, o, "pin,pin...", &pins, NULL, "");
-  if (cli_arg(argc, argv, o, "manifestid", &manifestid, cli_manifestid, NULL)
+  if (cli_arg(argc, argv, o, "manifestid", &manifestid, cli_manifestid, NULL) == -1
    || cli_arg(argc, argv, o, "manifestpath", &manifestpath, NULL, NULL) == -1)
     return -1;
   /* Ensure the Rhizome database exists and is open */
@@ -1290,12 +1284,14 @@ int app_rhizome_extract_manifest(int argc, const char *const *argv, const struct
 int app_rhizome_extract_file(int argc, const char *const *argv, const struct command_line_option *o, void *context)
 {
   if (config.debug.verbose) DEBUG_argv("command", argc, argv);
-  const char *fileid, *filepath, *manifestid, *pins;
+  const char *fileid, *filepath, *manifestid, *pins, *bskhex;
   if (cli_arg(argc, argv, o, "manifestid", &manifestid, cli_manifestid, NULL) == -1
       || cli_arg(argc, argv, o, "filepath", &filepath, NULL, "") == -1
       || cli_arg(argc, argv, o, "fileid", &fileid, cli_fileid, NULL) == -1
-      || cli_arg(argc, argv, o, "pin,pin...", &pins, NULL, "") == -1)
+      || cli_arg(argc, argv, o, "pin,pin...", &pins, NULL, "") == -1
+      || cli_arg(argc, argv, o, "bsk", &bskhex, cli_optional_bundle_key, NULL) == -1)
     return -1;
+  
   /* Ensure the Rhizome database exists and is open */
   if (create_serval_instance_dir() == -1)
     return -1;
@@ -1315,13 +1311,17 @@ int app_rhizome_extract_file(int argc, const char *const *argv, const struct com
     char manifestIdUpper[RHIZOME_MANIFEST_ID_STRLEN + 1];
     tohex(manifestIdUpper, manifest_id, RHIZOME_MANIFEST_ID_BYTES);
     
+    rhizome_bk_t bsk;
+    if (bskhex && fromhexstr(bsk.binary, bskhex, RHIZOME_BUNDLE_KEY_BYTES) == -1)
+      return WHYF("invalid bsk: %s", bskhex);
+    
     rhizome_manifest *m = rhizome_new_manifest();
     if (m==NULL)
       return WHY("Out of manifests");
     
     ret = rhizome_retrieve_manifest(manifestIdUpper, m);
     if (ret==0){
-      ret = rhizome_extract_file(m, filepath);
+      ret = rhizome_extract_file(m, filepath, bskhex?&bsk:NULL);
     }
     
     if (m)
@@ -1969,7 +1969,7 @@ struct command_line_option command_line_options[]={
    "Import a payload/manifest pair into Rhizome"},
   {app_rhizome_list,{"rhizome","list","<pin,pin...>","[<service>]","[<sender_sid>]","[<recipient_sid>]","[<offset>]","[<limit>]",NULL},CLIFLAG_STANDALONE,
    "List all manifests and files in Rhizome"},
-  {app_rhizome_extract_manifest,{"rhizome","extract","manifest","<manifestid>","[<manifestpath>]","[<pin,pin...>]",NULL},CLIFLAG_STANDALONE,
+  {app_rhizome_extract_manifest,{"rhizome","extract","manifest","<manifestid>","[<manifestpath>]","[<pin,pin...>]","[<bsk>]",NULL},CLIFLAG_STANDALONE,
    "Extract a manifest from Rhizome and write it to the given path"},
   {app_rhizome_extract_file,{"rhizome","extract","file","<manifestid>","[<filepath>]","[<pin,pin...>]",NULL},CLIFLAG_STANDALONE,
    "Extract a file from Rhizome and write it to the given path"},

rhizome.h

@@ -121,7 +121,7 @@ typedef struct rhizome_manifest {
 
   int errors; /* if non-zero, then manifest should not be trusted */
   time_ms_t inserttime;
-
+  
   /* Set non-zero after variables have been packed and
      signature blocks appended.
      All fields below may not be valid until the manifest has been finalised */
@@ -139,8 +139,11 @@ typedef struct rhizome_manifest {
   char *dataFileName;
   /* If set, unlink(2) the associated file when freeing the manifest */
   int dataFileUnlinkOnFree;
+  
   /* Whether the paylaod is encrypted or not */
-  int payloadEncryption; 
+  int payloadEncryption;
+  unsigned char payloadKey[RHIZOME_CRYPT_KEY_BYTES];
+  unsigned char payloadNonce[crypto_stream_xsalsa20_NONCEBYTES];
 
   /* Whether we have the secret for this manifest on hand */
   int haveSecret;
@@ -338,6 +341,7 @@ int rhizome_secret2bk(
 		      );
 unsigned char *rhizome_bundle_shared_secret(rhizome_manifest *m);
 int rhizome_extract_privatekey(rhizome_manifest *m, rhizome_bk_t *bsk);
+int rhizome_extract_privatekey_required(rhizome_manifest *m, rhizome_bk_t *bsk);
 int rhizome_sign_hash_with_key(rhizome_manifest *m,const unsigned char *sk,
 			       const unsigned char *pk,rhizome_signature *out);
 int rhizome_verify_bundle_privatekey(rhizome_manifest *m, const unsigned char *sk,
@@ -640,11 +644,12 @@ int rhizome_finish_write(struct rhizome_write *write);
 int rhizome_import_file(rhizome_manifest *m, const char *filepath);
 int rhizome_stat_file(rhizome_manifest *m, const char *filepath);
 int rhizome_add_file(rhizome_manifest *m, const char *filepath);
+int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk);
 int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stream_offset, 
 			    const unsigned char *key, const unsigned char *nonce);
 int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash);
 int rhizome_read(struct rhizome_read *read, unsigned char *buffer, int buffer_length);
-int rhizome_extract_file(rhizome_manifest *m, const char *filepath);
+int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk);
 int rhizome_dump_file(const char *id, const char *filepath);
 
 #endif //__SERVALDNA__RHIZOME_H

rhizome_bundle.c

@@ -720,26 +720,26 @@ int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t
     if (config.debug.rhizome) DEBUGF("modifying existing bundle bid=%s", id);
     
     // Modifying an existing bundle.  Make sure we can find the bundle secret.
-    int result = rhizome_extract_privatekey(m, bsk);
-    switch (result) {
-      case -1:
-	return -1;
-      case 0:
-	break;
-      case 1:
-	return WHY("bundle contains no BK field, and no bundle secret supplied");
-      case 2:
-	return WHY("Author unknown");
-      case 3:
-	return WHY("Author does not have a Rhizome Secret");
-      case 4:
-	return WHY("Author does not have permission to modify manifest");
-      default:
-	return WHYF("Unknown result from rhizome_extract_privatekey(): %d", result);
-    }
+    if (rhizome_extract_privatekey_required(m, bsk))
+      return -1;
     
     // TODO assert that new version > old version?
   }
   
+  int crypt = rhizome_manifest_get_ll(m,"crypt"); 
+  if (crypt==-1 && m->fileLength){
+    // no explicit crypt flag, should we encrypt this bundle?
+    char *sender = rhizome_manifest_get(m, "sender", NULL, 0);
+    char *recipient = rhizome_manifest_get(m, "recipient", NULL, 0);
+    
+    // anything sent from one person to another should be considered private and encrypted by default
+    if (sender && recipient){
+      if (config.debug.rhizome)
+	DEBUGF("Implicitly adding payload encryption due to presense of sender & recipient fields");
+      m->payloadEncryption=1;
+      rhizome_manifest_set_ll(m,"crypt",1); 
+    }
+  }
+  
   return 0;
 }

rhizome_crypto.c

@@ -253,6 +253,27 @@ int rhizome_extract_privatekey(rhizome_manifest *m, rhizome_bk_t *bsk)
   }
 }
 
+/* Same as rhizome_extract_privatekey, except warnings become errors and are logged */
+int rhizome_extract_privatekey_required(rhizome_manifest *m, rhizome_bk_t *bsk)
+{
+  int result = rhizome_extract_privatekey(m, bsk);
+  switch (result) {
+    case -1:
+    case 0:
+      return result;
+    case 1:
+      return WHY("bundle contains no BK field, and no bundle secret supplied");
+    case 2:
+      return WHY("Author unknown");
+    case 3:
+      return WHY("Author does not have a Rhizome Secret");
+    case 4:
+      return WHY("Author does not have permission to modify manifest");
+    default:
+      return WHYF("Unknown result from rhizome_extract_privatekey(): %d", result);
+  }
+}
+
 /* Discover if the given manifest was created (signed) by any unlocked identity currently in the
  * keyring.
  *
@@ -362,10 +383,11 @@ int rhizome_sign_hash(rhizome_manifest *m,
 		      rhizome_signature *out)
 {
   IN();
-  if (!m->haveSecret && rhizome_extract_privatekey(m, NULL))
-    RETURN(WHY("Cannot find secret key to sign manifest data."));
+  if (!m->haveSecret && rhizome_extract_privatekey_required(m, NULL))
+    RETURN(-1);
 
-  RETURN(rhizome_sign_hash_with_key(m,m->cryptoSignSecret,m->cryptoSignPublic,out));
+  int ret=rhizome_sign_hash_with_key(m,m->cryptoSignSecret,m->cryptoSignPublic,out);
+  RETURN(ret);
 }
 
 int rhizome_sign_hash_with_key(rhizome_manifest *m,const unsigned char *sk,
@@ -573,3 +595,70 @@ int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stre
   
   return 0;
 }
+
+int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk)
+{
+  // don't do anything if the manifest isn't flagged as being encrypted
+  if (!m->payloadEncryption)
+    return 0;
+  if (m->payloadEncryption!=1)
+    return WHYF("Unsupported encryption scheme %d", m->payloadEncryption);
+  
+  char *sender = rhizome_manifest_get(m, "sender", NULL, 0);
+  char *recipient = rhizome_manifest_get(m, "recipient", NULL, 0);
+  
+  if (sender && recipient){
+    sid_t sender_sid, recipient_sid;
+    if (cf_opt_sid(&sender_sid, sender)!=CFOK)
+      return WHYF("Unable to parse sender sid");
+    if (cf_opt_sid(&recipient_sid, recipient)!=CFOK)
+      return WHYF("Unable to parse recipient sid");
+    
+    unsigned char *nm_bytes=NULL;
+    int cn=0,in=0,kp=0;
+    if (!keyring_find_sid(keyring,&cn,&in,&kp,sender_sid.binary)){
+      cn=in=kp=0;
+      if (!keyring_find_sid(keyring,&cn,&in,&kp,recipient_sid.binary)){
+	return WHYF("Neither the sender %s nor the recipient %s appears in our keyring", sender, recipient);
+      }
+      nm_bytes=keyring_get_nm_bytes(recipient_sid.binary, sender_sid.binary);
+    }else{
+      nm_bytes=keyring_get_nm_bytes(sender_sid.binary, recipient_sid.binary);
+    }
+    
+    if (!nm_bytes)
+      return -1;
+    
+    unsigned char hash[crypto_hash_sha512_BYTES];
+    crypto_hash_sha512(hash, nm_bytes, crypto_box_curve25519xsalsa20poly1305_BEFORENMBYTES);
+    bcopy(hash, m->payloadKey, RHIZOME_CRYPT_KEY_BYTES);
+    
+  }else{
+    if(!m->haveSecret){
+      if (rhizome_extract_privatekey_required(m, bsk))
+	return -1;
+    }
+    
+    unsigned char raw_key[9+crypto_sign_edwards25519sha512batch_SECRETKEYBYTES]="sasquatch";
+    bcopy(m->cryptoSignSecret, &raw_key[9], crypto_sign_edwards25519sha512batch_SECRETKEYBYTES);
+    
+    unsigned char hash[crypto_hash_sha512_BYTES];
+    
+    crypto_hash_sha512(hash, raw_key, sizeof(raw_key));
+    bcopy(hash, m->payloadKey, RHIZOME_CRYPT_KEY_BYTES);
+  }
+  
+  // generate nonce from version#bundle id#version;
+  unsigned char raw_nonce[8+8+sizeof(m->cryptoSignPublic)];
+  
+  write_uint64(&raw_nonce[0], m->version);
+  bcopy(m->cryptoSignPublic, &raw_nonce[8], sizeof(m->cryptoSignPublic));
+  write_uint64(&raw_nonce[8+sizeof(m->cryptoSignPublic)], m->version);
+  
+  unsigned char hash[crypto_hash_sha512_BYTES];
+  
+  crypto_hash_sha512(hash, raw_nonce, sizeof(raw_nonce));
+  bcopy(hash, m->payloadNonce, sizeof(m->payloadNonce));
+  
+  return 0;  
+}

rhizome_store.c

@@ -330,6 +330,19 @@ int rhizome_add_file(rhizome_manifest *m, const char *filepath)
   if (rhizome_open_write(&write, NULL, m->fileLength, RHIZOME_PRIORITY_DEFAULT))
     return -1;
 
+  write.crypt=m->payloadEncryption;
+  if (write.crypt){
+    // if the manifest specifies encryption, make sure we can generate the payload key and encrypt the contents as we go
+    if (rhizome_derive_key(m, NULL))
+      return -1;
+    
+    if (config.debug.rhizome)
+      DEBUGF("Encrypting file contents");
+    
+    bcopy(m->payloadKey, write.key, sizeof(write.key));
+    bcopy(m->payloadNonce, write.nonce, sizeof(write.nonce));
+  }
+  
   if (rhizome_write_file(&write, filepath)){
     rhizome_fail_write(&write);
     return -1;
@@ -485,7 +498,7 @@ static int write_file(struct rhizome_read *read, const char *filepath){
  * The file will be de-crypted and verified while reading.
  * If filepath is not supplied, the file will still be checked.
  */
-int rhizome_extract_file(rhizome_manifest *m, const char *filepath){
+int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk){
   struct rhizome_read read_state;
   bzero(&read_state, sizeof read_state);
   
@@ -493,6 +506,19 @@ int rhizome_extract_file(rhizome_manifest *m, const char *filepath){
   if (rhizome_open_read(&read_state, m->fileHexHash, 1))
     return -1;
   
+  read_state.crypt=m->payloadEncryption;
+  if (read_state.crypt){
+    // if the manifest specifies encryption, make sure we can generate the payload key and encrypt the contents as we go
+    if (rhizome_derive_key(m, bsk))
+      return -1;
+    
+    if (config.debug.rhizome)
+      DEBUGF("Decrypting file contents");
+    
+    bcopy(m->payloadKey, read_state.key, sizeof(read_state.key));
+    bcopy(m->payloadNonce, read_state.nonce, sizeof(read_state.nonce));
+  }
+  
   return write_file(&read_state, filepath);
 }
 

testdefs_rhizome.sh

@@ -24,6 +24,7 @@ rexp_bundlesecret="$rexp_bundlekey"
 rexp_filehash='[0-9a-fA-F]\{128\}'
 rexp_filesize='[0-9]\{1,\}'
 rexp_version='[0-9]\{1,\}'
+rexp_crypt='[0-9]\{1,\}'
 rexp_date='[0-9]\{1,\}'
 
 assert_manifest_complete() {
@@ -228,6 +229,10 @@ extract_manifest_version() {
    extract_manifest "$1" "$2" version "$rexp_version"
 }
 
+extract_manifest_crypt() {
+   extract_manifest "$1" "$2" crypt "$rexp_crypt"
+}
+
 compute_filehash() {
    local _var="$1"
    local _file="$2"
@@ -276,10 +281,10 @@ bundle_received_by() {
          matches_rexp "$rexp_manifestid" "$bid" || error "invalid bundle ID: $bid"
          bundles+=("$arg")
          if [ "$bid" = "$arg" ]; then
-            rexps+=("RHIZOME ADD MANIFEST service=file bid=$bid")
+            rexps+=("RHIZOME ADD MANIFEST service=.* bid=$bid")
          else
             version="${arg#*:}"
-            rexps+=("RHIZOME ADD MANIFEST service=file bid=$bid version=$version")
+            rexps+=("RHIZOME ADD MANIFEST service=.* bid=$bid version=$version")
          fi
          ;;
       +[A-Z])

tests/rhizomeops

@@ -510,6 +510,29 @@ test_AddUnsupportedService() {
    assertExitStatus '!=' 0
 }
 
+doc_EncryptedPayload="Add an encrypted payload"
+setup_EncryptedPayload() {
+   setup_servald
+   setup_rhizome
+   echo "Clear Text" >file1
+   echo -e "service=file\nname=private\ncrypt=1" >file1.manifest
+}
+test_EncryptedPayload() {
+   executeOk_servald rhizome add file $SIDB1 '' file1 file1.manifest
+   tfw_cat --stdout --stderr
+   assert_stdout_add_file file1
+   assert_manifest_complete file1.manifest
+   executeOk_servald rhizome list ''
+   assert_rhizome_list --fromhere=1 file1
+   extract_manifest_id BID file1.manifest
+   executeOk_servald rhizome extract file $BID file1x
+   tfw_cat --stdout --stderr
+   assert diff file1 file1x
+   extract_manifest_filehash filehash file1.manifest
+   executeOk_servald rhizome dump file $filehash file1y
+   assert ! diff file1 file1y
+}
+
 doc_MeshMSAddCreate="First add MeshMS creates manifest"
 setup_MeshMSAddCreate() {
    setup_servald
@@ -521,6 +544,8 @@ test_MeshMSAddCreate() {
    executeOk_servald rhizome add file $SIDB1 '' file1 file1.manifest
    assert_stdout_add_file file1
    assert_manifest_complete file1.manifest
+   extract_manifest_crypt crypt file1.manifest
+   assert [ $crypt = 1 ]
    executeOk_servald rhizome list ''
    assert_rhizome_list --fromhere=1 file1
    extract_manifest_id BID file1.manifest

tests/rhizomeprotocol

@@ -89,6 +89,26 @@ test_FileTransfer() {
    assert_rhizome_received file2
 }
 
+doc_EncryptedTransfer="Encrypted payload can be opened by destination"
+setup_EncryptedTransfer() {
+   setup_common
+   set_instance +A
+   echo "Clear Text" >file1
+   echo -e "service=MeshMS1\nsender=$SIDA\nrecipient=$SIDB" >file1.manifest
+   executeOk_servald rhizome add file $SIDA '' file1 file1.manifest
+   extract_manifest_id BID file1.manifest
+   extract_manifest_version VERSION file1.manifest
+   start_servald_instances +A +B
+   foreach_instance +A assert_peers_are_instances +B
+   foreach_instance +B assert_peers_are_instances +A
+}
+test_EncryptedTransfer() {
+   wait_until bundle_received_by $BID:$VERSION +B
+   set_instance +B
+   executeOk_servald rhizome extract file $BID file1x
+   assert diff file1 file1x
+}
+
 doc_DisablingHTTPServer="Disabling HTTP rhizome transports works"
 setup_DisablingHTTPServer() {
    setup_common