serval-dna/tests/rhizomeops

#!/bin/bash

# Tests for Serval rhizome command-line operations.
#
# Copyright 2012-2015 Serval Project, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

source "${0%/*}/../testframework.sh"
source "${0%/*}/../testdefs.sh"
source "${0%/*}/../testdefs_rhizome.sh"

shopt -s extglob

setup_rhizome() {
   [ -z "$A_IDENTITY_COUNT" ] && A_IDENTITY_COUNT=1
   [ -z "$B_IDENTITY_COUNT" ] && B_IDENTITY_COUNT=0
   >sids
   if [ $A_IDENTITY_COUNT -ne 0 ]; then
      set_instance +A
      set_rhizome_config
      if [ $A_IDENTITY_COUNT -eq 1 ]; then
         create_single_identity
         echo $SIDA >>sids
      else
         create_identities $A_IDENTITY_COUNT
         for ((i=0; i!=A_IDENTITY_COUNT; ++i)); do
            eval echo \"\$SIDA$i\" >>sids
         done
      fi
   fi
   if [ $B_IDENTITY_COUNT -ne 0 ]; then
      set_instance +B
      set_rhizome_config
      create_identities $B_IDENTITY_COUNT
      for ((i=0; i!=B_IDENTITY_COUNT; ++i)); do
         eval echo \"\$SIDB$i\" >>sids
      done
   fi
   assert [ $(sort sids | uniq | wc -l) -eq $((A_IDENTITY_COUNT + B_IDENTITY_COUNT)) ]
   set_instance +A
}

set_rhizome_config() {
   executeOk_servald config \
      set debug.rhizome on \
      set debug.rhizome_manifest on \
      set debug.verbose on \
      set log.console.level debug
}

doc_InitialEmptyList="Initial list is empty"
setup_InitialEmptyList() {
   setup_servald
   setup_rhizome
}
test_InitialEmptyList() {
   executeOk_servald rhizome list
   assert_rhizome_list
}

doc_AddNoAuthorNoManifest="Add with no author and no manifest file"
setup_AddNoAuthorNoManifest() {
   setup_servald
   setup_rhizome
   executeOk_servald rhizome list
   assert_rhizome_list
   echo "A test file" >file1
   echo "Another test file" >file2
}
test_AddNoAuthorNoManifest() {
   executeOk_servald rhizome add file '' file1
   assert_stdout_add_file file1 !.author !BK
   executeOk_servald rhizome add file '' "$PWD/file2"
   assert_stdout_add_file file2 !.author !BK
}

doc_AddNoManifest="Add with no manifest file"
setup_AddNoManifest() {
   setup_servald
   setup_rhizome
   executeOk_servald rhizome list
   assert_rhizome_list
   echo "A test file" >file1
   echo "Another test file" >file2
}
test_AddNoManifest() {
   executeOk_servald rhizome add file $SIDA file1
   assert_stdout_add_file file1
   executeOk_servald rhizome add file $SIDA "$PWD/file2"
   assert_stdout_add_file file2
}

doc_AddManifestFieldUnsupported="Add with unsupported manifest field"
setup_AddManifestFieldUnsupported() {
   setup_servald
   setup_rhizome
   executeOk_servald rhizome list
   assert_rhizome_list
   echo "A test file" >file1
   echo "Another test file" >file2
   echo "bogus=one" >file1.manifest
}
test_AddManifestFieldUnsupported() {
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   assert_stdout_add_file file1
   tfw_cat -v file1.manifest
   assert_manifest_complete file1.manifest
   assert_manifest_fields file1.manifest bogus=one
}

doc_AddManifestFieldUnsupportedArgs="Add with unsupported manifest field argument"
setup_AddManifestFieldUnsupportedArgs() {
   setup_servald
   setup_rhizome
   executeOk_servald rhizome list
   assert_rhizome_list
   echo "A test file" >file1
   echo "Another test file" >file2
}
test_AddManifestFieldUnsupportedArgs() {
   executeOk_servald rhizome add file $SIDA file1 file1.manifest '' bogus=two
   assert_stdout_add_file file1
   tfw_cat -v file1.manifest
   assert_manifest_complete file1.manifest
   assert_manifest_fields file1.manifest bogus=two
}

doc_AddNoAuthor="Add with no author makes manifest without BK"
setup_AddNoAuthor() {
   setup_servald
   setup_rhizome
   echo "A test file" >file1
}
test_AddNoAuthor() {
   executeOk_servald rhizome add file '' file1 file1.manifest
   assert_manifest_fields file1.manifest !BK
   assert_stdout_add_file file1 !.author !BK
}

doc_AddInvalidAuthor="Add with invalid author fails"
setup_AddInvalidAuthor() {
   setup_servald
   setup_rhizome
   echo "A test file" >file1
}
test_AddInvalidAuthor() {
   execute $servald rhizome add file 'not_a_valid_SID' file1
   assertExitStatus '==' 255
   assertStderrGrep '^ERROR:.*[Ii]nvalid .*[Ss][Ii][Dd]'
}

doc_AddNoAuthorEncrypted="Add encrypted payload with no author"
setup_AddNoAuthorEncrypted() {
   setup_servald
   setup_rhizome
   echo "A test file" >file1
   echo "crypt=1" >file1.manifest
}
test_AddNoAuthorEncrypted() {
   executeOk_servald rhizome add file '' file1 file1.manifest
   tfw_cat --stdout --stderr -v file1.manifest
   assert_manifest_fields file1.manifest !BK
   assert_stdout_add_file file1 !.author !BK
   extract_stdout_secret file1_secret
   executeOk_servald rhizome extract file $re_manifestid file1x $file1_secret
   tfw_cat --stdout --stderr
   assert diff file1 file1x
}

doc_AddNonExistManifest="Add with non-existent manifest file"
setup_AddNonExistManifest() {
   setup_servald
   setup_rhizome
   executeOk_servald rhizome list
   assert_rhizome_list
   echo "A test file" >file1
   echo "Another test file" >file2
}
test_AddNonExistManifest() {
   assert --error-on-fail [ ! -e file1.manifest ]
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   assert_stdout_add_file file1
   assert [ -r file1.manifest ]
   assert_manifest_complete file1.manifest
   assert_manifest_fields file1.manifest service=file name=file1
   assert --error-on-fail [ ! -e file2.manifest ]
   executeOk_servald rhizome add file $SIDA "$PWD/file2" file2.manifest
   assert_stdout_add_file file2
   assert [ -r file2.manifest ]
   assert_manifest_complete file2.manifest
   assert_manifest_fields file2.manifest service=file name=file2
}

doc_AddManifest="Add with minimal manifest file"
setup_AddManifest() {
   setup_servald
   setup_rhizome
   executeOk_servald rhizome list
   assert_rhizome_list
   echo "A test file" >file1
   echo -e 'name=wah\ndate=12345' >file1.manifest
}
test_AddManifest() {
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   tfw_cat --stdout --stderr -v file1.manifest
   assert_stdout_add_file file1 name=wah
   assert_manifest_complete file1.manifest
   assert_manifest_fields file1.manifest service=file name=wah date=12345
}

doc_AddManifestArgs="Add with minimal manifest from arguments"
setup_AddManifestArgs() {
   setup_servald
   setup_rhizome
   executeOk_servald rhizome list
   assert_rhizome_list
   echo "A test file" >file1
}
test_AddManifestArgs() {
   executeOk_servald rhizome add file $SIDA file1 file1.manifest '' name=wah date=12345
   tfw_cat --stdout --stderr -v file1.manifest
   assert_stdout_add_file file1 name=wah
   assert_manifest_complete file1.manifest
   assert_manifest_fields file1.manifest service=file name=wah date=12345
}

doc_AddEmpty="Add with empty payload"
setup_AddEmpty() {
   setup_servald
   setup_rhizome
   executeOk_servald rhizome list
   assert_rhizome_list
}
test_AddEmpty() {
   executeOk_servald rhizome add file $SIDA '' empty.manifest
   tfw_cat --stdout --stderr -v empty.manifest
   assert_stdout_add_file --manifest=empty.manifest ''
   assert_manifest_complete empty.manifest
   assert_manifest_fields empty.manifest service=file name= filesize=0
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 --author=$SIDA --manifest=empty.manifest ''
}

doc_AddMissing="Attempt to add a file and manifest that don't exist"
setup_AddMissing() {
   setup_servald
   setup_rhizome
}
test_AddMissing() {
   execute $servald rhizome add file $SIDA 'file1' 'file1.manifest'
   tfw_cat --stdout --stderr
   assertExitStatus '==' 255
}

doc_AddThenList="List contains one file after one add"
setup_AddThenList() {
   setup_servald
   setup_rhizome
   executeOk_servald rhizome list
   assert_rhizome_list
   echo "A test file" >file1
   echo "Another test file" >file2
}
test_AddThenList() {
   # Add first file
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 --author=$SIDA file1
   # Add second file
   executeOk_servald rhizome add file $SIDA file2 file2.manifest
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 --author=$SIDA file1 file2
}

doc_ExtractManifestAfterAdd="Export manifest after one add"
setup_ExtractManifestAfterAdd() {
   setup_servald
   setup_rhizome
   echo "A test file" >file1
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   extract_stdout_rowid rowid
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 --author=$SIDA file1
   extract_manifest_id manifestid file1.manifest
   extract_manifest_version version file1.manifest
   extract_manifest_filehash filehash file1.manifest
   extract_manifest_BK BK file1.manifest
   extract_manifest_date date file1.manifest
}
test_ExtractManifestAfterAdd() {
   executeOk_servald rhizome export manifest $manifestid file1x.manifest
   tfw_cat --stdout --stderr
   assertStdoutLineCount '==' 13
   local size=$(( $(cat file1 | wc -c) + 0 ))
   assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
   assertStdoutGrep --matches=1 "^version:$version\$"
   assertStdoutGrep --matches=1 "^filesize:$size\$"
   assertStdoutGrep --matches=1 "^filehash:$filehash\$"
   assertStdoutGrep --matches=1 "^BK:$BK\$"
   assertStdoutGrep --matches=1 "^date:$date\$"
   assertStdoutGrep --matches=1 "^service:file\$"
   assertStdoutGrep --matches=1 "^name:file1\$"
   assertStdoutGrep --matches=1 "^\.readonly:0\$"
   assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
   assertStdoutGrep --matches=1 "^\.author:$SIDA\$"
   assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
   assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
   assert [ -e file1x.manifest ]
   assert diff file1.manifest file1x.manifest
}

doc_ExtractManifestFileAfterAdd="Export bundle after one add"
setup_ExtractManifestFileAfterAdd() {
   setup_servald
   setup_rhizome
   echo "A test file" >file1
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   extract_stdout_rowid rowid
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 --author=$SIDA file1
   extract_manifest_id manifestid file1.manifest
   extract_manifest_version version file1.manifest
   extract_manifest_filehash filehash file1.manifest
   extract_manifest_BK BK file1.manifest
   extract_manifest_date date file1.manifest
}
test_ExtractManifestFileAfterAdd() {
   executeOk_servald rhizome export bundle $manifestid file1x.manifest file1x
   tfw_cat --stdout --stderr
   assertStdoutLineCount '==' 13
   local size=$(( $(cat file1 | wc -c) + 0 ))
   assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
   assertStdoutGrep --matches=1 "^version:$version\$"
   assertStdoutGrep --matches=1 "^filesize:$size\$"
   assertStdoutGrep --matches=1 "^filehash:$filehash\$"
   assertStdoutGrep --matches=1 "^BK:$BK\$"
   assertStdoutGrep --matches=1 "^date:$date\$"
   assertStdoutGrep --matches=1 "^service:file\$"
   assertStdoutGrep --matches=1 "^name:file1\$"
   assertStdoutGrep --matches=1 "^\.readonly:0\$"
   assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
   assertStdoutGrep --matches=1 "^\.author:$SIDA\$"
   assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
   assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
   assert [ -e file1x.manifest ]
   assert diff file1.manifest file1x.manifest
   assert [ -e file1x ]
   assert diff file1 file1x
}

doc_ExtractManifestFileFromExtBlob="Export bundle from external blob"
setup_ExtractManifestFileFromExtBlob() {
   setup_servald
   setup_rhizome
   executeOk_servald config set rhizome.max_blob_size 0
   echo "A test file" >file1
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   extract_stdout_rowid rowid1
   executeOk_servald config set rhizome.max_blob_size 1000
   echo "Another test file" >file2
   executeOk_servald rhizome add file $SIDA file2 file2.manifest
   extract_stdout_rowid rowid2
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 --author=$SIDA file1 file2
   extract_manifest_id manifestid1 file1.manifest
   extract_manifest_version version1 file1.manifest
   extract_manifest_filehash filehash1 file1.manifest
   extract_manifest_BK BK1 file1.manifest
   extract_manifest_date date1 file1.manifest
   extract_manifest_id manifestid2 file2.manifest
   extract_manifest_version version2 file2.manifest
   extract_manifest_filehash filehash2 file2.manifest
   extract_manifest_BK BK2 file2.manifest
   extract_manifest_date date2 file2.manifest
}
test_ExtractManifestFileFromExtBlob() {
   executeOk_servald rhizome export bundle $manifestid1 file1x.manifest file1x
   tfw_cat --stdout --stderr
   assertStdoutLineCount '==' 13
   local size=$(( $(cat file1 | wc -c) + 0 ))
   assertStdoutGrep --matches=1 "^manifestid:$manifestid1\$"
   assertStdoutGrep --matches=1 "^version:$version1\$"
   assertStdoutGrep --matches=1 "^filesize:$size\$"
   assertStdoutGrep --matches=1 "^filehash:$filehash1\$"
   assertStdoutGrep --matches=1 "^BK:$BK1\$"
   assertStdoutGrep --matches=1 "^date:$date1\$"
   assertStdoutGrep --matches=1 "^service:file\$"
   assertStdoutGrep --matches=1 "^name:file1\$"
   assertStdoutGrep --matches=1 "^\.readonly:0\$"
   assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
   assertStdoutGrep --matches=1 "^\.author:$SIDA\$"
   assertStdoutGrep --matches=1 "^\.rowid:$rowid1\$"
   assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
   assert [ -e file1x.manifest ]
   assert diff file1.manifest file1x.manifest
   assert [ -e file1x ]
   assert diff file1 file1x
   executeOk_servald rhizome export bundle $manifestid2 file2x.manifest file2x
   tfw_cat --stdout --stderr
   assertStdoutLineCount '==' 13
   local size=$(( $(cat file2 | wc -c) + 0 ))
   assertStdoutGrep --matches=1 "^manifestid:$manifestid2\$"
   assertStdoutGrep --matches=1 "^version:$version2\$"
   assertStdoutGrep --matches=1 "^filesize:$size\$"
   assertStdoutGrep --matches=1 "^filehash:$filehash2\$"
   assertStdoutGrep --matches=1 "^BK:$BK2\$"
   assertStdoutGrep --matches=1 "^date:$date2\$"
   assertStdoutGrep --matches=1 "^service:file\$"
   assertStdoutGrep --matches=1 "^name:file2\$"
   assertStdoutGrep --matches=1 "^\.readonly:0\$"
   assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
   assertStdoutGrep --matches=1 "^\.author:$SIDA\$"
   assertStdoutGrep --matches=1 "^\.rowid:$rowid2\$"
   assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
   assert [ -e file2x.manifest ]
   assert diff file2.manifest file2x.manifest
   assert [ -e file2x ]
   assert diff file2 file2x
}

doc_LargePayload="Export huge bundle after one add"
setup_LargePayload() {
   setup_servald
   setup_rhizome
   executeOk_servald config set debug.rhizome_store on
}
test_LargePayload() {
   rhizome_add_file file1 100000
   executeOk_servald rhizome export bundle $BID file1x.manifest file1x
   assert [ -e file1x.manifest ]
   assert diff file1.manifest file1x.manifest
   assert [ -e file1x ]
   assert diff file1 file1x
}

doc_CorruptExternalBlob="Corrupted payload fails to export"
setup_CorruptExternalBlob() {
   setup_servald
   setup_rhizome
   executeOk_servald config set rhizome.max_blob_size 0
   echo "A test file" >file1
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   extract_manifest_id manifestid file1.manifest
   extract_manifest_filehash filehash file1.manifest
   assert cmp file1 "$SERVALINSTANCE_PATH/blob/$filehash"
   echo "Replacement" >"$SERVALINSTANCE_PATH/blob/$filehash"
}
test_CorruptExternalBlob() {
   execute --exit-status=255 $servald rhizome extract file $manifestid file1a
   tfw_cat --stderr
}

doc_ExtractManifestToStdout="Export manifest to standard output"
setup_ExtractManifestToStdout() {
   setup_servald
   setup_rhizome
   echo "A test file" >file1
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   extract_stdout_rowid rowid
   extract_manifest_id manifestid file1.manifest
   extract_manifest_version version file1.manifest
   extract_manifest_filehash filehash file1.manifest
   extract_manifest_BK BK file1.manifest
   extract_manifest_date date file1.manifest
}
test_ExtractManifestToStdout() {
   executeOk_servald rhizome export manifest $manifestid -
   assertStdoutLineCount '>=' 14
   local size=$(( $(cat file1 | wc -c) + 0 ))
   assertStdoutGrep --line=..13 --matches=1 "^manifestid:$manifestid\$"
   assertStdoutGrep --line=..13 --matches=1 "^version:$version\$"
   assertStdoutGrep --line=..13 --matches=1 "^filesize:$size\$"
   assertStdoutGrep --line=..13 --matches=1 "^filehash:$filehash\$"
   assertStdoutGrep --line=..13 --matches=1 "^BK:$BK\$"
   assertStdoutGrep --line=..13 --matches=1 "^date:$date\$"
   assertStdoutGrep --line=..13 --matches=1 "^service:file\$"
   assertStdoutGrep --line=..13 --matches=1 "^name:file1\$"
   assertStdoutGrep --line=..13 --matches=1 "^\.readonly:0\$"
   assertStdoutGrep --line=..13 --matches=1 "^\.secret:$rexp_bundlesecret\$"
   assertStdoutGrep --line=..13 --matches=1 "^\.author:$SIDA\$"
   assertStdoutGrep --line=..13 --matches=1 "^\.rowid:$rowid\$"
   assertStdoutGrep --line=..13 --matches=1 "^\.inserttime:$rexp_date\$"
   assertStdoutGrep --line=14 --matches=1 "^manifest:"
   replayStdout | $SED -n '14s/^manifest://p' >file1x.manifest
   replayStdout | $SED -n '15,$p' >>file1x.manifest
   cat file1.manifest >file1n.manifest
   echo >>file1n.manifest
   tfw_cat file1n.manifest file1x.manifest
   assert diff file1n.manifest file1x.manifest
}

doc_ExtractManifestAfterAddNoAuthor="Export manifest after one add with no author"
setup_ExtractManifestAfterAddNoAuthor() {
   setup_servald
   setup_rhizome
   echo "A test file" >file1
   executeOk_servald rhizome add file '' file1 file1.manifest
   extract_stdout_rowid rowid
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 file1
   extract_manifest_id manifestid file1.manifest
   extract_manifest_version version file1.manifest
   extract_manifest_filehash filehash file1.manifest
   extract_manifest_date date file1.manifest
}
test_ExtractManifestAfterAddNoAuthor() {
   executeOk_servald rhizome export manifest $manifestid file1x.manifest
   assert diff file1.manifest file1x.manifest
   assertStdoutLineCount '==' 10
   local size=$(( $(cat file1 | wc -c) + 0 ))
   assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
   assertStdoutGrep --matches=1 "^version:$version\$"
   assertStdoutGrep --matches=1 "^filesize:$size\$"
   assertStdoutGrep --matches=1 "^filehash:$filehash\$"
   assertStdoutGrep --matches=1 "^date:$date\$"
   assertStdoutGrep --matches=1 "^service:file\$"
   assertStdoutGrep --matches=1 "^name:file1\$"
   assertStdoutGrep --matches=1 "^\.readonly:1\$"
   assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
   assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
}

doc_ExtractManifestNonExistent="Export non-existent manifest"
setup_ExtractManifestNonExistent() {
   setup_servald
   setup_rhizome
   manifestid=0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF
}
test_ExtractManifestNonExistent() {
   execute --exit-status=1 $servald rhizome export manifest $manifestid foo.manifest
   assertStdoutLineCount '==' 0
   assert [ ! -e foo.manifest ]
}

doc_ExtractManifestInvalidID="Export manifest using invalid ID"
setup_ExtractManifestInvalidID() {
   setup_servald
   setup_rhizome
}
test_ExtractManifestInvalidID() {
   execute --exit-status=255 $servald rhizome export manifest 0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEx foo.manifest
   assertStdoutLineCount '==' 0
   assert [ ! -e foo.manifest ]
   execute --exit-status=255 $servald rhizome export manifest 0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDE foo.manifest
   assertStdoutLineCount '==' 0
   assert [ ! -e foo.manifest ]
   execute --exit-status=255 $servald rhizome export manifest foo.manifest
   assertStdoutLineCount '==' 0
   assert [ ! -e foo.manifest ]
}

doc_ExtractFileAfterAdd="Extract file after one add"
setup_ExtractFileAfterAdd() {
   setup_servald
   setup_rhizome
   echo "A test file" >file1
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   tfw_cat --stderr
   extract_stdout_rowid rowid
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 --author=$SIDA file1
   extract_manifest_id manifestid file1.manifest
   extract_manifest_version version file1.manifest
   extract_manifest_filehash filehash file1.manifest
   extract_manifest_BK BK file1.manifest
   extract_manifest_date date file1.manifest
}
test_ExtractFileAfterAdd() {
   executeOk_servald rhizome extract file $manifestid file1x
   tfw_cat --stderr
   assert diff file1 file1x
   local size=$(( $(cat file1 | wc -c) + 0 ))
   assertStdoutLineCount '==' 13
   assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
   assertStdoutGrep --matches=1 "^version:$version\$"
   assertStdoutGrep --matches=1 "^filesize:$size\$"
   assertStdoutGrep --matches=1 "^filehash:$filehash\$"
   assertStdoutGrep --matches=1 "^BK:$BK\$"
   assertStdoutGrep --matches=1 "^date:$date\$"
   assertStdoutGrep --matches=1 "^service:file\$"
   assertStdoutGrep --matches=1 "^name:file1\$"
   assertStdoutGrep --matches=1 "^\.readonly:0\$"
   assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
   assertStdoutGrep --matches=1 "^\.author:$SIDA\$"
   assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
   assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
}

doc_ExtractFileMissing="Extract and export non-existent file"
setup_ExtractFileMissing() {
   setup_servald
   setup_rhizome
   BID=0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF
   filehash=0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF
}
test_ExtractFileMissing() {
   execute --exit-status=1 $servald rhizome extract file $BID foo
   tfw_cat --stderr
   assertStdoutLineCount '==' 0
   assert [ ! -e foo ]
   execute --exit-status=1 $servald rhizome export file $filehash foo
   tfw_cat --stderr
   assertStdoutLineCount '==' 0
   assert [ ! -e foo ]
}

doc_ExtractFileInvalidID="Extract and export file using invalid ID"
setup_ExtractFileInvalidID() {
   setup_servald
   setup_rhizome
}
test_ExtractFileInvalidID() {
   execute --exit-status=255 $servald rhizome extract file 0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEx foo
   assertStdoutLineCount '==' 0
   assert [ ! -e foo ]
   execute --exit-status=255 $servald rhizome extract file 0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDE foo
   assertStdoutLineCount '==' 0
   assert [ ! -e foo ]
   execute --exit-status=255 $servald rhizome extract file foo
   assertStdoutLineCount '==' 0
   assert [ ! -e foo ]
   execute --exit-status=255 $servald rhizome export file 0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEx foo
   assertStdoutLineCount '==' 0
   assert [ ! -e foo ]
   execute --exit-status=255 $servald rhizome export file 0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDE foo
   assertStdoutLineCount '==' 0
   assert [ ! -e foo ]
   execute --exit-status=255 $servald rhizome export file foo
   assertStdoutLineCount '==' 0
   assert [ ! -e foo ]
}

doc_AddDeDuplicate="Add duplicate file does not create new bundle by default"
setup_AddDeDuplicate() {
   setup_servald
   setup_rhizome
   executeOk_servald rhizome list
   assert_rhizome_list
   echo "A test file" >file1
   echo "Another test file" >file2
   echo "A test file, second version" >file1_2
   # Add first file
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   extract_stdout_secret file1_secret
   # Add second file
   executeOk_servald rhizome add file $SIDA file2 file2.manifest
   extract_stdout_secret file2_secret
   # Make sure they are both in the list.
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 --author=$SIDA file1 file2
}
test_AddDeDuplicate() {
   # Add first file again - should return a "duplicate" status code and nothing
   # should change in its manifests.
   execute --exit-status=2 --stderr --core-backtrace $servald rhizome add file $SIDA file1 file1.manifestA
   assert [ -s file1.manifestA ]
   assert_stdout_add_file file1
   extract_stdout_secret file1_dup_secret
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 --author=$SIDA file1 file2
   strip_signatures file1.manifest file1.manifestA
   assert diff file1.manifest file1.manifestA
   assert [ $file1_secret = $file1_dup_secret ]
   # Repeat for second file.
   execute --exit-status=2 --stderr $servald rhizome add file $SIDA file2 file2.manifestA
   assert [ -s file2.manifestA ]
   assert_stdout_add_file file2
   extract_stdout_secret file2_dup_secret
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 --author=$SIDA file1 file2
   strip_signatures file2.manifest file2.manifestA
   assert diff file2.manifest file2.manifestA
   assert [ $file2_secret = $file2_dup_secret ]
}

doc_AddForceDuplicate="Add duplicate file with --force-new option"
setup_AddForceDuplicate() {
   setup_AddDeDuplicate
}
test_AddForceDuplicate() {
   # Add first file again with the --force-new option.  A new manifest
   # should be created with a new ID.
   executeOk_servald rhizome add file --force-new $SIDA file1 file1.manifestA
   assert [ -s file1.manifestA ]
   assert_stdout_add_file --manifest=file1.manifestA file1
   extract_stdout_secret file1_dup_secret
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 --author=$SIDA file1 file2 --manifest=file1.manifestA file1
   strip_signatures file1.manifest file1.manifestA
   assert ! diff file1.manifest file1.manifestA
   assert [ $file1_secret != $file1_dup_secret ]
   # Repeat for second file.
   executeOk_servald rhizome add file --force-new $SIDA file2 file2.manifestA
   assert [ -s file2.manifestA ]
   assert_stdout_add_file --manifest=file2.manifestA file2
   extract_stdout_secret file2_dup_secret
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 --author=$SIDA file1 file2 --manifest=file1.manifestA file1 --manifest=file2.manifestA file2
   strip_signatures file2.manifest file2.manifestA
   assert ! diff file2.manifest file2.manifestA
   assert [ $file2_secret != $file2_dup_secret ]
}

doc_AddMismatched="Add mismatched manifest/payload fails"
setup_AddMismatched() {
   setup_AddDeDuplicate
}
test_AddMismatched() {
   # Try to add another file using an existing manifest, should fail with status
   # code indicating inconsistency.
   cp file1.manifest file1_2.manifest
   # Exit status 6 means manifest and payload do not match (filesize/filehash).
   execute --exit-status=6 --stderr --core-backtrace $servald rhizome add file $SIDA file1_2 file1_2.manifest
   tfw_cat file1.manifest file1_2.manifest
   # Output manifest should be the same as the re-used manigfest
   assert --stderr cmp file1.manifest file1_2.manifest
   # And rhizome store should be unchanged.
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 --author=$SIDA file1 file2
}

doc_AddUpdateSameVersion="Add new payload to existing manifest with same version fails"
setup_AddUpdateSameVersion() {
   setup_AddDeDuplicate
   cp file1.manifest file1_2.manifest
   strip_signatures file1_2.manifest
   $SED -i -e '/^date=/d;/^filehash=/d;/^filesize=/d' file1_2.manifest
   tfw_cat -v file1_2.manifest
   assert_manifest_fields file1_2.manifest !date !filehash !filesize version id
   cp file1_2.manifest file1_2.manifest.orig
}
test_AddUpdateSameVersion() {
   # Try to add another file using an existing manifest Id and Version, should
   # fail and update the manifest file to show existing bundle's manifest.
   tfw_cat -v file1_2.manifest
   execute $servald rhizome add file $SIDA file1_2 file1_2.manifest
   assertExitStatus --stderr '==' 1
   tfw_cat -v file1_2.manifest file1.manifest
   assert cmp file1_2.manifest file1.manifest
   # And rhizome store should be unchanged.
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 --author=$SIDA file1 file2
}

doc_AddUpdateNewVersion="Add new payload to existing manifest with new version"
setup_AddUpdateNewVersion() {
   setup_AddUpdateSameVersion
   extract_manifest_version version file1_2.manifest
   let version=version+1
   $SED -i -e "/^version=/s/=.*/=$version/" file1_2.manifest
   assert_manifest_fields file1_2.manifest version="$version"
}
test_AddUpdateNewVersion() {
   tfw_cat -v file1_2.manifest
   executeOk_servald rhizome add file $SIDA file1_2 file1_2.manifest
   tfw_cat --stderr
   assert_stdout_add_file file1_2 name=file1
   assert_manifest_newer file1.manifest file1_2.manifest
   # Rhizome store contents reflect new payload.
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 --author=$SIDA file1_2 file2
}

doc_AddUpdateDiscoverAuthor="Add new payload to manifest with author discovery"
setup_AddUpdateDiscoverAuthor() {
   setup_AddUpdateNewVersion
}
test_AddUpdateDiscoverAuthor() {
   tfw_cat -v file1_2.manifest
   executeOk_servald rhizome add file '' file1_2 file1_2.manifest
   tfw_cat --stderr
   # Rhizome store contents have new payload.
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 --author=$SIDA file1_2 file2
}

doc_AddUpdateNoAuthor="Cannot add new payload to authorless manifest"
setup_AddUpdateNoAuthor() {
   setup_servald
   setup_rhizome
   executeOk_servald rhizome list
   assert_rhizome_list
   echo "A test file" >file1
   echo "A test file, second version" >file1_2
   # Add first file
   executeOk_servald rhizome add file '' file1 file1.manifest
   extract_stdout_secret file1_secret
   assert_manifest_fields file1.manifest !BK
   # Create second manifest
   cp file1.manifest file1_2.manifest
   strip_signatures file1_2.manifest
   $SED -i -e '/^date=/d;/^filehash=/d;/^filesize=/d;/^version=/d' file1_2.manifest
   tfw_cat -v file1_2.manifest
}
test_AddUpdateNoAuthor() {
   execute $servald rhizome add file $SIDA file1_2 file1_2.manifest
   tfw_cat --stderr
   assertExitStatus '!=' 0
   # Rhizome store contents still have old payload
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 file1
}

doc_AddUpdateNoAuthorWithSecret="Add new payload to authorless manifest with bundle secret"
setup_AddUpdateNoAuthorWithSecret() {
   setup_AddUpdateNoAuthor
}
test_AddUpdateNoAuthorWithSecret() {
   executeOk_servald rhizome add file '' file1_2 file1_2.manifest "$file1_secret"
   tfw_cat --stderr
   # Rhizome store contents have new payload, but it has lost its author (no BK
   # field any more).
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 file1_2
}

doc_AddUpdateAutoVersion="Add new payload to existing manifest with automatic version"
setup_AddUpdateAutoVersion() {
   setup_AddUpdateSameVersion
   $SED -i -e '/^version=/d' file1_2.manifest
   assert_manifest_fields file1_2.manifest !version
}
test_AddUpdateAutoVersion() {
   tfw_cat -v file1_2.manifest
   sleep 0.001 # Ensure that at least one millisecond has elapsed
   executeOk_servald rhizome add file $SIDA file1_2 file1_2.manifest
   assert_manifest_newer file1.manifest file1_2.manifest
   # Rhizome store contents reflect new payload.
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 file1_2 file2
}

doc_AddUpdateArgs="Update existing bundle using manifest args"
setup_AddUpdateArgs() {
   setup_AddDeDuplicate
   extract_manifest_id BID file1.manifest
   cp file1.manifest file1_2.manifest
}
test_AddUpdateArgs() {
   sleep 0.001 # Ensure that at least one millisecond has elapsed
   executeOk_servald rhizome add file $SIDA file1_2 file1_2.manifest '' !version !filesize !filehash !date
   assert_stdout_add_file file1_2 name=file1
   assert_manifest_fields file1_2.manifest name=file1
   extract_manifest_id BID2 file1_2.manifest
   assert [ $BID = $BID2 ]
   assert_manifest_newer file1.manifest file1_2.manifest
   # Rhizome store contents reflect new payload.
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 file1_2 file2
}

doc_AddServiceInvalid="Add with invalid service fails"
setup_AddServiceInvalid() {
   setup_servald
   setup_rhizome
   echo "Message1" >file1
   echo 'service=Fubar!' >file1.manifest
}
test_AddServiceInvalid() {
   execute $servald rhizome add file $SIDA file1 file1.manifest
   tfw_cat --stdout --stderr
   assertExitStatus '!=' 0
}

doc_AddServiceUnsupported="Add with unsupported service succeeds"
setup_AddServiceUnsupported() {
   setup_servald
   setup_rhizome
   echo "Message1" >file1
   echo 'service=Fubar' >file1.manifest
}
test_AddServiceUnsupported() {
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   tfw_cat --stdout --stderr
}

doc_AddUpdateNoAuthorWithPassphrase="Add and update authorless bundle with only secret passphrase"
setup_AddUpdateNoAuthorWithPassphrase() {
   setup_servald
   setup_rhizome
   create_file file1 1k
   create_file file1_2 2k
   pass='On the Ning Nang Nong'
}
test_AddUpdateNoAuthorWithPassphrase() {
   executeOk_servald rhizome add file '' file1 file1.manifest "#$pass"
   tfw_cat --stdout --stderr
   assert_stdout_add_file file1 !.author '!BK'
   assert_manifest_complete file1.manifest
   extract_manifest_id BID file1.manifest
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 file1
   executeOk_servald rhizome add file '' file1_2 file1_2.manifest "#$pass"
   tfw_cat --stdout --stderr
   assert_stdout_add_file file1_2 !.author '!BK'
   assert_manifest_complete file1_2.manifest
   extract_manifest_id BID2 file1.manifest
   assert [ "$BID" = "$BID2" ]
   executeOk_servald rhizome extract file $BID file1x
   tfw_cat --stdout --stderr
   assert diff file1_2 file1x
}

doc_AddUpdateWithPassphrase="Add and update bundle using secret passphrase"
setup_AddUpdateWithPassphrase() {
   setup_servald
   setup_rhizome
   create_file file1 1k
   create_file file1_2 2k
   pass='On the Ning Nang Nong'
}
test_AddUpdateWithPassphrase() {
   executeOk_servald rhizome add file $SIDA file1 file1.manifest "#$pass"
   tfw_cat --stdout --stderr
   assert_stdout_add_file file1 .author=$SIDA
   assert_manifest_complete file1.manifest
   extract_manifest_id BID file1.manifest
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 file1
   executeOk_servald rhizome add file $SIDA file1_2 file1_2.manifest "#$pass"
   tfw_cat --stdout --stderr
   assert_stdout_add_file file1_2 .author=$SIDA
   assert_manifest_complete file1_2.manifest
   extract_manifest_id BID2 file1.manifest
   assert [ "$BID" = "$BID2" ]
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 file1_2
   executeOk_servald rhizome extract file $BID file1x
   tfw_cat --stdout --stderr
   assert diff file1_2 file1x
}

doc_EncryptedPayload="Add and extract an encrypted payload"
setup_EncryptedPayload() {
   setup_servald
   setup_rhizome
   echo "Clear Text" >file1
   echo -e "service=file\nname=private\ncrypt=1" >file1.manifest
}
test_EncryptedPayload() {
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   tfw_cat --stdout --stderr
   assert_stdout_add_file file1
   assert_manifest_complete file1.manifest
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 file1
   extract_manifest_id BID file1.manifest
   executeOk_servald rhizome extract file $BID file1x
   tfw_cat --stdout --stderr
   assert diff file1 file1x
   extract_manifest_filehash filehash file1.manifest
   executeOk_servald rhizome export file $filehash file1y
   assert ! diff file1 file1y
}

doc_RecipientIsEncrypted="Setting recipient triggers encryption by default"
setup_RecipientIsEncrypted() {
   A_IDENTITY_COUNT=2
   setup_servald
   setup_rhizome
   echo "Clear Text1" >file1
   echo -e "service=file\nsender=$SIDA1\nrecipient=$SIDA2" >file1.manifest
   echo "Clear Text2" >file2
   echo -e "service=file\nrecipient=$SIDA2" >file2.manifest
}
test_RecipientIsEncrypted() {
# add file with sender & recipient
   executeOk_servald rhizome add file $SIDA1 file1 file1.manifest
   tfw_cat --stdout --stderr
   assert_stdout_add_file file1
   assert_manifest_complete file1.manifest
   assertGrep file1.manifest "crypt=1"
# add file sith only recipient
   executeOk_servald rhizome add file $SIDA1 file2 file2.manifest
   tfw_cat --stdout --stderr
   assert_stdout_add_file file2
   assert_manifest_complete file2.manifest
   assertGrep file2.manifest "crypt=1"
# test that both files can be decrypted
   extract_manifest_id BID1 file1.manifest
   extract_manifest_filehash filehash1 file1.manifest
   extract_manifest_id BID2 file2.manifest
   extract_manifest_filehash filehash2 file2.manifest
   executeOk_servald rhizome extract file $BID1 file1x
   tfw_cat --stdout --stderr
   assert diff file1 file1x
   executeOk_servald rhizome extract file $BID2 file2x
   tfw_cat --stdout --stderr
   assert diff file2 file2x
# and that the stored files are obscured
   executeOk_servald rhizome export file $filehash1 file1y
   tfw_cat file1 -v file1y
   assert ! diff file1 file1y
   executeOk_servald rhizome export file $filehash2 file2y
   tfw_cat file2 -v file2y
   assert ! diff file2 file2y
}

doc_BroadcastNotEncrypted="Broadcast recipients are not encrypted by default"
setup_BroadcastNotEncrypted() {
   setup_servald
   setup_rhizome
   echo "Clear Text" >file1
   echo -e "service=file\nsender=$SIDA\nrecipient=FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" >file1.manifest
}
test_BroadcastNotEncrypted() {
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   tfw_cat --stdout --stderr
   assert_stdout_add_file file1
   assert_manifest_complete file1.manifest
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 file1
   extract_manifest_filehash filehash file1.manifest
   executeOk_servald rhizome export file $filehash file1y
   assert diff file1 file1y
}

doc_AddReuseManifest="Add --bundle copies fields from existing manifest"
setup_AddReuseManifest() {
   B_IDENTITY_COUNT=1
   setup_servald
   setup_rhizome
   echo "First content" >file1
   echo "Second content" >file1a
   echo -e "service=wazoo\nunexpected=true\nrecipient=$SIDB1\nnothing=here" >file1.manifest
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   tfw_cat --stderr
   extract_stdout_manifestid BID
}
test_AddReuseManifest() {
   executeOk_servald rhizome add file --bundle=$BID $SIDA file1a file1a.manifest '' !nothing something=there
   tfw_cat --stderr
   assert_manifest_fields file1a.manifest service=wazoo unexpected=true recipient=$SIDB1 !nothing something=there
   executeOk_servald rhizome export bundle $BID file1ax.manifest file1ax
   assert diff file1a file1ax
}

doc_JournalAppend="Create and append to a journal"
setup_JournalAppend() {
   setup_servald
   setup_rhizome
   echo "Part One" > file1
   echo "Part Two" > file2
   cat file1 file2 > file
}
test_JournalAppend() {
   executeOk_servald rhizome journal append $SIDA "" file1
   tfw_cat --stdout --stderr
   assert_stdout_add_file file1
   extract_stdout_manifestid BID
   executeOk_servald rhizome journal append $SIDA $BID file2
   tfw_cat --stdout --stderr
   executeOk_servald rhizome extract file $BID filex
   tfw_cat --stdout --stderr
   assert diff file filex
}

doc_JournalAppendNoHash="Appending to a journal should not rehash"
setup_JournalAppendNoHash() {
   setup_servald
   setup_rhizome
   executeOk_servald config \
	set rhizome.max_blob_size 0 \
	set debug.rhizome on
   echo "Part One" > file1
   echo "Part Two" > file2
   cat file1 file2 > file
}
test_JournalAppendNoHash() {
   executeOk_servald rhizome journal append $SIDA "" file1
   tfw_cat --stdout --stderr
   assert_stdout_add_file file1
   extract_stdout_manifestid BID
   assert [ $(ls "$SERVALINSTANCE_PATH/hash" | wc -l) -eq 1 ]
   executeOk_servald rhizome journal append $SIDA $BID file2
   tfw_cat --stdout --stderr
   assert [ $(ls "$SERVALINSTANCE_PATH/hash" | wc -l) -eq 2 ]
   assertStderrGrep 'Reusing journal'
   executeOk_servald rhizome extract file $BID filex
   tfw_cat --stdout --stderr
   assert diff file filex
}

doc_JournalAppendSharedPayload="Journal append produces a shared payload"
setup_JournalAppendSharedPayload() {
   setup_servald
   setup_rhizome
   executeOk_servald config set rhizome.max_blob_size 0
   create_file file1 101
   >manifest1
   create_file file2 102
   >manifest2
   cat file1 file2 >file12
   executeOk_servald rhizome add file '' file1
   extract_stdout_filehash HASH1
   assert cmp file1 "$SERVALINSTANCE_PATH/blob/$HASH1"
   executeOk_servald rhizome add file '' file12
   extract_stdout_filehash HASH12
   assert cmp file12 "$SERVALINSTANCE_PATH/blob/$HASH12"
   assert [ $(ls "$SERVALINSTANCE_PATH/blob" | wc -l) -eq 2 ]
}
test_JournalAppendSharedPayload() {
   executeOk_servald rhizome journal append $SIDA "" file1
   tfw_cat --stdout --stderr
   assert_stdout_add_file file1
   extract_stdout_filehash addedhash
   assert [ $addedhash = $HASH1 ]
   assert [ $(ls "$SERVALINSTANCE_PATH/blob" | wc -l) -eq 2 ]
   extract_stdout_manifestid BID
   executeOk_servald rhizome journal append $SIDA $BID file2
   tfw_cat --stdout --stderr
   assert_stdout_add_file file12 !name
   extract_stdout_filehash addedhash
   assert [ $addedhash = $HASH12 ]
   assert [ $(ls "$SERVALINSTANCE_PATH/blob" | wc -l) -eq 2 ]
}

doc_JournalAddCreate="Cannot create a journal using file add"
setup_JournalAddCreate() {
   setup_servald
   setup_rhizome
   echo "Part One" > file1
   echo "tail=0" >file1.manifest
}
test_JournalAddCreate() {
   # TODO: servald should return a status code reserved for this case, instead
   # of the 4 error (which means "invalid manifest")
   execute --exit-status=4 --stderr $servald rhizome add file $SIDA file1 file1.manifest
   tfw_cat --stdout --stderr
}

doc_JournalAddUpdate="Cannot update a journal using file add"
setup_JournalAddUpdate() {
   setup_servald
   setup_rhizome
   echo "Part One" > file1
   executeOk_servald rhizome journal append $SIDA "" file1
   assert_stdout_add_file file1
   extract_stdout_manifestid BID
   executeOk_servald rhizome extract bundle $BID file1x.manifest file1x
   assert diff file1 file1x
   extract_manifest_version version file1x.manifest
   let version=version+1
   $SED -i -e "/^version=/s/=.*/=$version/" file1x.manifest
   assert_manifest_fields file1x.manifest version="$version"
}
test_JournalAddUpdate() {
   # TODO: servald should return a status code reserved for this case, instead
   # of the 4 error (which means "invalid manifest")
   execute --exit-status=4 $servald rhizome add file $SIDA file1x file1x.manifest
   tfw_cat --stdout --stderr
}

doc_AppendFile="Cannot append to a non-journal"
setup_AppendFile() {
   setup_servald
   setup_rhizome
   echo "Part One" > file1
   echo "Part Two" > file2
   executeOk_servald rhizome add file $SIDA file1
   tfw_cat --stdout --stderr
   assert_stdout_add_file file1
   extract_stdout_manifestid BID
}
test_AppendFile() {
   # TODO: servald should return a status code reserved for this case, instead
   # of the 4 error (which means "invalid manifest")
   execute --exit-status=4 $servald rhizome journal append $SIDA $BID file2
   tfw_cat --stdout --stderr
}

doc_MeshMSAddCreate="First add MeshMS creates manifest"
setup_MeshMSAddCreate() {
   B_IDENTITY_COUNT=1
   setup_servald
   setup_rhizome
   echo "Message1" >file1
   echo -e "service=MeshMS1\nsender=$SIDA\nrecipient=$SIDB1" >file1.manifest
}
test_MeshMSAddCreate() {
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   assert_stdout_add_file file1
   assert_manifest_complete file1.manifest
   extract_manifest_crypt crypt file1.manifest
   assert [ $crypt = 1 ]
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 file1
   extract_manifest_id BID file1.manifest
   executeOk_servald rhizome extract file $BID file1x
   assert diff file1 file1x
}

doc_MeshMSAddGrow="Subsequent add MeshMS updates manifest and removes old payload"
setup_MeshMSAddGrow() {
   B_IDENTITY_COUNT=1
   setup_servald
   setup_rhizome
   executeOk_servald config set rhizome.clean_on_open on
   export SERVALD_ORPHAN_PAYLOAD_PERSIST_MS=0
   echo "Message1" >file1
   echo -e "service=MeshMS1\nsender=$SIDA\nrecipient=$SIDB1" >file1.manifest
}
test_MeshMSAddGrow() {
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   assert_stdout_add_file file1
   assert_manifest_complete file1.manifest
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 file1
   extract_manifest_id id file1.manifest
   extract_manifest_filehash filehash file1.manifest
   extract_manifest_BK bk file1.manifest
   local -a ofilehashes=()
   for m in 2 3 4 5; do
      ofilehashes+=("$filehash")
      echo -e "id=$id\nBK=$bk\nservice=MeshMS1\nsender=$SIDA\nrecipient=$SIDB1" >file1.manifest
      echo "Message$m" >>file1
      executeOk_servald rhizome add file $SIDA file1 file1.manifest
      executeOk_servald rhizome list
      assert_rhizome_list --fromhere=1 file1
      extract_manifest_id idx file1.manifest
      extract_manifest_filehash filehashx file1.manifest
      extract_manifest_BK bkx file1.manifest
      assert --message="manifest ID remains the same" [ "$idx" = "$id" ]
      assert --message="manifest BK remains the same" [ "$bkx" = "$bk" ]
      assert --message="filehash is for new file" [ "$filehash" != "$filehashx" ]
      executeOk_servald rhizome extract file "$id" file1x
      assert --message="extracted payload is correct" diff file1 file1x
      for ofilehash in "${ofilehashes[@]}"; do
         execute --exit-status=1 --stderr $servald rhizome export file "$ofilehash"
      done
      filehash="$filehashx"
   done
}

doc_MeshMSAddMissingSender="Add MeshMS without sender fails"
setup_MeshMSAddMissingSender() {
   B_IDENTITY_COUNT=1
   setup_servald
   setup_rhizome
   echo "Message1" >file1
   echo -e "service=MeshMS1\nrecipient=$SIDB1" >file1.manifest
}
test_MeshMSAddMissingSender() {
   execute $servald rhizome add file $SIDA file1 file1.manifest
   assertExitStatus --stdout --stderr '!=' 0
}

doc_MeshMSAddMissingRecipient="Add MeshMS without recipient fails"
setup_MeshMSAddMissingRecipient() {
   setup_servald
   setup_rhizome
   executeOk_servald rhizome list
   assert_rhizome_list
   echo "Message1" >file1
   echo -e "service=MeshMS1\nsender=$SIDA" >file1.manifest
}
test_MeshMSAddMissingRecipient() {
   execute $servald rhizome add file $SIDA file1 file1.manifest
   assertExitStatus '!=' 0
}

doc_MeshMSAddMissingAuthor="Add MeshMS without author uses sender"
setup_MeshMSAddMissingAuthor() {
   A_IDENTITY_COUNT=2
   setup_servald
   setup_rhizome
   echo "Message1" >file1
   echo -e "service=MeshMS1\nsender=$SIDA1\nrecipient=$SIDA2" >file1.manifest
}
test_MeshMSAddMissingAuthor() {
   executeOk_servald rhizome add file '' file1 file1.manifest
   assert_stdout_add_file file1 !.author !BK
   assert_manifest_complete file1.manifest
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 file1
}

doc_ListFilter="List manifests by filter"
setup_ListFilter() {
   setup_servald
   setup_rhizome
   echo "File1" > file1
   echo "File2" > file2
   echo "File3" > file3
   echo "File4" > file4
   executeOk_servald rhizome add file '' file1 file1.manifest
   assert_stdout_add_file file1 !.author !BK
   executeOk_servald rhizome add file '' file2 file2.manifest
   assert_stdout_add_file file2 !.author !BK
   executeOk_servald rhizome add file '' file3 file3.manifest
   assert_stdout_add_file file3 !.author !BK
   executeOk_servald rhizome add file '' file4 file4.manifest
   assert_stdout_add_file file4 !.author !BK
}
test_ListFilter() {
   executeOk_servald rhizome list file
   assert_rhizome_list file1 file2 file3 file4
   executeOk_servald rhizome list file 'file%'
   assert_rhizome_list file1 file2 file3 file4
   executeOk_servald rhizome list '' 'file2'
   assert_rhizome_list file2
}

doc_MeshMSListFilter="List MeshMS manifests by filter"
setup_MeshMSListFilter() {
   A_IDENTITY_COUNT=4
   setup_servald
   setup_rhizome
   echo "Message1" >file1
   echo -e "service=MeshMS1\nsender=$SIDA1\nrecipient=$SIDA2" >file1.manifest
   echo "Message2" >file2
   echo -e "service=MeshMS1\nsender=$SIDA1\nrecipient=$SIDA3" >file2.manifest
   echo "Message3" >file3
   echo -e "service=MeshMS1\nsender=$SIDA1\nrecipient=$SIDA4" >file3.manifest
   echo "Message3" >file4
   echo -e "service=MeshMS1\nsender=$SIDA2\nrecipient=$SIDA3" >file4.manifest
   executeOk_servald rhizome add file '' file1 file1.manifest
   assert_stdout_add_file file1 !.author !BK
   assert_manifest_complete file1.manifest
   executeOk_servald rhizome add file '' file2 file2.manifest
   assert_stdout_add_file file2 !.author !BK
   assert_manifest_complete file2.manifest
   executeOk_servald rhizome add file '' file3 file3.manifest
   assert_stdout_add_file file3 !.author !BK
   assert_manifest_complete file3.manifest
   executeOk_servald rhizome add file '' file4 file4.manifest
   assert_stdout_add_file file4 !.author !BK
   assert_manifest_complete file4.manifest
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 file1 file2 file3 file4
}
test_MeshMSListFilter() {
   executeOk_servald rhizome list file
   assert_rhizome_list
   executeOk_servald rhizome list MeshMS1
   assert_rhizome_list --fromhere=1 file1 file2 file3 file4
   executeOk_servald rhizome list '' '' $SIDA1
   assert_rhizome_list --fromhere=1 file1 file2 file3
   executeOk_servald rhizome list '' '' $SIDA2
   assert_rhizome_list --fromhere=1 file4
   executeOk_servald rhizome list '' '' $SIDA3
   assert_rhizome_list
   executeOk_servald rhizome list '' '' $SIDA4
   assert_rhizome_list
   executeOk_servald rhizome list '' '' '' $SIDA1
   assert_rhizome_list
   executeOk_servald rhizome list '' '' '' $SIDA2
   assert_rhizome_list --fromhere=1 file1
   executeOk_servald rhizome list '' '' '' $SIDA3
   assert_rhizome_list --fromhere=1 file2 file4
   executeOk_servald rhizome list file '' '' $SIDA3
   assert_rhizome_list
   executeOk_servald rhizome list '' '' '' $SIDA4
   assert_rhizome_list --fromhere=1 file3
   executeOk_servald rhizome list '' '' $SIDA1 $SIDA4
   assert_rhizome_list --fromhere=1 file3
   executeOk_servald rhizome list '' '' $SIDA2 $SIDA4
   assert_rhizome_list
   executeOk_servald rhizome list '' '' $SIDA2 $SIDA3
   assert_rhizome_list --fromhere=1 file4
}

doc_ImportForeignBundle="Import a bundle created by another instance"
setup_ImportForeignBundle() {
   B_IDENTITY_COUNT=1
   setup_servald
   setup_rhizome
   set_instance +A
   echo "Hello from A" >fileA
   executeOk_servald rhizome add file $SIDA fileA fileA.manifest
   assert_stdout_add_file fileA
   set_instance +B
}
test_ImportForeignBundle() {
   executeOk_servald rhizome import bundle fileA fileA.manifest
   assert_stdout_import_bundle fileA
   # Exit status 1 means bundle is already in store
   execute --exit-status=1 --stdout --stderr $servald rhizome import bundle fileA fileA.manifest
   assert_stdout_import_bundle fileA
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 fileA
}

doc_ImportOwnBundle="Import a bundle created by same instance"
setup_ImportOwnBundle() {
   A_IDENTITY_COUNT=0
   B_IDENTITY_COUNT=2
   setup_servald
   setup_rhizome
   set_instance +B
   echo "Hello from B" >fileB
   executeOk_servald rhizome add file $SIDB2 fileB fileB.manifest
   assert_stdout_add_file fileB
   extract_stdout_rowid rowid
   extract_manifest_id manifestid fileB.manifest
   extract_manifest_version version fileB.manifest
   extract_manifest_filehash filehash fileB.manifest
   extract_manifest_BK BK fileB.manifest
   extract_manifest_date date fileB.manifest
   assert [ -e "$SERVALINSTANCE_PATH/rhizome.db" ]
   rm -f "$SERVALINSTANCE_PATH/rhizome.db"
   executeOk_servald rhizome list
   assert_rhizome_list
}
test_ImportOwnBundle() {
   executeOk_servald rhizome import bundle fileB fileB.manifest
   assert_stdout_import_bundle fileB
   # Bundle author and sender are unknown, so appears not to be from here
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 fileB
   # Extracting the manifest discovers that it is ours.
   executeOk_servald rhizome extract bundle $manifestid fileBx.manifest fileBx
   tfw_cat --stderr
   assert cmp fileB.manifest fileBx.manifest
   assert cmp fileB fileBx
   assertStdoutLineCount '==' 13
   local size=$(( $(cat fileB | wc -c) + 0 ))
   assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
   assertStdoutGrep --matches=1 "^version:$version\$"
   assertStdoutGrep --matches=1 "^filesize:$size\$"
   assertStdoutGrep --matches=1 "^filehash:$filehash\$"
   assertStdoutGrep --matches=1 "^BK:$BK\$"
   assertStdoutGrep --matches=1 "^date:$date\$"
   assertStdoutGrep --matches=1 "^service:file\$"
   assertStdoutGrep --matches=1 "^name:fileB\$"
   assertStdoutGrep --matches=1 "^\.readonly:0\$"
   assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
   assertStdoutGrep --matches=1 "^\.author:$SIDB2\$"
   assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
   assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
   # Now bundle author should be known, so appears to be from here
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=1 --author=$SIDB2 fileB
}

doc_ImportCombinedBundle="Create and import combined bundle"
setup_ImportCombinedBundle() {
   # A "combined bundle" is a single file consisting of a payload with its
   # manifest appended to the end.
   setup_servald
   setup_rhizome
   echo "Hello from A" >fileA
   executeOk_servald rhizome add file $SIDA fileA fileA.manifest
   assert_stdout_add_file fileA
   extract_manifest_id manifestid fileA.manifest
   extract_manifest_filehash filehash fileA.manifest
   extract_manifest_filesize filesize fileA.manifest
}
test_ImportCombinedBundle() {
   # Create the combined bundle
   executeOk_servald rhizome append manifest fileA fileA.manifest
   # Import the combined bundle
   set_instance +B
   executeOk_servald rhizome import bundle fileA fileA
   assertStdoutGrep --matches=1 "^service:file$"
   assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
   assertStdoutGrep --matches=1 "^filehash:$filehash\$"
   assertStdoutGrep --matches=1 "^filesize:$filesize\$"
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 fileA
   executeOk_servald rhizome export bundle $manifestid fileAx fileAx
   assert diff fileA fileAx
}

doc_ImportCombinedZipBundle="Create and import combined zip bundle"
setup_ImportCombinedZipBundle() {
   # A "combined bundle" is a single file consisting of a payload with its
   # manifest appended to the end.
   setup_servald
   setup_rhizome
   echo "Hello from A" >fileA
   zip fileA.zip fileA
}
test_ImportCombinedZipBundle() {
   # Create the combined bundle
   executeOk_servald rhizome add file --zip-comment $SIDA fileA.zip fileA.manifest
   extract_manifest_id manifestid fileA.manifest
   extract_manifest_filehash filehash fileA.manifest
   extract_manifest_filesize filesize fileA.manifest
   # Import the combined bundle
   set_instance +B
   executeOk_servald rhizome import bundle --zip-comment fileA.zip fileA.zip
   assertStdoutGrep --matches=1 "^service:file$"
   assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
   assertStdoutGrep --matches=1 "^filehash:$filehash\$"
   assertStdoutGrep --matches=1 "^filesize:$filesize\$"
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 fileA.zip
   executeOk_servald rhizome export bundle --zip-comment $manifestid fileAx.zip fileAx.zip
   assert diff fileA.zip fileAx.zip
}

doc_ImportJournal="Import a journal bundle"
setup_ImportJournal() {
   B_IDENTITY_COUNT=1
   setup_servald
   setup_rhizome
   echo "Part One" > file1
   echo "Part Two2" > file2
   cat file1 file2 >file3
   executeOk_servald rhizome journal append $SIDA "" file1
   assert_stdout_add_file file1
   extract_stdout_manifestid BID
   executeOk_servald rhizome extract bundle $BID file1x.manifest file1x
   assert diff file1 file1x
}
test_ImportJournal() {
   # Import a journal bundle from a foreign instance.
   set_instance +B
   executeOk_servald rhizome import bundle file1x file1x.manifest
   assert_stdout_import_bundle file1x
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 file1x
   # Try to import the same journal bundle again, exit status 1 means bundle is
   # already in store.
   execute --exit-status=1 --stdout --stderr $servald rhizome import bundle file1x file1x.manifest
   assert_stdout_import_bundle file1x
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 file1x
   # Grow the journal and import it again.
   begin_fixture
   set_instance +A
   executeOk_servald rhizome journal append $SIDA $BID file2
   assert_stdout_add_file file3 manifestid=$BID name=file1
   executeOk_servald rhizome extract bundle $BID file3x.manifest file3x
   assert diff file3 file3x
   end_fixture
   set_instance +B
   executeOk_servald rhizome import bundle file3x file3x.manifest
   assert_stdout_import_bundle file3x
   executeOk_servald rhizome list
   assert_rhizome_list --fromhere=0 file3x
}

setup_delete() {
   setup_servald
   setup_rhizome
   executeOk_servald config set rhizome.clean_on_open off
   rhizome_add_files file{1..4}
   for i in {1..4}; do
      extract_manifest_id BID$i file$i.manifest
      extract_manifest_filehash HASH$i file$i.manifest
   done
}
rhizome_clean() {
   executeOk_servald rhizome clean
   extract_stdout_keyvalue deleted_files 'deleted_orphan_files' '[0-9]\+'
   extract_stdout_keyvalue deleted_fileblobs 'deleted_orphan_fileblobs' '[0-9]\+'
   extract_stdout_keyvalue deleted_manifests 'deleted_orphan_manifests' '[0-9]\+'
}

doc_DeleteManifest="Delete a manifest from store"
setup_DeleteManifest() {
   setup_delete
}
test_DeleteManifest() {
   executeOk_servald rhizome delete manifest "$BID2"
   tfw_cat --stderr
   executeOk_servald rhizome list
   assert_rhizome_list file{1,3,4}
   execute --exit-status=1 --stderr $servald rhizome export manifest "$BID2"
   executeOk_servald rhizome export file "$HASH2" file2x
   assert diff file2 file2x
   rhizome_clean
   assert [ $deleted_files = 1 ]
   assert [ $deleted_fileblobs = 0 ]
   assert [ $deleted_manifests = 0 ]
}

doc_DeletePayload="Delete a payload from store"
setup_DeletePayload() {
   setup_delete
}
test_DeletePayload() {
   executeOk_servald rhizome delete payload "$BID3"
   tfw_cat --stderr
   executeOk_servald rhizome list
   assert_rhizome_list file{1..4}
   executeOk_servald rhizome export manifest "$BID3"
   execute --exit-status=1 --stderr $servald rhizome export file "$HASH3" file3x
   rhizome_clean
   assert [ $deleted_files = 0 ]
   assert [ $deleted_fileblobs = 0 ]
   assert [ $deleted_manifests = 1 ]
   executeOk_servald rhizome list
   assert_rhizome_list file{1,2,4}
}

doc_DeleteBundle="Delete a bundle from store"
setup_DeleteBundle() {
   setup_delete
}
test_DeleteBundle() {
   executeOk_servald rhizome delete bundle "$BID4"
   tfw_cat --stderr
   executeOk_servald rhizome list
   assert_rhizome_list file{1..3}
   execute --exit-status=1 --stderr $servald rhizome export manifest "$BID4"
   execute --exit-status=1 --stderr $servald rhizome export file "$HASH4" file4x
   rhizome_clean
   assert [ $deleted_files = 0 ]
   assert [ $deleted_fileblobs = 0 ]
   assert [ $deleted_manifests = 0 ]
}

doc_DeleteFile="Delete a file from store"
setup_DeleteFile() {
   setup_delete
}
test_DeleteFile() {
   executeOk_servald rhizome delete file "$HASH1"
   tfw_cat --stderr
   executeOk_servald rhizome list
   assert_rhizome_list file{1..4}
   executeOk_servald rhizome export manifest "$BID1"
   execute --exit-status=1 --stderr $servald rhizome export file "$HASH1" file1x
   rhizome_clean
   assert [ $deleted_files = 0 ]
   assert [ $deleted_fileblobs = 0 ]
   assert [ $deleted_manifests = 1 ]
   executeOk_servald rhizome list
   assert_rhizome_list file{2..4}
}

doc_payloadTooBig="Fail to insert a payload that is larger than the database"
setup_payloadTooBig() {
   setup_servald
   setup_rhizome
   executeOk_servald config set rhizome.database_size 32K
}
test_payloadTooBig(){
   create_file file1 32K
   execute $servald rhizome add file $SIDA file1 file1.manifest
   assertExitStatus '==' 7
}

doc_payloadUninteresting="Fail to insert a payload that is uninteresting"
setup_payloadUninteresting() {
   setup_servald
   setup_rhizome
   executeOk_servald config set rhizome.database_size 64K
}
test_payloadUninteresting(){
   create_file file1 32K
   create_file file2 32K
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   execute $servald rhizome add file $SIDA file2 file2.manifest
   assertExitStatus '==' 7
}

doc_evictUninteresting="Evict a large payload to make room for smaller payloads"
setup_evictUninteresting() {
   setup_servald
   setup_rhizome
   executeOk_servald config set rhizome.database_size 1M
   create_file file1 512K
   create_file file2 256K
   create_file file3 128K
   create_file file4 128K
}
test_evictUninteresting(){
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   executeOk_servald rhizome add file $SIDA file2 file2.manifest
   executeOk_servald rhizome add file $SIDA file3 file3.manifest
   executeOk_servald rhizome add file $SIDA file4 file4.manifest
   tfw_cat --stderr
   rhizome_clean
   assert [ $deleted_files = 0 ]
   assert [ $deleted_fileblobs = 0 ]
   assert [ $deleted_manifests = 1 ]
   executeOk_servald rhizome list
   assert_rhizome_list file{2,3,4}
}

doc_evictFreeSpace="Reduce database size due to insufficient free space"
setup_evictFreeSpace() {
   setup_servald
   setup_rhizome
   create_file file1 512K
   create_file file2 256K
   create_file file3 128K
   create_file file4 128K
   executeOk_servald rhizome add file $SIDA file1 file1.manifest
   executeOk_servald rhizome add file $SIDA file2 file2.manifest
   executeOk_servald rhizome add file $SIDA file3 file3.manifest
   executeOk_servald rhizome add file $SIDA file4 file4.manifest
}
test_evictFreeSpace() {
   executeOk_servald config \
      set rhizome.min_free_space 1M
   # only 640K free...
   export SERVALD_FREE_SPACE=655360
   rhizome_clean
   assert [ $deleted_files = 0 ]
   assert [ $deleted_fileblobs = 0 ]
   assert [ $deleted_manifests = 1 ]
   executeOk_servald rhizome list
   assert_rhizome_list file{2,3,4}
}
runTests "$@"