#!/bin/bash # Tests for Serval DNA MeshMS REST API # # Copyright 2013-2014 Serval Project, Inc. # Copyright 2016-2018 Flinders University # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. source "${0%/*}/../testframework.sh" source "${0%/*}/../testdefs.sh" source "${0%/*}/../testdefs_rest.sh" source "${0%/*}/../testdefs_meshms.sh" shopt -s extglob setup_instance() { set_instance "$1" setup_rest_config set_meshms_config set_extra_config if [ -z "$IDENTITY_COUNT" ]; then create_single_identity else create_identities "$IDENTITY_COUNT" fi } setup() { setup_rest_utilities setup_servald export SERVALD_RHIZOME_DB_RETRY_LIMIT_MS=60000 setup_instance +A start_servald_instances +A wait_until_rest_server_ready +A } finally() { stop_all_servald_servers } teardown() { kill_all_servald_processes assert_no_servald_processes report_all_servald_servers } set_extra_config() { : } set_meshms_config() { executeOk_servald config \ set debug.meshms on \ set debug.verbose on \ set log.console.level debug } doc_AuthBasicMissing="REST API missing Basic Authentication credentials" test_AuthBasicMissing() { rest_request GET "/restful/meshms/$SIDA/conversationlist.json" 401 --no-auth assertGrep response.headers "^WWW-Authenticate: Basic realm=\"Serval RESTful API\"$CR\$" assertJq response.json 'contains({"http_status_code": 401})' assertJq response.json 'contains({"http_status_message": ""})' } doc_AuthBasicWrong="REST API incorrect Basic Authentication credentials" test_AuthBasicWrong() { rest_request GET "/restful/meshms/$SIDA/conversationlist.json" 401 --user=fred:nurks assertGrep response.headers "^WWW-Authenticate: Basic realm=\"Serval RESTful API\"$CR\$" assertJq response.json 'contains({"http_status_code": 401})' assertJq response.json 'contains({"http_status_message": ""})' rest_request GET "/restful/meshms/$SIDA/conversationlist.json" 200 --user=ron:weasley } doc_MeshmsTalkToYourself="REST API talk to yourself" setup_MeshmsTalkToYourself() { IDENTITY_COUNT=1 setup } test_MeshmsTalkToYourself() { rest_request POST "/restful/meshms/$SIDA1/$SIDA1/sendmessage" 201 \ --form-part="message=Hello World;type=text/plain;charset=utf-8" rest_request GET "/restful/meshms/$SIDA1/conversationlist.json" rest_request GET "/restful/meshms/$SIDA1/$SIDA1/messagelist.json" } doc_MeshmsListConversations="REST API list MeshMS conversations as JSON" setup_MeshmsListConversations() { IDENTITY_COUNT=5 setup # create 3 threads, with all permutations of incoming and outgoing messages executeOk_servald meshms send message "$SIDA1" "$SIDA2" "Message1" executeOk_servald meshms send message "$SIDA3" "$SIDA1" "Message2" executeOk_servald meshms send message "$SIDA1" "$SIDA4" "Message3" executeOk_servald meshms send message "$SIDA4" "$SIDA1" "Message4" } test_MeshmsListConversations() { rest_request GET "/restful/meshms/$SIDA1/conversationlist.json" assert [ "$(jq '.rows | length' response.json)" = 3 ] transform_list_json response.json conversations1.json tfw_preserve conversations1.json assertJq conversations1.json \ "contains([ { my_sid: \"$SIDA1\", their_sid: \"$SIDA2\", read: true, last_message: 0, read_offset: 0 } ])" assertJq conversations1.json \ "contains([ { my_sid: \"$SIDA1\", their_sid: \"$SIDA3\", read: false, last_message: 11, read_offset: 0 } ])" assertJq conversations1.json \ "contains([ { my_sid: \"$SIDA1\", their_sid: \"$SIDA4\", read: false, last_message: 14, read_offset: 0 } ])" # mark all incoming messages as read executeOk_servald meshms read messages "$SIDA1" executeOk_servald meshms read messages "$SIDA1" "$SIDA3" tfw_cat --stderr rest_request GET "/restful/meshms/$SIDA1/conversationlist.json" assert [ "$(jq '.rows | length' response.json)" = 3 ] transform_list_json response.json conversations2.json tfw_preserve conversations2.json assertJq conversations2.json \ "contains([ { my_sid: \"$SIDA1\", their_sid: \"$SIDA2\", read: true, last_message: 0, read_offset: 0 } ])" assertJq conversations2.json \ "contains([ { my_sid: \"$SIDA1\", their_sid: \"$SIDA3\", read: true, last_message: 11, read_offset: 11 } ])" assertJq conversations2.json \ "contains([ { my_sid: \"$SIDA1\", their_sid: \"$SIDA4\", read: true, last_message: 14, read_offset: 14 } ])" } doc_MeshmsListMessages="REST API list MeshMS messages in one conversation as JSON" setup_MeshmsListMessages() { IDENTITY_COUNT=2 setup meshms_add_messages "$SIDA1" "$SIDA2" '><>>A>A<>><><><>>>A>A><<<<<>><>>A<<>' let NROWS=NSENT+NRECV+(NACK?1:0) executeOk_servald meshms list messages "$SIDA1" "$SIDA2" tfw_cat --stdout delivered_offset=$($SED -n -e '/^[0-9]\+:[0-9]\+:[0-9]\+:[0-9]\+:ACK:delivered$/{n;s/^[0-9]\+:\([0-9]\+\):[0-9]\+:[0-9]\+:>:.*/\1/p;q}' "$TFWSTDOUT") [ -z "$delivered_offset" ] && delivered_offset=0 read_offset=$($SED -n -e 's/^[0-9]\+:[0-9]\+:\([0-9]\+\):[0-9]\+:MARK:read$/\1/p' "$TFWSTDOUT") [ -z "$read_offset" ] && read_offset=0 tfw_log "delivered: $delivered_offset; read: $read_offset" } test_MeshmsListMessages() { rest_request GET "/restful/meshms/$SIDA1/$SIDA2/messagelist.json" assert [ "$(jq '.rows | length' response.json)" = $NROWS ] transform_list_json response.json messages.json tfw_preserve messages.json seen_ack=false let i=0 for ((j = NMESSAGE-1; j >= 0; --j)); do case ${MESSAGE[$j]} in 'ACK') $seen_ack && continue esac assertJq messages.json '(.['$i'].token | length) > 0' assertJq messages.json '.['$i'].my_sid == "'$SIDA1'"' assertJq messages.json '.['$i'].their_sid == "'$SIDA2'"' case ${MESSAGE[$j]} in '>') assertJq messages.json '.['$i'].type == ">"' assertJqIs messages.json '.['$i'].text' "${TEXT[$j]}" assertJq messages.json '.['$i'].delivered == (.['$i'].my_offset <= '$delivered_offset')' let ++i ;; '<') assertJq messages.json '.['$i'].type == "<"' assertJqIs messages.json '.['$i'].text' "${TEXT[$j]}" assertJq messages.json '.['$i'].read == (.['$i'].their_offset <= '$read_offset')' let ++i ;; 'ACK') assertJq messages.json '.['$i'].type == "ACK"' assertJq messages.json '.['$i'].text == null' assertJq messages.json '.['$i'].ack_offset == '$delivered_offset let ++i seen_ack=true ;; esac done } doc_MeshmsListMessagesNoIdentity="REST API list MeshMS messages from unknown identity" setup_MeshmsListMessagesNoIdentity() { setup SIDX=0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF } test_MeshmsListMessagesNoIdentity() { rest_request GET "/restful/meshms/$SIDX/$SIDA/messagelist.json" 419 assertJq response.json 'contains({"http_status_code": 419})' assertJqGrep --ignore-case response.json '.http_status_message' 'identity locked' assertJq response.json 'contains({"meshms_status_code": 2})' assertJqGrep --ignore-case response.json '.meshms_status_message' 'identity.*unknown' } doc_MeshmsEmptyNewSince="REST API list MeshMS since token with no messages" setup_MeshmsEmptyNewSince() { IDENTITY_COUNT=2 set_extra_config() { executeOk_servald config set api.restful.newsince_timeout 1s } setup } test_MeshmsEmptyNewSince() { rest_request GET "/restful/meshms/$SIDA1/$SIDA2/newsince/messagelist.json" } doc_MeshmsListMessagesNewSince="REST API list MeshMS messages in one conversation since token as JSON" setup_MeshmsListMessagesNewSince() { IDENTITY_COUNT=2 set_extra_config() { executeOk_servald config set api.restful.newsince_timeout 1s } setup meshms_add_messages "$SIDA1" "$SIDA2" '><>>A>A<>><><><>>>A>A><<<<<>><>>A<<>' let NROWS=NSENT+NRECV+(NACK?1:0) rest_request GET "/restful/meshms/$SIDA1/$SIDA2/messagelist.json" assert [ "$(jq '.rows | length' response.json)" = $NROWS ] transform_list_json response.json messages.json tfw_preserve messages.json for ((i = 0; i < NROWS; i += 3)); do token[$i]=$(jq --raw-output '.['$i'].token' messages.json) done } test_MeshmsListMessagesNewSince() { for ((i = 0; i < NROWS; i += 3)); do # At most five requests going at once [ $i -ge 15 ] && fork_wait %client$((i-15)) fork %client$i rest_request GET "/restful/meshms/$SIDA1/$SIDA2/newsince/${token[$i]}/messagelist.json" \ --output=response$i.json \ --no-buffer done fork_wait_all for ((i = 0; i < NROWS; i += 3)); do transform_list_json response$i.json messages$i.json tfw_preserve messages$i.json jq '.[:'$i']' messages.json > messages_expected$i.json tfw_preserve messages_expected$i.json { echo '{"a":'; cat messages_expected$i.json; echo ',"b":'; cat messages$i.json; echo '}'; } >tmp.json assertJq tmp.json '.a == .b' done } grepall() { local pattern="$1" shift for file; do grep "$pattern" "$file" || return $? done return 0 } doc_MeshmsListMessagesNewSinceArrival="REST API list newly arriving MeshMS messages in one conversation as JSON" setup_MeshmsListMessagesNewSinceArrival() { IDENTITY_COUNT=2 set_extra_config() { executeOk_servald config set api.restful.newsince_timeout 60s } setup # Use REST interface to send messages, not CLI, in order to avoid a database # locking storm meshms_use_restful harry potter meshms_add_messages "$SIDA1" "$SIDA2" '><>A>' let NROWS=NSENT+NRECV+(NACK?1:0) rest_request GET "/restful/meshms/$SIDA1/$SIDA2/messagelist.json" assert [ "$(jq '.rows | length' response.json)" = $NROWS ] transform_list_json response.json messages.json tfw_preserve messages.json token=$(jq --raw-output '.[0].token' messages.json) assert [ -n "$token" ] } test_MeshmsListMessagesNewSinceArrival() { for i in 1 2 3; do fork %client$i rest_request GET "/restful/meshms/$SIDA1/$SIDA2/newsince/$token/messagelist.json" \ --timeout=360 \ --output=response$i.json \ --no-buffer done wait_until [ -e response1.json -a -e response2.json -a -e response3.json ] for message in '>Rumplestiltskin' 'A' 'Eulenspiegel'; do meshms_add_messages "$SIDA1" "$SIDA2" "${message:0:1}" "${message:1}" case ${message:0:1} in '<'|'>') waitfor="${message:1}";; 'A') waitfor="ACK";; *) error "message=${message}";; esac wait_until --timeout=10 grepall "$waitfor" response{1,2,3}.json done fork_wait_all } doc_MeshmsSend="REST API send MeshMS message" setup_MeshmsSend() { IDENTITY_COUNT=2 setup } test_MeshmsSend() { rest_request POST "/restful/meshms/$SIDA1/$SIDA2/sendmessage" 201 \ --form-part="message=Hello World;type=text/plain;charset=utf-8" executeOk_servald meshms list messages "$SIDA1" "$SIDA2" assertStdoutGrep --matches=1 ':>:Hello World' rest_request POST "/restful/meshms/$SIDA2/$SIDA1/sendmessage" 201 \ --form-part="message=Hello back!;type=text/plain;charset=utf-8" executeOk_servald meshms list messages "$SIDA1" "$SIDA2" assertStdoutGrep --matches=1 ':>:Hello World$' assertStdoutGrep --matches=1 ':<:Hello back!$' } doc_MeshmsSendMissingMessage="REST API MeshMS send missing 'message' form part" setup_MeshmsSendMissingMessage() { IDENTITY_COUNT=2 setup } test_MeshmsSendMissingMessage() { rest_request POST "/restful/meshms/$SIDA2/$SIDA1/sendmessage" 400 --data='' assertJq response.json 'contains({"http_status_code": 400})' assertJqGrep --ignore-case response.json '.http_status_message' 'missing.*message.*form.*part' executeOk_servald meshms list messages "$SIDA1" "$SIDA2" assertStdoutLineCount '==' 2 } doc_MeshmsSendDuplicateMessage="REST API MeshMS send duplicate 'message' form parts" setup_MeshmsSendDuplicateMessage() { IDENTITY_COUNT=2 setup } test_MeshmsSendDuplicateMessage() { rest_request POST "/restful/meshms/$SIDA2/$SIDA1/sendmessage" 400 \ --form-part="message=Hello one;type=text/plain;charset=utf-8" \ --form-part="message=Hello two;type=text/plain;charset=utf-8" assertJq response.json 'contains({"http_status_code": 400})' assertJqGrep --ignore-case response.json '.http_status_message' 'duplicate.*message.*form.*part' executeOk_servald meshms list messages "$SIDA1" "$SIDA2" assertStdoutLineCount '==' 2 } doc_MeshmsSendMessageMissingContentType="REST API MeshMS send 'message' form part missing Content-Type" setup_MeshmsSendMessageMissingContentType() { IDENTITY_COUNT=2 setup } test_MeshmsSendMessageMissingContentType() { rest_request POST "/restful/meshms/$SIDA2/$SIDA1/sendmessage" 400 \ --form-part="message=Hello there" assertJq response.json 'contains({"http_status_code": 400})' assertJqGrep --ignore-case response.json '.http_status_message' 'missing.*content.*type' executeOk_servald meshms list messages "$SIDA1" "$SIDA2" assertStdoutLineCount '==' 2 } doc_MeshmsSendMessageUnsupportedContentType="REST API MeshMS send 'message' form part unsupported Content-Type" setup_MeshmsSendMessageUnsupportedContentType() { IDENTITY_COUNT=2 setup } test_MeshmsSendMessageUnsupportedContentType() { rest_request POST "/restful/meshms/$SIDA2/$SIDA1/sendmessage" 415 \ --form-part="message=Hello there;type=text/rich" assertJq response.json 'contains({"http_status_code": 415})' assertJqGrep --ignore-case response.json '.http_status_message' 'unsupported.*content.*type' executeOk_servald meshms list messages "$SIDA1" "$SIDA2" assertStdoutLineCount '==' 2 } doc_MeshmsSendMessageMissingCharset="REST API MeshMS send 'message' form part missing charset" setup_MeshmsSendMessageMissingCharset() { IDENTITY_COUNT=2 setup } test_MeshmsSendMessageMissingCharset() { rest_request POST "/restful/meshms/$SIDA2/$SIDA1/sendmessage" 400 \ --form-part="message=Hello there;type=text/plain" assertJq response.json 'contains({"http_status_code": 400})' assertJqGrep --ignore-case response.json '.http_status_message' 'missing.*charset' executeOk_servald meshms list messages "$SIDA1" "$SIDA2" assertStdoutLineCount '==' 2 } doc_MeshmsSendMessageUnsupportedCharset="REST API MeshMS send 'message' form part unsupported charset" setup_MeshmsSendMessageUnsupportedCharset() { IDENTITY_COUNT=2 setup } test_MeshmsSendMessageUnsupportedCharset() { rest_request POST "/restful/meshms/$SIDA2/$SIDA1/sendmessage" 415 \ --form-part="message=Hello there;type=text/plain;charset=latin-1" assertJq response.json 'contains({"http_status_code": 415})' assertJqGrep --ignore-case response.json '.http_status_message' 'unsupported.*charset' executeOk_servald meshms list messages "$SIDA1" "$SIDA2" assertStdoutLineCount '==' 2 } doc_MeshmsSendNoIdentity="REST API MeshMS send from unknown identity" setup_MeshmsSendNoIdentity() { setup SIDX=0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF } test_MeshmsSendNoIdentity() { rest_request POST "/restful/meshms/$SIDX/$SIDA/sendmessage" 419 \ --form-part="message=Hello;type=text/plain;charset=utf-8" assertJq response.json 'contains({"http_status_code": 419})' assertJqGrep --ignore-case response.json '.http_status_message' 'identity locked' assertJq response.json 'contains({"meshms_status_code": 2})' assertJqGrep --ignore-case response.json '.meshms_status_message' 'identity.*unknown' } doc_MeshmsReadAllConversations="REST API MeshMS mark all conversations read" setup_MeshmsReadAllConversations() { IDENTITY_COUNT=5 setup # create 3 threads, with all permutations of incoming and outgoing messages executeOk_servald meshms send message "$SIDA1" "$SIDA2" "Message1" executeOk_servald meshms send message "$SIDA3" "$SIDA1" "Message2" executeOk_servald meshms send message "$SIDA1" "$SIDA4" "Message3" executeOk_servald meshms send message "$SIDA4" "$SIDA1" "Message4" executeOk_servald meshms list conversations "$SIDA1" assertStdoutGrep --stderr --matches=1 ":$SIDA2::0:0\$" assertStdoutGrep --stderr --matches=1 ":$SIDA3:unread:11:0\$" assertStdoutGrep --stderr --matches=1 ":$SIDA4:unread:14:0\$" } test_MeshmsReadAllConversations() { rest_request POST "/restful/meshms/$SIDA1/readall" 201 executeOk_servald meshms list conversations "$SIDA1" assertStdoutGrep --stderr --matches=1 ":$SIDA2::0:0\$" assertStdoutGrep --stderr --matches=1 ":$SIDA3::11:11\$" assertStdoutGrep --stderr --matches=1 ":$SIDA4::14:14\$" } doc_MeshmsPostSpuriousContent="REST API MeshMS rejects unwanted content in POST request" setup_MeshmsPostSpuriousContent() { IDENTITY_COUNT=2 setup # create 3 threads, with all permutations of incoming and outgoing messages executeOk_servald meshms send message "$SIDA1" "$SIDA2" "Message1" executeOk_servald meshms send message "$SIDA2" "$SIDA1" "Message2" executeOk_servald meshms send message "$SIDA1" "$SIDA2" "Message3" executeOk_servald meshms send message "$SIDA2" "$SIDA1" "Message4" executeOk_servald meshms list conversations "$SIDA1" assertStdoutGrep --stderr --matches=1 ":$SIDA2:unread:35:0\$" } test_MeshmsPostSpuriousContent() { rest_request POST "/restful/meshms/$SIDA1/readall" 400 --form-part="offset=0" assertJq response.json 'contains({"http_status_code": 400})' assertJqGrep --ignore-case response.json '.http_status_message' 'content length' executeOk_servald meshms list conversations "$SIDA1" assertStdoutGrep --stderr --matches=1 ":$SIDA2:unread:35:0\$" } doc_MeshmsReadAllMessages="REST API MeshMS mark all conversations read" setup_MeshmsReadAllMessages() { IDENTITY_COUNT=5 setup # create 3 threads, with all permutations of incoming and outgoing messages executeOk_servald meshms send message "$SIDA1" "$SIDA2" "Message1" executeOk_servald meshms send message "$SIDA3" "$SIDA1" "Message2" executeOk_servald meshms send message "$SIDA1" "$SIDA2" "Message3" executeOk_servald meshms send message "$SIDA1" "$SIDA4" "Message4" executeOk_servald meshms send message "$SIDA4" "$SIDA1" "Message5" executeOk_servald meshms send message "$SIDA1" "$SIDA2" "Message6" executeOk_servald meshms list messages "$SIDA2" "$SIDA1" tfw_cat --stdout executeOk_servald meshms list conversations "$SIDA2" assertStdoutGrep --stderr --matches=1 ":$SIDA1:unread:45:0\$" } test_MeshmsReadAllMessages() { rest_request POST "/restful/meshms/$SIDA2/$SIDA1/readall" 201 executeOk_servald meshms list conversations "$SIDA2" assertStdoutGrep --stderr --matches=1 ":$SIDA1::45:45\$" } doc_MeshmsReadMessage="REST API MeshMS mark a message as read" setup_MeshmsReadMessage() { IDENTITY_COUNT=5 setup # create 3 threads, with all permutations of incoming and outgoing messages executeOk_servald meshms send message "$SIDA1" "$SIDA2" "Message1" executeOk_servald meshms send message "$SIDA3" "$SIDA1" "Message2" executeOk_servald meshms send message "$SIDA1" "$SIDA2" "Message3" executeOk_servald meshms send message "$SIDA1" "$SIDA4" "Message4" executeOk_servald meshms send message "$SIDA4" "$SIDA1" "Message5" executeOk_servald meshms send message "$SIDA1" "$SIDA2" "Message6" executeOk_servald meshms list conversations "$SIDA2" assertStdoutGrep --stderr --matches=1 ":$SIDA1:unread:45:0\$" } test_MeshmsReadMessage() { rest_request POST "/restful/meshms/$SIDA2/$SIDA1/recv/22/read" 201 executeOk_servald meshms list conversations "$SIDA2" assertStdoutGrep --stderr --matches=1 ":$SIDA1:unread:45:22\$" rest_request POST "/restful/meshms/$SIDA2/$SIDA1/recv/11/read" 200 executeOk_servald meshms list conversations "$SIDA2" assertStdoutGrep --stderr --matches=1 ":$SIDA1:unread:45:22\$" } doc_sendFlood="REST API send lots of MeshMS messages" setup_sendFlood() { setup_rest_utilities setup_servald export SERVALD_RHIZOME_DB_RETRY_LIMIT_MS=60000 set_extra_config() { executeOk_servald config set debug.rhizome_sync_keys on } setup_instance +A setup_instance +B start_servald_instances +A +B wait_until_rest_server_ready +A wait_until_rest_server_ready +B } test_sendFlood() { set_instance +A for (( j=0; j<5; j++ )) do for (( i=0; i<5; i++ )) do tfw_log "Sending message $i $j" rest_request +A POST "/restful/meshms/$SIDA/$SIDB/sendmessage" 201 \ --form-part="message=0123456789012345678901234567890123456789 $(date +%s%N) $j $i;type=text/plain;charset=utf-8" \ --output=request$i.json done # TODO wait for sync complete sleep 2 set_instance +B executeOk_servald rhizome list tfw_cat --stdout executeOk_servald route print tfw_cat --stdout rest_request GET "/restful/meshms/$SIDB/conversationlist.json" done set_instance +B rest_request GET "/restful/meshms/$SIDB/conversationlist.json" } doc_unicode="REST API MeshMS UTF-8 emoticons" setup_unicode() { IDENTITY_COUNT=2 setup } test_unicode() { SMILEY1=`echo -e "\xf0\x9f\x98\x80\x0a"` SMILEY2=`echo -e "\xf0\x9f\x98\x90\x0a"` executeOk_servald meshms send message "$SIDA1" "$SIDA2" "$SMILEY1" rest_request POST "/restful/meshms/$SIDA2/$SIDA1/sendmessage" 201 \ --form-part="message=$SMILEY2;type=text/plain;charset=utf-8" rest_request GET "/restful/meshms/$SIDA1/$SIDA2/messagelist.json" assertGrep --matches=1 response.json '\\uD83D\\uDE10' assertGrep --matches=1 response.json '\\uD83D\\uDE00' executeOk_servald meshms list messages "$SIDA1" "$SIDA2" tfw_cat --stdout assertStdoutGrep --matches=1 "$SMILEY1" assertStdoutGrep --matches=1 "$SMILEY2" } runTests "$@"