Merge branch 'whacktmps' into 'master'

commandline.c

@@ -83,20 +83,24 @@ static int outv_growbuf(size_t needed)
 
 static int outv_end_field()
 {
-  outv_growbuf(1);
+  size_t length = outv_current - outv_buffer;
-  *outv_current++ = '\0';
-  jstring str = (jstring)(*jni_env)->NewStringUTF(jni_env, outv_buffer);
   outv_current = outv_buffer;
-  if (str == NULL) {
+  jbyteArray arr = (*jni_env)->NewByteArray(jni_env, length);
+  if (arr == NULL) {
     jni_exception = 1;
-    return WHY("Exception thrown from NewStringUTF()");
+    return WHY("Exception thrown from NewByteArray()");
   }
-  (*jni_env)->CallBooleanMethod(jni_env, outv_list, listAddMethodId, str);
+  (*jni_env)->SetByteArrayRegion(jni_env, arr, 0, length, (jbyte*)outv_buffer);
+  if ((*jni_env)->ExceptionOccurred(jni_env)) {
+    jni_exception = 1;
+    return WHY("Exception thrown from SetByteArrayRegion()");
+  }
+  (*jni_env)->CallBooleanMethod(jni_env, outv_list, listAddMethodId, arr);
   if ((*jni_env)->ExceptionOccurred(jni_env)) {
     jni_exception = 1;
     return WHY("Exception thrown from CallBooleanMethod()");
   }
-  (*jni_env)->DeleteLocalRef(jni_env, str);
+  (*jni_env)->DeleteLocalRef(jni_env, arr);
   return 0;
 }
 
@@ -215,6 +219,32 @@ int cli_putchar(char c)
       return putchar(c);
 }
 
+/* Write a buffer of data to output.  If in a JNI call, then this appends the data to the
+   current output field, including any embedded nul characters.  Returns a non-negative integer on
+   success, EOF on error.
+ */
+int cli_write(const unsigned char *buf, size_t len)
+{
+#ifdef HAVE_JNI_H
+    if (jni_env) {
+      size_t avail = outv_limit - outv_current;
+      if (avail < len) {
+	memcpy(outv_current, buf, avail);
+	outv_current = outv_limit;
+	if (outv_growbuf(len) == -1)
+	  return EOF;
+	len -= avail;
+	buf += avail;
+      }
+      memcpy(outv_current, buf, len);
+      outv_current += len;
+      return 0;
+    }
+    else
+#endif
+      return fwrite(buf, len, 1, stdout);
+}
+
 /* Write a null-terminated string to output.  If in a JNI call, then this appends the string to the
    current output field.  The terminating null is not included.  Returns a non-negative integer on
    success, EOF on error.
@@ -222,21 +252,8 @@ int cli_putchar(char c)
 int cli_puts(const char *str)
 {
 #ifdef HAVE_JNI_H
-    if (jni_env) {
+    if (jni_env)
-      size_t len = strlen(str);
+      return cli_write((const unsigned char *) str, strlen(str));
-      size_t avail = outv_limit - outv_current;
-      if (avail < len) {
-	strncpy(outv_current, str, avail);
-	outv_current = outv_limit;
-	if (outv_growbuf(len) == -1)
-	  return EOF;
-	len -= avail;
-	str += avail;
-      }
-      strncpy(outv_current, str, len);
-      outv_current += len;
-      return 0;
-    }
     else
 #endif
       return fputs(str, stdout);
@@ -307,11 +324,21 @@ void cli_flush()
 int app_echo(int argc, const char *const *argv, struct command_line_option *o, void *context)
 {
   if (debug & DEBUG_VERBOSE) DEBUG_argv("command", argc, argv);
-  int i;
+  int i = 1;
-  for (i = 1; i < argc; ++i) {
+  int escapes = 0;
+  if (i < argc && strcmp(argv[i], "-e") == 0) {
+    escapes = 1;
+    ++i;
+  }
+  for (; i < argc; ++i) {
     if (debug & DEBUG_VERBOSE)
-      DEBUGF("echo:argv[%d]=%s", i, argv[i]);
+      DEBUGF("echo:argv[%d]=\"%s\"", i, argv[i]);
-    cli_puts(argv[i]);
+    if (escapes) {
+      unsigned char buf[strlen(argv[i])];
+      size_t len = str_fromprint(buf, argv[i]);
+      cli_write(buf, len);
+    } else
+      cli_puts(argv[i]);
     cli_delim(NULL);
   }
   return 0;
@@ -1251,11 +1278,16 @@ int app_rhizome_extract_manifest(int argc, const char *const *argv, struct comma
   switch (ret) {
     case 0: ret = 1; break;
     case 1: ret = 0;
-      if (manifestpath) {
+      if (manifestpath && strcmp(manifestpath, "-") == 0) {
+	cli_puts("manifest");
+	cli_delim(":");
+	cli_write(m->manifestdata, m->manifest_all_bytes);
+	cli_delim("\n");
+      } else if (manifestpath) {
 	/* If the manifest has been read in from database, the blob is there,
 	   and we can lie and say we are finalised and just want to write it
-	   out.  XXX really should have a dirty/clean flag, so that write
+	   out.  TODO: really should have a dirty/clean flag, so that write
-	   works is clean but not finalised. */
+	   works if clean but not finalised. */
 	m->finalised=1;
 	if (rhizome_write_manifest_file(m, manifestpath) == -1)
 	  ret = -1;

java/org/servalproject/servald/ServalD.java

@@ -26,18 +26,18 @@ import java.util.LinkedList;
 class ServalD
 {
 	int status;
-	List<String> outv;
+	List<byte[]> outv;
 
 	public ServalD()
 	{
 		System.loadLibrary("servald");
 	}
 
-	public native int rawCommand(List<String> outv, String... args);
+	public native int rawCommand(List<byte[]> outv, String... args);
 
 	public void command(String... args)
 	{
-		this.outv = new LinkedList<String>();
+		this.outv = new LinkedList<byte[]>();
 		this.status = this.rawCommand(this.outv, args);
 	}
 
@@ -45,8 +45,8 @@ class ServalD
 	{
 		ServalD servald = new ServalD();
 		servald.command(args);
-		for (String s: servald.outv) {
+		for (byte[] a: servald.outv) {
-			System.out.println(s);
+			System.out.println(new String(a));
 		}
 		System.exit(servald.status);
 	}

java/org/servalproject/servald/ServalDTests.java

@@ -11,7 +11,7 @@ class ServalDTests
 	public static void main(String[] args)
 	{
 		try {
-			Class cls = new Object() { }.getClass().getEnclosingClass();
+			Class<?> cls = new Object() { }.getClass().getEnclosingClass();
 			Method m = cls.getMethod(args[0], String[].class);
 			m.invoke(null, (Object) Arrays.copyOfRange(args, 1, args.length));
 		}
@@ -29,9 +29,9 @@ class ServalDTests
 		for (int i = 0; i != repeat; ++i) {
 			servald.command(Arrays.copyOfRange(args, 1, args.length));
 			System.out.print(servald.status);
-			for (String s: servald.outv) {
+			for (byte[] a: servald.outv) {
 				System.out.print(":");
-				System.out.print(s);
+				System.out.print(new String(a));
 			}
 			System.out.println("");
 		}

overlay_address.c

@@ -209,26 +209,27 @@ int set_reachable(struct subscriber *subscriber, int reachable){
   
   subscriber->reachable=reachable;
 
-  // these log messages may be used in tests
+  // These log messages are for use in tests.  Changing them may break test scripts.
-  switch(reachable){
+  if (debug&DEBUG_OVERLAYROUTING) {
-    case REACHABLE_NONE:
+    switch (reachable) {
-      DEBUGF("%s is not reachable", alloca_tohex_sid(subscriber->sid));
+      case REACHABLE_NONE:
-      break;
+	DEBUGF("NOT REACHABLE sid=%s", alloca_tohex_sid(subscriber->sid));
-    case REACHABLE_SELF:
+	break;
-      break;
+      case REACHABLE_SELF:
-      
+	break;
-    case REACHABLE_DIRECT:
+      case REACHABLE_DIRECT:
-      DEBUGF("%s is now reachable directly", alloca_tohex_sid(subscriber->sid));
+	DEBUGF("REACHABLE DIRECTLY sid=%s", alloca_tohex_sid(subscriber->sid));
-      break;
+	break;
-    case REACHABLE_INDIRECT:
+      case REACHABLE_INDIRECT:
-      DEBUGF("%s is now reachable indirectly", alloca_tohex_sid(subscriber->sid));
+	DEBUGF("REACHABLE INDIRECTLY sid=%s", alloca_tohex_sid(subscriber->sid));
-      break;
+	break;
-    case REACHABLE_UNICAST:
+      case REACHABLE_UNICAST:
-      DEBUGF("%s is now reachable via unicast", alloca_tohex_sid(subscriber->sid));
+	DEBUGF("REACHABLE VIA UNICAST sid=%s", alloca_tohex_sid(subscriber->sid));
-      break;
+	break;
-    case REACHABLE_BROADCAST:
+      case REACHABLE_BROADCAST:
-      DEBUGF("%s is now reachable via broadcast", alloca_tohex_sid(subscriber->sid));
+	DEBUGF("REACHABLE VIA BROADCAST sid=%s", alloca_tohex_sid(subscriber->sid));
-      break;
+	break;
+    }
   }
 
   /* Pre-emptively send a sas request */

rhizome_database.c

@@ -909,14 +909,14 @@ int rhizome_list_manifests(const char *service, const char *sender_sid, const ch
 	long long blob_filesize = rhizome_manifest_get_ll(m, "filesize");
 	int from_here = 0;
 	if (q_author) {
-	  DEBUGF("q_author=%s", alloca_str_toprint(q_author));
+	  if (debug & DEBUG_RHIZOME) DEBUGF("q_author=%s", alloca_str_toprint(q_author));
 	  unsigned char authorSid[SID_SIZE];
 	  stowSid(authorSid, 0, q_author);
 	  int cn = 0, in = 0, kp = 0;
 	  from_here = keyring_find_sid(keyring, &cn, &in, &kp, authorSid);
 	}
 	if (!from_here && blob_sender) {
-	  DEBUGF("blob_sender=%s", alloca_str_toprint(blob_sender));
+	  if (debug & DEBUG_RHIZOME) DEBUGF("blob_sender=%s", alloca_str_toprint(blob_sender));
 	  unsigned char senderSid[SID_SIZE];
 	  stowSid(senderSid, 0, blob_sender);
 	  int cn = 0, in = 0, kp = 0;

testframework.sh

@@ -719,14 +719,17 @@ assertStderrGrep() {
 }
 
 assertGrep() {
-   _tfw_getopts assertgrep "$@"
+   _tfw_getopts assertcontentgrep "$@"
    shift $_tfw_getopts_shift
    if [ $# -ne 2 ]; then
       _tfw_error "incorrect arguments"
       return $?
    fi
    _tfw_dump_on_fail "$1"
-   _tfw_assert_grep "$1" "$1" "$2" || _tfw_failexit
+   _tfw_get_content "$1" || return $?
+   local s=s
+   local message
+   _tfw_assert_grep "${_tfw_opt_line_msg:+$_tfw_opt_line_msg of }$1" "$_tfw_tmp/content" "$2" || _tfw_failexit
 }
 
 # Internal (private) functions that are not to be invoked directly from test
@@ -977,6 +980,8 @@ _tfw_getopts() {
    _tfw_opt_sleep=
    _tfw_opt_matches=
    _tfw_opt_line=
+   _tfw_opt_line_sed=
+   _tfw_opt_line_msg=
    _tfw_getopts_shift=0
    local oo
    _tfw_shopt oo -s extglob
@@ -985,6 +990,8 @@ _tfw_getopts() {
       *:--stdout) _tfw_dump_on_fail --stdout;;
       *:--stderr) _tfw_dump_on_fail --stderr;;
       assert*:--dump-on-fail=*) _tfw_dump_on_fail "${1#*=}";;
+      assert*:--error-on-fail) _tfw_opt_error_on_fail=true;;
+      assert*:--message=*) _tfw_message="${1#*=}";;
       execute:--exit-status=+([0-9])) _tfw_opt_exit_status="${1#*=}";;
       execute:--exit-status=*) _tfw_error "invalid value: $1";;
       execute*:--executable=) _tfw_error "missing value: $1";;
@@ -994,12 +1001,13 @@ _tfw_getopts() {
       wait_until:--timeout=*) _tfw_error "invalid value: $1";;
       wait_until:--sleep=@(+([0-9])?(.+([0-9]))|*([0-9]).+([0-9]))) _tfw_opt_sleep="${1#*=}";;
       wait_until:--sleep=*) _tfw_error "invalid value: $1";;
-      assert*:--error-on-fail) _tfw_opt_error_on_fail=true;;
+      assertcontentgrep:--matches=+([0-9])) _tfw_opt_matches="${1#*=}";;
-      assert*:--message=*) _tfw_message="${1#*=}";;
+      assertcontentgrep:--matches=*) _tfw_error "invalid value: $1";; 
-      assertgrep:--matches=+([0-9])) _tfw_opt_matches="${1#*=}";;
+      assertcontent*:--line=+([0-9])) _tfw_opt_line="${1#*=}"; _tfw_opt_line_msg="line $_tfw_opt_line";;
-      assertgrep:--matches=*) _tfw_error "invalid value: $1";; 
+      assertcontent*:--line=+([0-9])..) _tfw_opt_line="${1#*=}\$"; _tfw_opt_line_msg="lines $_tfw_opt_line";;
-      assertfilecontent:--line=+([0-9])) _tfw_opt_line="${1#*=}";;
+      assertcontent*:--line=..+([0-9])) _tfw_opt_line="1${1#*=}"; _tfw_opt_line_msg="lines $_tfw_opt_line";;
-      assertfilecontent:--line=*) _tfw_error "invalid value: $1";; 
+      assertcontent*:--line=+([0-9])..+([0-9])) _tfw_opt_line="${1#*=}"; _tfw_opt_line_msg="lines $_tfw_opt_line";;
+      assertcontent*:--line=*) _tfw_error "invalid value: $1";; 
       *:--) let _tfw_getopts_shift=_tfw_getopts_shift+1; shift; break;;
       *:--*) _tfw_error "unsupported option: $1";;
       *) break;;
@@ -1007,6 +1015,7 @@ _tfw_getopts() {
       let _tfw_getopts_shift=_tfw_getopts_shift+1
       shift
    done
+   [ -n "$_tfw_opt_line" ] && _tfw_opt_line_sed="${_tfw_opt_line/../,}"
    case "$context" in
    execute*)
       if [ -z "$_tfw_executable" ]; then
@@ -1074,20 +1083,24 @@ _tfw_assertExpr() {
    _tfw_assert eval "${_tfw_args[@]}"
 }
 
+_tfw_get_content() {
+   case "$_tfw_opt_line_sed" in
+   '') ln -f "$1" "$_tfw_tmp/content" || error "ln failed";;
+   *) $SED -n -e "${_tfw_opt_line_sed}p" "$1" >"$_tfw_tmp/content" || error "sed failed";;
+   esac
+}
+
 _tfw_assert_stdxxx_is() {
    local qual="$1"
    shift
-   _tfw_getopts assertfilecontent --$qual --stderr "$@"
+   _tfw_getopts assertcontentis --$qual --stderr "$@"
    shift $((_tfw_getopts_shift - 2))
    if [ $# -lt 1 ]; then
       _tfw_error "incorrect arguments"
       return $?
    fi
-   case "$_tfw_opt_line" in
+   _tfw_get_content "$_tfw_tmp/$qual" || return $?
-   '') ln -f "$_tfw_tmp/$qual" "$_tfw_tmp/content";;
+   local message="${_tfw_message:-${_tfw_opt_line_msg:+$_tfw_opt_line_msg of }$qual of ($executed) is $(shellarg "$@")}"
-   *) $SED -n -e "${_tfw_opt_line}p" "$_tfw_tmp/$qual" >"$_tfw_tmp/content";;
-   esac
-   local message="${_tfw_message:-${_tfw_opt_line:+line $_tfw_opt_line of }$qual of ($executed) is $(shellarg "$@")}"
    echo -n "$@" >$_tfw_tmp/stdxxx_is.tmp
    if ! cmp -s $_tfw_tmp/stdxxx_is.tmp "$_tfw_tmp/content"; then
       _tfw_failmsg "assertion failed: $message"
@@ -1101,7 +1114,7 @@ _tfw_assert_stdxxx_is() {
 _tfw_assert_stdxxx_linecount() {
    local qual="$1"
    shift
-   _tfw_getopts assertfilecontent --$qual --stderr "$@"
+   _tfw_getopts assert --$qual --stderr "$@"
    shift $((_tfw_getopts_shift - 2))
    if [ $# -lt 1 ]; then
       _tfw_error "incorrect arguments"
@@ -1117,13 +1130,14 @@ _tfw_assert_stdxxx_linecount() {
 _tfw_assert_stdxxx_grep() {
    local qual="$1"
    shift
-   _tfw_getopts assertgrep --$qual --stderr "$@"
+   _tfw_getopts assertcontentgrep --$qual --stderr "$@"
    shift $((_tfw_getopts_shift - 2))
    if [ $# -ne 1 ]; then
       _tfw_error "incorrect arguments"
       return $?
    fi
-   _tfw_assert_grep "$qual of ($executed)" $_tfw_tmp/$qual "$@"
+   _tfw_get_content "$_tfw_tmp/$qual" || return $?
+   _tfw_assert_grep "${_tfw_opt_line_msg:+$_tfw_opt_line_msg of }$qual of ($executed)" "$_tfw_tmp/content" "$@"
 }
 
 _tfw_assert_grep() {

tests/jni

@@ -24,6 +24,7 @@ source "${0%/*}/../testconfig.sh"
 
 setup() {
    setup_servald
+   executeOk_servald config set debug.verbose 1
    assert_echo_works
    compile_java_classes
    setup_servald_so
@@ -32,21 +33,21 @@ setup() {
 compile_java_classes() {
    assert --message='Java compiler was detected by ./configure' [ "$JAVAC" ]
    mkdir classes
-   assert $JAVAC -d classes "$servald_source_root"/java/org/servalproject/servald/*.java
+   assert $JAVAC -Xlint:unchecked -d classes "$servald_source_root"/java/org/servalproject/servald/*.java
    assert [ -r classes/org/servalproject/servald/ServalD.class ]
    assert [ -r classes/org/servalproject/servald/ServalDTests.class ]
 }
 
 # Make sure that the normal echo command-line works, without JNI.
 assert_echo_works() {
-   executeOk $servald echo 'Hello,' 'world!'
+   executeOk $servald echo -e 'Hello,\ttab' 'world\0!'
-   assertStdoutIs -e 'Hello,\nworld!\n'
+   assertStdoutIs -e 'Hello,\ttab\nworld\0!\n'
 }
 
 doc_Echo="Serval JNI echo Hello world"
 test_Echo() {
-   executeOk java -classpath "$PWD/classes" org.servalproject.servald.ServalD echo 'Hello,' 'world!'
+   executeOk java -classpath "$PWD/classes" org.servalproject.servald.ServalD echo -e 'Hello,\ttab' 'world\0!'
-   assertStdoutIs -e 'Hello,\nworld!\n'
+   assertStdoutIs -e 'Hello,\ttab\nworld\0!\n'
 }
 
 doc_Delim="Serval non-JNI output delimiter environment variable"

tests/rhizomeops

@@ -178,7 +178,6 @@ setup_ExtractManifestAfterAdd() {
 }
 test_ExtractManifestAfterAdd() {
    executeOk_servald rhizome extract manifest $manifestid file1x.manifest
-   assert cmp file1.manifest file1x.manifest
    assertStdoutLineCount '==' 8
    local size=$(( $(cat file1 | wc -c) + 0 ))
    assertStdoutGrep --matches=1 "^service:file$"
@@ -189,6 +188,38 @@ test_ExtractManifestAfterAdd() {
    assertStdoutGrep --matches=1 "^filesize:$size\$"
    assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
    assertStdoutGrep --matches=1 "^\.readonly:0\$"
+   assert cmp file1.manifest file1x.manifest
+}
+
+doc_ExtractManifestToStdout="Extract manifest to output field"
+setup_ExtractManifestToStdout() {
+   setup_servald
+   setup_rhizome
+   echo "A test file" >file1
+   executeOk_servald rhizome add file $SIDB1 '' file1 file1.manifest
+   extract_manifest_id manifestid file1.manifest
+   extract_manifest_version version file1.manifest
+   extract_manifest_filehash filehash file1.manifest
+}
+test_ExtractManifestToStdout() {
+   executeOk_servald rhizome extract manifest $manifestid -
+   assertStdoutLineCount '>=' 9
+   local size=$(( $(cat file1 | wc -c) + 0 ))
+   assertStdoutGrep --line=..8 --matches=1 "^service:file$"
+   assertStdoutGrep --line=..8 --matches=1 "^manifestid:$manifestid\$"
+   assertStdoutGrep --line=..8 --matches=1 "^version:$version\$"
+   assertStdoutGrep --line=..8 --matches=1 "^inserttime:$rexp_date\$"
+   assertStdoutGrep --line=..8 --matches=1 "^filehash:$filehash\$"
+   assertStdoutGrep --line=..8 --matches=1 "^filesize:$size\$"
+   assertStdoutGrep --line=..8 --matches=1 "^\.author:$SIDB1\$"
+   assertStdoutGrep --line=..8 --matches=1 "^\.readonly:0\$"
+   assertStdoutGrep --line=9 --matches=1 "^manifest:"
+   replayStdout | $SED -n '9s/^manifest://p' >file1x.manifest
+   replayStdout | $SED -n '10,$p' >>file1x.manifest
+   cat file1.manifest >file1n.manifest
+   echo >>file1n.manifest
+   tfw_cat file1n.manifest file1x.manifest
+   assert cmp file1n.manifest file1x.manifest
 }
 
 doc_ExtractManifestAfterAddNoAuthor="Extract manifest after one add with no author"