Add serval_maps_push.sh script

To be used as the --exec-on-unpack option to rhizome_mirrord

Rsyncs all the unpacked zip files to the Serval Maps testing server and fetches
the Serval Maps re-scan URL

Is invoked with arg1 = path of unpack directory, args2..N = names of newly
unpacked directories within arg1

If the RHIZOME_MIRRORD_LOG_STDOUT env var is set, then enables shell execution
tracing

utilities/rhizome_mirrord

@@ -49,17 +49,39 @@ import tarfile
 def main():
     instancepath = os.environ.get('SERVALINSTANCE_PATH')
     parser = argparse.ArgumentParser(description='Continuously extract Rhizome store into mirror directory.')
-    parser.add_argument('--mirror-dir', dest='mirror_dir', metavar='PATH', required=True, help='Path of directory to store extracted payloads')
-    parser.add_argument('--servald', dest='servald', metavar='PATH', required=True, help='Path of servald executable')
-    parser.add_argument('--instance', dest='instancepath', metavar='PATH', required=not instancepath, default=instancepath, help='Path of servald instance directory')
-    parser.add_argument('--interval', dest='interval', metavar='N', type=float, default=0, help='Sleep N seconds between polling Rhizome; N=0 means run once and exit')
-    parser.add_argument('--filter-name', dest='name_filter', metavar='GLOB', help='Only mirror bundles whose names match GLOB pattern')
-    parser.add_argument('--unpack-dir', dest='unpack_dir', metavar='PATH', default=None, help='Path of directory in which to unpack bundles (zip, tar, etc.)')
-    parser.add_argument('--exec-on-unpack', dest='exec_on_unpack', metavar='EXECUTABLE', help='Run EXECUTABLE after unpacking one or more bundles')
-    parser.add_argument('--expire-delay', dest='expire_delay', metavar='N', type=int, default=0, help='Keep bundles in mirror for N seconds after no longer listed by Rhizome')
-    parser.add_argument('--error-retry', dest='error_retry', metavar='N', type=int, default=600, help='Wait N seconds before retrying failed operations')
-    parser.add_argument('--paranoid', dest='paranoid', action='store_true', help='Continually check for and correct corrupted mirror contents')
-    parser.add_argument('--log-to-stdout', dest='log_to_stdout', action='store_true', help='Log activity on standard output')
+    parser.add_argument('--mirror-dir',
+        dest='mirror_dir', metavar='PATH', required=True,
+        help='Path of directory to store extracted payloads')
+    parser.add_argument('--servald',
+        dest='servald', metavar='PATH', required=True,
+        help='Path of servald executable')
+    parser.add_argument('--instance',
+        dest='instancepath', metavar='PATH', required=not instancepath, default=instancepath,
+        help='Path of servald instance directory')
+    parser.add_argument('--interval',
+        dest='interval', metavar='N', type=float, default=0,
+        help='Sleep N seconds between polling Rhizome; N=0 means run once and exit')
+    parser.add_argument('--filter-name',
+        dest='name_filter', metavar='GLOB',
+        help='Only mirror bundles whose names match GLOB pattern')
+    parser.add_argument('--unpack-dir',
+        dest='unpack_dir', metavar='PATH', default=None,
+        help='Path of directory in which to unpack bundles (zip, tar, etc.)')
+    parser.add_argument('--exec-on-unpack',
+        dest='exec_on_unpack', metavar='EXECUTABLE',
+        help='Run EXECUTABLE after unpacking one or more bundles, arg1 = PATH (from --unpack-dir=PATH), arg2..n = names of unpacked directories in PATH')
+    parser.add_argument('--expire-delay',
+        dest='expire_delay', metavar='N', type=int, default=0,
+        help='Keep bundles in mirror for N seconds after no longer listed by Rhizome')
+    parser.add_argument('--error-retry',
+        dest='error_retry', metavar='N', type=int, default=600,
+        help='Wait N seconds before retrying failed operations')
+    parser.add_argument('--paranoid',
+        dest='paranoid', action='store_true',
+        help='Continually check for and correct corrupted mirror contents')
+    parser.add_argument('--log-to-stdout',
+        dest='log_to_stdout', action='store_true',
+        help='Log activity on standard output')
     opts = parser.parse_args()
     global log_output
     log_output = sys.stderr if opts.log_to_stdout else None
@@ -78,15 +100,7 @@ def main():
         mirror.list()
         mirror.update()
         mirror.expire()
-        if mirror.unpacked:
-            if opts.exec_on_unpack:
-                args = [opts.exec_on_unpack] + sorted(mirror.unpacked)
-                log("execute " + ' '.join(args))
-                try:
-                    subprocess.call(args, stdin=open('/dev/null'))
-                    mirror.unpacked.clear()
-                except OSError, e:
-                    error('cannot execute %r - %s' % (opts.exec_on_unpack, e))
+        mirror.exec_on_unpack()
         if not opts.interval:
             break
         time.sleep(opts.interval)
@@ -324,6 +338,23 @@ class RhizomeMirror(object):
                     else:
                         self.unlink(payload_path)
 
+    def exec_on_unpack(self):
+        if self.unpacked and self.opts.exec_on_unpack:
+            env = dict(os.environ)
+            global log_output
+            if log_output:
+                env['RHIZOME_MIRRORD_LOG_STDOUT'] = 'true'
+            args = [self.opts.exec_on_unpack, self.unpacks_path] + [os.path.relpath(p, self.unpacks_path) for p in sorted(self.unpacked)]
+            log("execute " + ' '.join(args))
+            try:
+                stat = subprocess.call(args, stdin= open('/dev/null'), env= env)
+                if stat == 0:
+                    self.unpacked.clear()
+                else:
+                    error('exec-on-unpack script %r failed - error status %d' % (self.opts.exec_on_unpack, stat))
+            except OSError, e:
+                error('cannot execute %r - %s' % (self.opts.exec_on_unpack, e))
+
     def mkdirs(self, path):
         if os.path.isdir(path):
             return False
@@ -591,7 +622,7 @@ def invoke_servald(opts, args, output_keyvalue=False, output_words=False):
     if output_words or output_keyvalue:
         delim = '\x01'
         env['SERVALD_OUTPUT_DELIMITER'] = delim
-    env['SERVALD_INSTANCEPATH'] = opts.instancepath
+    env['SERVALINSTANCE_PATH'] = opts.instancepath
     try:
         allargs = (opts.servald,) + tuple(args)
         log('execute ' + ' '.join(map(repr, allargs)))

utilities/serval_maps_push.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+
+# Rhizome Maps push script
+# Copyright (C) 2013 Serval Project Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+set -e
+
+UNPACK_DIR="${1?}"
+shift
+
+cd "${TMPDIR:-/tmp}" >/dev/null
+
+if [ -n "$RHIZOME_MIRRORD_LOG_STDOUT" ]; then
+   set -x
+fi
+
+rsync -a \
+   "$UNPACK_DIR/" \
+   servalp@servalproject.org:/home/servalp/maps.servalproject.org/testing/admin/data/instances/
+
+curl -s \
+   -o serval_maps_push_result.html \
+   -D serval_maps_push_headers.txt \
+   http://maps.servalproject.org/testing/admin/cache-update/instances/2798a6651e9caecd3d30fdc5e6a0e0f5