From 48f7cd7d5d638f0239bd3086c76a22395b5edf34 Mon Sep 17 00:00:00 2001 From: Andrew Bettison Date: Fri, 22 Feb 2013 15:28:41 +1030 Subject: [PATCH] Add serval_maps_push.sh script To be used as the --exec-on-unpack option to rhizome_mirrord Rsyncs all the unpacked zip files to the Serval Maps testing server and fetches the Serval Maps re-scan URL Is invoked with arg1 = path of unpack directory, args2..N = names of newly unpacked directories within arg1 If the RHIZOME_MIRRORD_LOG_STDOUT env var is set, then enables shell execution tracing --- utilities/rhizome_mirrord | 73 +++++++++++++++++++++++++---------- utilities/serval_maps_push.sh | 38 ++++++++++++++++++ 2 files changed, 90 insertions(+), 21 deletions(-) create mode 100755 utilities/serval_maps_push.sh diff --git a/utilities/rhizome_mirrord b/utilities/rhizome_mirrord index 644d479b..e1d62bbc 100755 --- a/utilities/rhizome_mirrord +++ b/utilities/rhizome_mirrord @@ -49,17 +49,39 @@ import tarfile def main(): instancepath = os.environ.get('SERVALINSTANCE_PATH') parser = argparse.ArgumentParser(description='Continuously extract Rhizome store into mirror directory.') - parser.add_argument('--mirror-dir', dest='mirror_dir', metavar='PATH', required=True, help='Path of directory to store extracted payloads') - parser.add_argument('--servald', dest='servald', metavar='PATH', required=True, help='Path of servald executable') - parser.add_argument('--instance', dest='instancepath', metavar='PATH', required=not instancepath, default=instancepath, help='Path of servald instance directory') - parser.add_argument('--interval', dest='interval', metavar='N', type=float, default=0, help='Sleep N seconds between polling Rhizome; N=0 means run once and exit') - parser.add_argument('--filter-name', dest='name_filter', metavar='GLOB', help='Only mirror bundles whose names match GLOB pattern') - parser.add_argument('--unpack-dir', dest='unpack_dir', metavar='PATH', default=None, help='Path of directory in which to unpack bundles (zip, tar, etc.)') - parser.add_argument('--exec-on-unpack', dest='exec_on_unpack', metavar='EXECUTABLE', help='Run EXECUTABLE after unpacking one or more bundles') - parser.add_argument('--expire-delay', dest='expire_delay', metavar='N', type=int, default=0, help='Keep bundles in mirror for N seconds after no longer listed by Rhizome') - parser.add_argument('--error-retry', dest='error_retry', metavar='N', type=int, default=600, help='Wait N seconds before retrying failed operations') - parser.add_argument('--paranoid', dest='paranoid', action='store_true', help='Continually check for and correct corrupted mirror contents') - parser.add_argument('--log-to-stdout', dest='log_to_stdout', action='store_true', help='Log activity on standard output') + parser.add_argument('--mirror-dir', + dest='mirror_dir', metavar='PATH', required=True, + help='Path of directory to store extracted payloads') + parser.add_argument('--servald', + dest='servald', metavar='PATH', required=True, + help='Path of servald executable') + parser.add_argument('--instance', + dest='instancepath', metavar='PATH', required=not instancepath, default=instancepath, + help='Path of servald instance directory') + parser.add_argument('--interval', + dest='interval', metavar='N', type=float, default=0, + help='Sleep N seconds between polling Rhizome; N=0 means run once and exit') + parser.add_argument('--filter-name', + dest='name_filter', metavar='GLOB', + help='Only mirror bundles whose names match GLOB pattern') + parser.add_argument('--unpack-dir', + dest='unpack_dir', metavar='PATH', default=None, + help='Path of directory in which to unpack bundles (zip, tar, etc.)') + parser.add_argument('--exec-on-unpack', + dest='exec_on_unpack', metavar='EXECUTABLE', + help='Run EXECUTABLE after unpacking one or more bundles, arg1 = PATH (from --unpack-dir=PATH), arg2..n = names of unpacked directories in PATH') + parser.add_argument('--expire-delay', + dest='expire_delay', metavar='N', type=int, default=0, + help='Keep bundles in mirror for N seconds after no longer listed by Rhizome') + parser.add_argument('--error-retry', + dest='error_retry', metavar='N', type=int, default=600, + help='Wait N seconds before retrying failed operations') + parser.add_argument('--paranoid', + dest='paranoid', action='store_true', + help='Continually check for and correct corrupted mirror contents') + parser.add_argument('--log-to-stdout', + dest='log_to_stdout', action='store_true', + help='Log activity on standard output') opts = parser.parse_args() global log_output log_output = sys.stderr if opts.log_to_stdout else None @@ -78,15 +100,7 @@ def main(): mirror.list() mirror.update() mirror.expire() - if mirror.unpacked: - if opts.exec_on_unpack: - args = [opts.exec_on_unpack] + sorted(mirror.unpacked) - log("execute " + ' '.join(args)) - try: - subprocess.call(args, stdin=open('/dev/null')) - mirror.unpacked.clear() - except OSError, e: - error('cannot execute %r - %s' % (opts.exec_on_unpack, e)) + mirror.exec_on_unpack() if not opts.interval: break time.sleep(opts.interval) @@ -324,6 +338,23 @@ class RhizomeMirror(object): else: self.unlink(payload_path) + def exec_on_unpack(self): + if self.unpacked and self.opts.exec_on_unpack: + env = dict(os.environ) + global log_output + if log_output: + env['RHIZOME_MIRRORD_LOG_STDOUT'] = 'true' + args = [self.opts.exec_on_unpack, self.unpacks_path] + [os.path.relpath(p, self.unpacks_path) for p in sorted(self.unpacked)] + log("execute " + ' '.join(args)) + try: + stat = subprocess.call(args, stdin= open('/dev/null'), env= env) + if stat == 0: + self.unpacked.clear() + else: + error('exec-on-unpack script %r failed - error status %d' % (self.opts.exec_on_unpack, stat)) + except OSError, e: + error('cannot execute %r - %s' % (self.opts.exec_on_unpack, e)) + def mkdirs(self, path): if os.path.isdir(path): return False @@ -591,7 +622,7 @@ def invoke_servald(opts, args, output_keyvalue=False, output_words=False): if output_words or output_keyvalue: delim = '\x01' env['SERVALD_OUTPUT_DELIMITER'] = delim - env['SERVALD_INSTANCEPATH'] = opts.instancepath + env['SERVALINSTANCE_PATH'] = opts.instancepath try: allargs = (opts.servald,) + tuple(args) log('execute ' + ' '.join(map(repr, allargs))) diff --git a/utilities/serval_maps_push.sh b/utilities/serval_maps_push.sh new file mode 100755 index 00000000..b9a04b76 --- /dev/null +++ b/utilities/serval_maps_push.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +# Rhizome Maps push script +# Copyright (C) 2013 Serval Project Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +set -e + +UNPACK_DIR="${1?}" +shift + +cd "${TMPDIR:-/tmp}" >/dev/null + +if [ -n "$RHIZOME_MIRRORD_LOG_STDOUT" ]; then + set -x +fi + +rsync -a \ + "$UNPACK_DIR/" \ + servalp@servalproject.org:/home/servalp/maps.servalproject.org/testing/admin/data/instances/ + +curl -s \ + -o serval_maps_push_result.html \ + -D serval_maps_push_headers.txt \ + http://maps.servalproject.org/testing/admin/cache-update/instances/2798a6651e9caecd3d30fdc5e6a0e0f5