mirror of
https://github.com/tahoe-lafs/tahoe-lafs.git
synced 2025-02-20 17:52:50 +00:00
zfec: use setuptools to construct executables, fix bug on Windows -- forgot "BINARY" flag to output files
This commit is contained in:
Zooko O'Whielacronx
parent
69f2ecc768
commit
4d0a984dd8
@ -1,85 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# zfec -- a fast C implementation of Reed-Solomon erasure coding with
|
||||
# command-line, C, and Python interfaces
|
||||
|
||||
# import bindann
|
||||
# import bindann.monkeypatch.all
|
||||
|
||||
import sys
|
||||
|
||||
import zfec
|
||||
from zfec.util import argparse
|
||||
from zfec import filefec
|
||||
from zfec.util.version import Version
|
||||
__version__ = Version("1.0.0a1-0-STABLE")
|
||||
|
||||
if '-V' in sys.argv or '--version' in sys.argv:
|
||||
print "zfec library version: ", zfec.__version__
|
||||
print "zfec command-line tool version: ", __version__
|
||||
sys.exit(0)
|
||||
|
||||
parser = argparse.ArgumentParser(description="Encode a file into a set of share files, a subset of which can later be used to recover the original file.")
|
||||
|
||||
parser.add_argument('inputfile', help='file to encode or "-" for stdin', type=argparse.FileType('rb'), metavar='INF')
|
||||
parser.add_argument('-d', '--output-dir', help='directory in which share file names will be created (default ".")', default='.', metavar='D')
|
||||
parser.add_argument('-p', '--prefix', help='prefix for share file names; If omitted, the name of the input file will be used.', metavar='P')
|
||||
parser.add_argument('-s', '--suffix', help='suffix for share file names (default ".fec")', default='.fec', metavar='S')
|
||||
parser.add_argument('-m', '--totalshares', help='the total number of share files created (default 16)', default=16, type=int, metavar='M')
|
||||
parser.add_argument('-k', '--requiredshares', help='the number of share files required to reconstruct (default 4)', default=4, type=int, metavar='K')
|
||||
parser.add_argument('-f', '--force', help='overwrite any file which already in place an output file (share file)', action='store_true')
|
||||
parser.add_argument('-v', '--verbose', help='print out messages about progress', action='store_true')
|
||||
parser.add_argument('-V', '--version', help='print out version number and exit', action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.prefix is None:
|
||||
args.prefix = args.inputfile.name
|
||||
if args.prefix == "<stdin>":
|
||||
args.prefix = ""
|
||||
|
||||
if args.totalshares < 3:
|
||||
print "Invalid parameters, totalshares is required to be >= 3\nPlease see the accompanying documentation."
|
||||
sys.exit(1)
|
||||
if args.totalshares > 256:
|
||||
print "Invalid parameters, totalshares is required to be <= 256\nPlease see the accompanying documentation."
|
||||
sys.exit(1)
|
||||
if args.requiredshares < 2:
|
||||
print "Invalid parameters, requiredshares is required to be >= 2\nPlease see the accompanying documentation."
|
||||
sys.exit(1)
|
||||
if args.requiredshares >= args.totalshares:
|
||||
print "Invalid parameters, requiredshares is required to be < totalshares\nPlease see the accompanying documentation."
|
||||
sys.exit(1)
|
||||
|
||||
args.inputfile.seek(0, 2)
|
||||
fsize = args.inputfile.tell()
|
||||
args.inputfile.seek(0, 0)
|
||||
ret = filefec.encode_to_files(args.inputfile, fsize, args.output_dir, args.prefix, args.requiredshares, args.totalshares, args.suffix, args.force, args.verbose)
|
||||
|
||||
sys.exit(ret)
|
||||
|
||||
# zfec -- a fast C implementation of Reed-Solomon erasure coding with
|
||||
# command-line, C, and Python interfaces
|
||||
#
|
||||
# Copyright (C) 2007 Allmydata, Inc.
|
||||
# Author: Zooko Wilcox-O'Hearn
|
||||
# mailto:zooko@zooko.com
|
||||
#
|
||||
# This file is part of zfec.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify it
|
||||
# under the terms of the GNU General Public License as published by the Free
|
||||
# Software Foundation; either version 2 of the License, or (at your option)
|
||||
# any later version. This package also comes with the added permission that,
|
||||
# in the case that you are obligated to release a derived work under this
|
||||
# licence (as per section 2.b of the GPL), you may delay the fulfillment of
|
||||
# this obligation for up to 12 months.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
|
||||
@ -1,79 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# zfec -- a fast C implementation of Reed-Solomon erasure coding with
|
||||
# command-line, C, and Python interfaces
|
||||
|
||||
# import bindann
|
||||
# import bindann.monkeypatch.all
|
||||
|
||||
import os, sys
|
||||
|
||||
from zfec.util import argparse
|
||||
|
||||
import zfec
|
||||
from zfec import filefec
|
||||
from zfec.util.version import Version
|
||||
__version__ = Version("1.0.0a1-0-STABLE")
|
||||
|
||||
if '-V' in sys.argv or '--version' in sys.argv:
|
||||
print "zfec library version: ", zfec.__version__
|
||||
print "zunfec command-line tool version: ", __version__
|
||||
sys.exit(0)
|
||||
|
||||
parser = argparse.ArgumentParser(description="Decode data from share files.")
|
||||
|
||||
parser.add_argument('-o', '--outputfile', required=True, help='file to write the resulting data to, or "-" for stdout', type=str, metavar='OUTF')
|
||||
parser.add_argument('sharefiles', nargs='*', help='shares file to read the encoded data from', type=argparse.FileType('rb'), metavar='SHAREFILE')
|
||||
parser.add_argument('-v', '--verbose', help='print out messages about progress', action='store_true')
|
||||
parser.add_argument('-f', '--force', help='overwrite any file which already in place of the output file', action='store_true')
|
||||
parser.add_argument('-V', '--version', help='print out version number and exit', action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
if len(args.sharefiles) < 2:
|
||||
print "At least two sharefiles are required."
|
||||
sys.exit(1)
|
||||
|
||||
if args.force:
|
||||
outf = open(args.outputfile, 'wb')
|
||||
else:
|
||||
try:
|
||||
outfd = os.open(args.outputfile, os.O_WRONLY|os.O_CREAT|os.O_EXCL)
|
||||
except OSError:
|
||||
print "There is already a file named %r -- aborting. Use --force to overwrite." % (args.outputfile,)
|
||||
sys.exit(2)
|
||||
outf = os.fdopen(outfd, "wb")
|
||||
|
||||
try:
|
||||
ret = filefec.decode_from_files(outf, args.sharefiles, args.verbose)
|
||||
except filefec.InsufficientShareFilesError, e:
|
||||
print str(e)
|
||||
sys.exit(3)
|
||||
|
||||
sys.exit(ret)
|
||||
|
||||
# zfec -- a fast C implementation of Reed-Solomon erasure coding with
|
||||
# command-line, C, and Python interfaces
|
||||
#
|
||||
# Copyright (C) 2007 Allmydata, Inc.
|
||||
# Author: Zooko Wilcox-O'Hearn
|
||||
# mailto:zooko@zooko.com
|
||||
#
|
||||
# This file is part of zfec.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify it
|
||||
# under the terms of the GNU General Public License as published by the Free
|
||||
# Software Foundation; either version 2 of the License, or (at your option)
|
||||
# any later version. This package also comes with the added permission that,
|
||||
# in the case that you are obligated to release a derived work under this
|
||||
# licence (as per section 2.b of the GPL), you may delay the fulfillment of
|
||||
# this obligation for up to 12 months.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
|
||||
@ -66,8 +66,8 @@ setup(name='zfec',
|
||||
url='http://allmydata.com/source/zfec',
|
||||
license='GNU GPL',
|
||||
platform='Any',
|
||||
packages=['zfec', 'zfec.util', 'zfec.test'],
|
||||
packages=['zfec', 'zfec.cmdline', 'zfec.util', 'zfec.test'],
|
||||
classifiers=trove_classifiers,
|
||||
scripts=['cmdline/zfec', 'cmdline/zunfec',],
|
||||
entry_points = { 'console_scripts': [ 'zfec = zfec.cmdline.zfec:main', 'zunfec = zfec.cmdline.zunfec:main' ] },
|
||||
ext_modules=[Extension('_fec', ['zfec/fec.c', 'zfec/_fecmodule.c',], extra_link_args=extra_link_args, extra_compile_args=extra_compile_args, undef_macros=undef_macros),],
|
||||
)
|
||||
|
||||
0
zfec/zfec/cmdline/__init__.py
Normal file
0
zfec/zfec/cmdline/__init__.py
Normal file
86
zfec/zfec/cmdline/zfec.py
Normal file
86
zfec/zfec/cmdline/zfec.py
Normal file
@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# zfec -- a fast C implementation of Reed-Solomon erasure coding with
|
||||
# command-line, C, and Python interfaces
|
||||
|
||||
# The zfec and zunfec command-line tools require Python 2.5 for relative imports.
|
||||
|
||||
import sys
|
||||
|
||||
from ..util import argparse
|
||||
from .. import filefec
|
||||
|
||||
from ..zfec import __version__ as libversion
|
||||
from ..util.version import Version
|
||||
__version__ = Version("1.0.0a1-0-STABLE")
|
||||
|
||||
def main():
|
||||
|
||||
if '-V' in sys.argv or '--version' in sys.argv:
|
||||
print "zfec library version: ", libversion
|
||||
print "zfec command-line tool version: ", __version__
|
||||
sys.exit(0)
|
||||
|
||||
parser = argparse.ArgumentParser(description="Encode a file into a set of share files, a subset of which can later be used to recover the original file.")
|
||||
|
||||
parser.add_argument('inputfile', help='file to encode or "-" for stdin', type=argparse.FileType('rb'), metavar='INF')
|
||||
parser.add_argument('-d', '--output-dir', help='directory in which share file names will be created (default ".")', default='.', metavar='D')
|
||||
parser.add_argument('-p', '--prefix', help='prefix for share file names; If omitted, the name of the input file will be used.', metavar='P')
|
||||
parser.add_argument('-s', '--suffix', help='suffix for share file names (default ".fec")', default='.fec', metavar='S')
|
||||
parser.add_argument('-m', '--totalshares', help='the total number of share files created (default 16)', default=16, type=int, metavar='M')
|
||||
parser.add_argument('-k', '--requiredshares', help='the number of share files required to reconstruct (default 4)', default=4, type=int, metavar='K')
|
||||
parser.add_argument('-f', '--force', help='overwrite any file which already in place an output file (share file)', action='store_true')
|
||||
parser.add_argument('-v', '--verbose', help='print out messages about progress', action='store_true')
|
||||
parser.add_argument('-V', '--version', help='print out version number and exit', action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.prefix is None:
|
||||
args.prefix = args.inputfile.name
|
||||
if args.prefix == "<stdin>":
|
||||
args.prefix = ""
|
||||
|
||||
if args.totalshares < 3:
|
||||
print "Invalid parameters, totalshares is required to be >= 3\nPlease see the accompanying documentation."
|
||||
sys.exit(1)
|
||||
if args.totalshares > 256:
|
||||
print "Invalid parameters, totalshares is required to be <= 256\nPlease see the accompanying documentation."
|
||||
sys.exit(1)
|
||||
if args.requiredshares < 2:
|
||||
print "Invalid parameters, requiredshares is required to be >= 2\nPlease see the accompanying documentation."
|
||||
sys.exit(1)
|
||||
if args.requiredshares >= args.totalshares:
|
||||
print "Invalid parameters, requiredshares is required to be < totalshares\nPlease see the accompanying documentation."
|
||||
sys.exit(1)
|
||||
|
||||
args.inputfile.seek(0, 2)
|
||||
fsize = args.inputfile.tell()
|
||||
args.inputfile.seek(0, 0)
|
||||
return filefec.encode_to_files(args.inputfile, fsize, args.output_dir, args.prefix, args.requiredshares, args.totalshares, args.suffix, args.force, args.verbose)
|
||||
|
||||
|
||||
# zfec -- a fast C implementation of Reed-Solomon erasure coding with
|
||||
# command-line, C, and Python interfaces
|
||||
#
|
||||
# Copyright (C) 2007 Allmydata, Inc.
|
||||
# Author: Zooko Wilcox-O'Hearn
|
||||
# mailto:zooko@zooko.com
|
||||
#
|
||||
# This file is part of zfec.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify it
|
||||
# under the terms of the GNU General Public License as published by the Free
|
||||
# Software Foundation; either version 2 of the License, or (at your option)
|
||||
# any later version. This package also comes with the added permission that,
|
||||
# in the case that you are obligated to release a derived work under this
|
||||
# licence (as per section 2.b of the GPL), you may delay the fulfillment of
|
||||
# this obligation for up to 12 months.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
|
||||
80
zfec/zfec/cmdline/zunfec.py
Normal file
80
zfec/zfec/cmdline/zunfec.py
Normal file
@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# zfec -- a fast C implementation of Reed-Solomon erasure coding with
|
||||
# command-line, C, and Python interfaces
|
||||
|
||||
# The zfec and zunfec command-line tools require Python 2.5 for relative imports.
|
||||
|
||||
import os, sys
|
||||
|
||||
from ..util import argparse
|
||||
from .. import filefec
|
||||
|
||||
from ..zfec import __version__ as libversion
|
||||
from ..util.version import Version
|
||||
__version__ = Version("1.0.0a1-0-STABLE")
|
||||
|
||||
def main():
|
||||
if '-V' in sys.argv or '--version' in sys.argv:
|
||||
print "zfec library version: ", libversion
|
||||
print "zunfec command-line tool version: ", __version__
|
||||
return 0
|
||||
|
||||
parser = argparse.ArgumentParser(description="Decode data from share files.")
|
||||
|
||||
parser.add_argument('-o', '--outputfile', required=True, help='file to write the resulting data to, or "-" for stdout', type=str, metavar='OUTF')
|
||||
parser.add_argument('sharefiles', nargs='*', help='shares file to read the encoded data from', type=argparse.FileType('rb'), metavar='SHAREFILE')
|
||||
parser.add_argument('-v', '--verbose', help='print out messages about progress', action='store_true')
|
||||
parser.add_argument('-f', '--force', help='overwrite any file which already in place of the output file', action='store_true')
|
||||
parser.add_argument('-V', '--version', help='print out version number and exit', action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
if len(args.sharefiles) < 2:
|
||||
print "At least two sharefiles are required."
|
||||
return 1
|
||||
|
||||
if args.force:
|
||||
outf = open(args.outputfile, 'wb')
|
||||
else:
|
||||
try:
|
||||
flags = os.O_WRONLY|os.O_CREAT|os.O_EXCL | (hasattr(os, 'O_BINARY') and os.O_BINARY)
|
||||
outfd = os.open(args.outputfile, flags)
|
||||
except OSError:
|
||||
print "There is already a file named %r -- aborting. Use --force to overwrite." % (args.outputfile,)
|
||||
return 2
|
||||
outf = os.fdopen(outfd, "wb")
|
||||
|
||||
try:
|
||||
ret = filefec.decode_from_files(outf, args.sharefiles, args.verbose)
|
||||
except filefec.InsufficientShareFilesError, e:
|
||||
print str(e)
|
||||
return 3
|
||||
|
||||
return 0
|
||||
|
||||
# zfec -- a fast C implementation of Reed-Solomon erasure coding with
|
||||
# command-line, C, and Python interfaces
|
||||
#
|
||||
# Copyright (C) 2007 Allmydata, Inc.
|
||||
# Author: Zooko Wilcox-O'Hearn
|
||||
# mailto:zooko@zooko.com
|
||||
#
|
||||
# This file is part of zfec.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify it
|
||||
# under the terms of the GNU General Public License as published by the Free
|
||||
# Software Foundation; either version 2 of the License, or (at your option)
|
||||
# any later version. This package also comes with the added permission that,
|
||||
# in the case that you are obligated to release a derived work under this
|
||||
# licence (as per section 2.b of the GPL), you may delay the fulfillment of
|
||||
# this obligation for up to 12 months.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
|
||||
@ -174,7 +174,7 @@ def encode_to_files(inf, fsize, dirname, prefix, k, m, suffix=".fec", overwrite=
|
||||
if overwrite:
|
||||
f = open(fn, "wb")
|
||||
else:
|
||||
fd = os.open(fn, os.O_WRONLY|os.O_CREAT|os.O_EXCL)
|
||||
fd = os.open(fn, os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_BINARY)
|
||||
f = os.fdopen(fd, "wb")
|
||||
f.write(hdr)
|
||||
fs.append(f)
|
||||
|
||||
@ -164,6 +164,21 @@ class FileFec(unittest.TestCase):
|
||||
def test_filefec_all_shares(self):
|
||||
return self._help_test_filefec("Yellow Whirled!", 3, 8)
|
||||
|
||||
def test_filefec_all_shares_2(self):
|
||||
return self._help_test_filefec("Yellow Whirled", 3, 8)
|
||||
|
||||
def test_filefec_all_shares_3(self):
|
||||
return self._help_test_filefec("Yellow Whirle", 3, 8)
|
||||
|
||||
def test_filefec_all_shares_3_b(self):
|
||||
return self._help_test_filefec("Yellow Whirle", 4, 16)
|
||||
|
||||
def test_filefec_all_shares_2_b(self):
|
||||
return self._help_test_filefec("Yellow Whirled", 4, 16)
|
||||
|
||||
def test_filefec_all_shares_1_b(self):
|
||||
return self._help_test_filefec("Yellow Whirled!", 4, 16)
|
||||
|
||||
def test_filefec_all_shares_with_padding(self, noisy=VERBOSE):
|
||||
return self._help_test_filefec("Yellow Whirled!A", 3, 8)
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user