clean up uri-vs-cap terminology, emphasize cap instances instead of URI strings

* "cap" means a python instance which encapsulates a filecap/dircap (uri.py)
 * "uri" means a string with a "URI:" prefix
 * FileNode instances are created with (and retain) a cap instance, and
   generate uri strings on demand
 * .get_cap/get_readcap/get_verifycap/get_repaircap return cap instances
 * .get_uri/get_readonly_uri return uri strings

* add filenode.download_to_filename() for control.py, should find a better way
* use MutableFileNode.init_from_cap, not .init_from_uri
* directory URI instances: use get_filenode_cap, not get_filenode_uri
* update/cleanup bench_dirnode.py to match, add Makefile target to run it
This commit is contained in:
Brian Warner 2009-11-11 14:25:42 -08:00
parent f47672d12a
commit 131e05b155
16 changed files with 214 additions and 176 deletions

View File

@ -223,6 +223,9 @@ check-grid: .built
if [ -z '$(TESTCLIENTDIR)' ]; then exit 1; fi if [ -z '$(TESTCLIENTDIR)' ]; then exit 1; fi
$(PYTHON) src/allmydata/test/check_grid.py $(TESTCLIENTDIR) bin/tahoe $(PYTHON) src/allmydata/test/check_grid.py $(TESTCLIENTDIR) bin/tahoe
bench-dirnode: .built
$(RUNPP) -p -c src/allmydata/test/bench_dirnode.py
# 'make repl' is a simple-to-type command to get a Python interpreter loop # 'make repl' is a simple-to-type command to get a Python interpreter loop
# from which you can type 'import allmydata' # from which you can type 'import allmydata'
repl: repl:

View File

@ -50,7 +50,8 @@ class ControlServer(Referenceable, service.Service):
return d return d
def remote_download_from_uri_to_file(self, uri, filename): def remote_download_from_uri_to_file(self, uri, filename):
d = self.parent.downloader.download_to_filename(uri, filename) filenode = self.parent.create_node_from_uri(uri)
d = filenode.download_to_filename(filename)
d.addCallback(lambda res: filename) d.addCallback(lambda res: filename)
return d return d

View File

@ -271,12 +271,15 @@ class DirectoryNode:
def get_readonly_uri(self): def get_readonly_uri(self):
return self._uri.get_readonly().to_string() return self._uri.get_readonly().to_string()
def get_cap(self):
return self._uri
def get_readcap(self):
return self._uri.get_readonly()
def get_verify_cap(self): def get_verify_cap(self):
return self._uri.get_verify_cap() return self._uri.get_verify_cap()
def get_repair_cap(self): def get_repair_cap(self):
if self._node.is_readonly(): if self._node.is_readonly():
return None return None # readonly (mutable) dirnodes are not yet repairable
return self._uri return self._uri
def get_storage_index(self): def get_storage_index(self):

View File

@ -7,8 +7,7 @@ from foolscap.api import DeadReferenceError, RemoteException, eventually
from allmydata.util import base32, deferredutil, hashutil, log, mathutil, idlib from allmydata.util import base32, deferredutil, hashutil, log, mathutil, idlib
from allmydata.util.assertutil import _assert, precondition from allmydata.util.assertutil import _assert, precondition
from allmydata import codec, hashtree, uri from allmydata import codec, hashtree, uri
from allmydata.interfaces import IDownloadTarget, IDownloader, \ from allmydata.interfaces import IDownloadTarget, IDownloader, IVerifierURI, \
IFileURI, IVerifierURI, \
IDownloadStatus, IDownloadResults, IValidatedThingProxy, \ IDownloadStatus, IDownloadResults, IValidatedThingProxy, \
IStorageBroker, NotEnoughSharesError, NoSharesError, NoServersError, \ IStorageBroker, NotEnoughSharesError, NoSharesError, NoServersError, \
UnableToFetchCriticalDownloadDataError UnableToFetchCriticalDownloadDataError
@ -1330,12 +1329,11 @@ class Downloader:
self._all_downloads = weakref.WeakKeyDictionary() # for debugging self._all_downloads = weakref.WeakKeyDictionary() # for debugging
def download(self, u, t, _log_msg_id=None, monitor=None, history=None): def download(self, u, t, _log_msg_id=None, monitor=None, history=None):
u = IFileURI(u) assert isinstance(u, uri.CHKFileURI)
t = IDownloadTarget(t) t = IDownloadTarget(t)
assert t.write assert t.write
assert t.close assert t.close
assert isinstance(u, uri.CHKFileURI)
if self.stats_provider: if self.stats_provider:
# these counters are meant for network traffic, and don't # these counters are meant for network traffic, and don't
# include LIT files # include LIT files

View File

@ -8,7 +8,7 @@ from foolscap.api import eventually
from allmydata.interfaces import IFileNode, ICheckable, \ from allmydata.interfaces import IFileNode, ICheckable, \
IDownloadTarget, IUploadResults IDownloadTarget, IUploadResults
from allmydata.util import dictutil, log, base32 from allmydata.util import dictutil, log, base32
from allmydata import uri as urimodule from allmydata.uri import CHKFileURI, LiteralFileURI
from allmydata.immutable.checker import Checker from allmydata.immutable.checker import Checker
from allmydata.check_results import CheckResults, CheckAndRepairResults from allmydata.check_results import CheckResults, CheckAndRepairResults
from allmydata.immutable.repairer import Repairer from allmydata.immutable.repairer import Repairer
@ -181,8 +181,8 @@ class DownloadCache:
class FileNode(_ImmutableFileNodeBase, log.PrefixingLogMixin): class FileNode(_ImmutableFileNodeBase, log.PrefixingLogMixin):
def __init__(self, filecap, storage_broker, secret_holder, def __init__(self, filecap, storage_broker, secret_holder,
downloader, history, cachedirectorymanager): downloader, history, cachedirectorymanager):
assert isinstance(filecap, str) assert isinstance(filecap, CHKFileURI)
self.u = urimodule.CHKFileURI.init_from_string(filecap) self.u = filecap
self._storage_broker = storage_broker self._storage_broker = storage_broker
self._secret_holder = secret_holder self._secret_holder = secret_holder
self._downloader = downloader self._downloader = downloader
@ -194,19 +194,22 @@ class FileNode(_ImmutableFileNodeBase, log.PrefixingLogMixin):
log.PrefixingLogMixin.__init__(self, "allmydata.immutable.filenode", prefix=prefix) log.PrefixingLogMixin.__init__(self, "allmydata.immutable.filenode", prefix=prefix)
self.log("starting", level=log.OPERATIONAL) self.log("starting", level=log.OPERATIONAL)
def get_uri(self):
return self.u.to_string()
def get_size(self): def get_size(self):
return self.u.get_size() return self.u.get_size()
def get_cap(self):
return self.u
def get_readcap(self):
return self.u.get_readonly()
def get_verify_cap(self): def get_verify_cap(self):
return self.u.get_verify_cap() return self.u.get_verify_cap()
def get_repair_cap(self): def get_repair_cap(self):
# CHK files can be repaired with just the verifycap # CHK files can be repaired with just the verifycap
return self.u.get_verify_cap() return self.u.get_verify_cap()
def get_uri(self):
return self.u.to_string()
def get_storage_index(self): def get_storage_index(self):
return self.u.storage_index return self.u.storage_index
@ -294,13 +297,15 @@ class FileNode(_ImmutableFileNodeBase, log.PrefixingLogMixin):
return d return d
def download(self, target): def download(self, target):
return self._downloader.download(self.get_uri(), target, return self._downloader.download(self.get_cap(), target,
self._parentmsgid, self._parentmsgid,
history=self._history) history=self._history)
def download_to_data(self): def download_to_data(self):
return self._downloader.download_to_data(self.get_uri(), return self._downloader.download_to_data(self.get_cap(),
history=self._history) history=self._history)
def download_to_filename(self, filename):
return self._downloader.download_to_filename(self.get_cap(), filename)
class LiteralProducer: class LiteralProducer:
implements(IPushProducer) implements(IPushProducer)
@ -313,21 +318,24 @@ class LiteralProducer:
class LiteralFileNode(_ImmutableFileNodeBase): class LiteralFileNode(_ImmutableFileNodeBase):
def __init__(self, filecap): def __init__(self, filecap):
assert isinstance(filecap, str) assert isinstance(filecap, LiteralFileURI)
self.u = urimodule.LiteralFileURI.init_from_string(filecap) self.u = filecap
def get_uri(self):
return self.u.to_string()
def get_size(self): def get_size(self):
return len(self.u.data) return len(self.u.data)
def get_cap(self):
return self.u
def get_readcap(self):
return self.u
def get_verify_cap(self): def get_verify_cap(self):
return None return None
def get_repair_cap(self): def get_repair_cap(self):
return None return None
def get_uri(self):
return self.u.to_string()
def get_storage_index(self): def get_storage_index(self):
return None return None

View File

@ -481,6 +481,32 @@ class UnhandledCapTypeError(Exception):
it.""" it."""
class IFilesystemNode(Interface): class IFilesystemNode(Interface):
def get_cap():
"""Return the strongest 'cap instance' associated with this node.
(writecap for writeable-mutable files/directories, readcap for
immutable or readonly-mutable files/directories). To convert this
into a string, call .to_string() on the result."""
def get_readcap():
"""Return a readonly cap instance for this node. For immutable or
readonly nodes, get_cap() and get_readcap() return the same thing."""
def get_repair_cap():
"""Return an IURI instance that can be used to repair the file, or
None if this node cannot be repaired (either because it is not
distributed, like a LIT file, or because the node does not represent
sufficient authority to create a repair-cap, like a read-only RSA
mutable file node [which cannot create the correct write-enablers]).
"""
def get_verify_cap():
"""Return an IVerifierURI instance that represents the
'verifiy/refresh capability' for this node. The holder of this
capability will be able to renew the lease for this node, protecting
it from garbage-collection. They will also be able to ask a server if
it holds a share for the file or directory.
"""
def get_uri(): def get_uri():
""" """
Return the URI string that can be used by others to get access to Return the URI string that can be used by others to get access to
@ -501,22 +527,6 @@ class IFilesystemNode(Interface):
will return the same thing as get_uri(). will return the same thing as get_uri().
""" """
def get_repair_cap():
"""Return an IURI instance that can be used to repair the file, or
None if this node cannot be repaired (either because it is not
distributed, like a LIT file, or because the node does not represent
sufficient authority to create a repair-cap, like a read-only RSA
mutable file node [which cannot create the correct write-enablers]).
"""
def get_verify_cap():
"""Return an IVerifierURI instance that represents the
'verifiy/refresh capability' for this node. The holder of this
capability will be able to renew the lease for this node, protecting
it from garbage-collection. They will also be able to ask a server if
it holds a share for the file or directory.
"""
def get_storage_index(): def get_storage_index():
"""Return a string with the (binary) storage index in use on this """Return a string with the (binary) storage index in use on this
download. This may be None if there is no storage index (i.e. LIT download. This may be None if there is no storage index (i.e. LIT

View File

@ -76,19 +76,16 @@ class MutableFileNode:
else: else:
return "<%s %x %s %s>" % (self.__class__.__name__, id(self), None, None) return "<%s %x %s %s>" % (self.__class__.__name__, id(self), None, None)
def init_from_uri(self, filecap): def init_from_cap(self, filecap):
# we have the URI, but we have not yet retrieved the public # we have the URI, but we have not yet retrieved the public
# verification key, nor things like 'k' or 'N'. If and when someone # verification key, nor things like 'k' or 'N'. If and when someone
# wants to get our contents, we'll pull from shares and fill those # wants to get our contents, we'll pull from shares and fill those
# in. # in.
assert isinstance(filecap, str) assert isinstance(filecap, (ReadonlySSKFileURI, WriteableSSKFileURI))
if filecap.startswith("URI:SSK:"): self._uri = filecap
self._uri = WriteableSSKFileURI.init_from_string(filecap) self._writekey = None
if isinstance(filecap, WriteableSSKFileURI):
self._writekey = self._uri.writekey self._writekey = self._uri.writekey
else:
assert filecap.startswith("URI:SSK-RO:")
self._uri = ReadonlySSKFileURI.init_from_string(filecap)
self._writekey = None
self._readkey = self._uri.readkey self._readkey = self._uri.readkey
self._storage_index = self._uri.storage_index self._storage_index = self._uri.storage_index
self._fingerprint = self._uri.fingerprint self._fingerprint = self._uri.fingerprint
@ -188,21 +185,33 @@ class MutableFileNode:
#################################### ####################################
# IFilesystemNode # IFilesystemNode
def get_uri(self):
return self._uri.to_string()
def get_size(self): def get_size(self):
return "?" # TODO: this is likely to cause problems, not being an int return "?" # TODO: this is likely to cause problems, not being an int
def get_cap(self):
return self._uri
def get_readcap(self):
return self._uri.get_readonly()
def get_verify_cap(self):
return IMutableFileURI(self._uri).get_verify_cap()
def get_repair_cap(self):
if self._uri.is_readonly():
return None
return self._uri
def get_uri(self):
return self._uri.to_string()
def get_readonly_uri(self):
return self._uri.get_readonly().to_string()
def get_readonly(self): def get_readonly(self):
if self.is_readonly(): if self.is_readonly():
return self return self
ro = MutableFileNode(self._storage_broker, self._secret_holder, ro = MutableFileNode(self._storage_broker, self._secret_holder,
self._default_encoding_parameters, self._history) self._default_encoding_parameters, self._history)
ro.init_from_uri(self.get_readonly_uri()) ro.init_from_cap(self._uri.get_readonly())
return ro return ro
def get_readonly_uri(self):
return self._uri.get_readonly().to_string()
def is_mutable(self): def is_mutable(self):
return self._uri.is_mutable() return self._uri.is_mutable()
def is_readonly(self): def is_readonly(self):
@ -217,14 +226,6 @@ class MutableFileNode:
return cmp(self.__class__, them.__class__) return cmp(self.__class__, them.__class__)
return cmp(self._uri, them._uri) return cmp(self._uri, them._uri)
def get_verify_cap(self):
return IMutableFileURI(self._uri).get_verify_cap()
def get_repair_cap(self):
if self._uri.is_readonly():
return None
return self._uri
def _do_serialized(self, cb, *args, **kwargs): def _do_serialized(self, cb, *args, **kwargs):
# note: to avoid deadlock, this callable is *not* allowed to invoke # note: to avoid deadlock, this callable is *not* allowed to invoke
# other serialized methods within this (or any other) # other serialized methods within this (or any other)

View File

@ -6,7 +6,7 @@ from allmydata.immutable.filenode import FileNode, LiteralFileNode
from allmydata.mutable.filenode import MutableFileNode from allmydata.mutable.filenode import MutableFileNode
from allmydata.dirnode import DirectoryNode, pack_children from allmydata.dirnode import DirectoryNode, pack_children
from allmydata.unknown import UnknownNode from allmydata.unknown import UnknownNode
from allmydata.uri import DirectoryURI, ReadonlyDirectoryURI from allmydata import uri
class NodeMaker: class NodeMaker:
implements(INodeMaker) implements(INodeMaker)
@ -35,41 +35,42 @@ class NodeMaker:
n = MutableFileNode(self.storage_broker, self.secret_holder, n = MutableFileNode(self.storage_broker, self.secret_holder,
self.default_encoding_parameters, self.default_encoding_parameters,
self.history) self.history)
return n.init_from_uri(cap) return n.init_from_cap(cap)
def _create_dirnode(self, filenode): def _create_dirnode(self, filenode):
return DirectoryNode(filenode, self, self.uploader) return DirectoryNode(filenode, self, self.uploader)
def create_from_cap(self, writecap, readcap=None): def create_from_cap(self, writecap, readcap=None):
# this returns synchronously. # this returns synchronously. It starts with a "cap string".
assert isinstance(writecap, (str, type(None))), type(writecap) assert isinstance(writecap, (str, type(None))), type(writecap)
assert isinstance(readcap, (str, type(None))), type(readcap) assert isinstance(readcap, (str, type(None))), type(readcap)
cap = writecap or readcap bigcap = writecap or readcap
if not cap: if not bigcap:
# maybe the writecap was hidden because we're in a readonly # maybe the writecap was hidden because we're in a readonly
# directory, and the future cap format doesn't have a readcap, or # directory, and the future cap format doesn't have a readcap, or
# something. # something.
return UnknownNode(writecap, readcap) return UnknownNode(writecap, readcap)
if cap in self._node_cache: if bigcap in self._node_cache:
return self._node_cache[cap] return self._node_cache[bigcap]
elif cap.startswith("URI:LIT:"): cap = uri.from_string(bigcap)
node = self._create_lit(cap) node = self._create_from_cap(cap)
elif cap.startswith("URI:CHK:"): if node:
node = self._create_immutable(cap) self._node_cache[bigcap] = node # note: WeakValueDictionary
elif cap.startswith("URI:SSK-RO:") or cap.startswith("URI:SSK:"):
node = self._create_mutable(cap)
elif cap.startswith("URI:DIR2-RO:") or cap.startswith("URI:DIR2:"):
if cap.startswith("URI:DIR2-RO:"):
dircap = ReadonlyDirectoryURI.init_from_string(cap)
elif cap.startswith("URI:DIR2:"):
dircap = DirectoryURI.init_from_string(cap)
filecap = dircap.get_filenode_uri().to_string()
filenode = self.create_from_cap(filecap)
node = self._create_dirnode(filenode)
else: else:
return UnknownNode(writecap, readcap) # don't cache UnknownNode node = UnknownNode(writecap, readcap) # don't cache UnknownNode
self._node_cache[cap] = node # note: WeakValueDictionary
return node return node
def _create_from_cap(self, cap):
# This starts with a "cap instance"
if isinstance(cap, uri.LiteralFileURI):
return self._create_lit(cap)
if isinstance(cap, uri.CHKFileURI):
return self._create_immutable(cap)
if isinstance(cap, (uri.ReadonlySSKFileURI, uri.WriteableSSKFileURI)):
return self._create_mutable(cap)
if isinstance(cap, (uri.ReadonlyDirectoryURI, uri.DirectoryURI)):
filenode = self._create_from_cap(cap.get_filenode_cap())
return self._create_dirnode(filenode)
return None
def create_mutable_file(self, contents=None, keysize=None): def create_mutable_file(self, contents=None, keysize=None):
n = MutableFileNode(self.storage_broker, self.secret_holder, n = MutableFileNode(self.storage_broker, self.secret_holder,

View File

@ -2,49 +2,32 @@ import hotshot.stats, os, random, sys
from pyutil import benchutil, randutil # http://allmydata.org/trac/pyutil from pyutil import benchutil, randutil # http://allmydata.org/trac/pyutil
from allmydata import client, dirnode, uri from allmydata import dirnode, uri
from allmydata.mutable import filenode as mut_filenode from allmydata.mutable import filenode as mut_filenode
from allmydata.immutable import filenode as immut_filenode from allmydata.immutable import filenode as immut_filenode
from allmydata.util import cachedir, fileutil
class FakeClient(client.Client):
# just enough
def __init__(self):
self._node_cache = {}
download_cachedir = fileutil.NamedTemporaryDirectory()
self.download_cache_dirman = cachedir.CacheDirectoryManager(download_cachedir.name)
def getServiceNamed(self, name):
return None
def get_storage_broker(self):
return None
_secret_holder=None
def get_history(self):
return None
def get_encoding_parameters(self):
return {"k": 3, "n": 10}
def get_writekey(self):
return os.urandom(16)
def create_node_from_uri(self, writecap, readcap):
return None
class FakeMutableFileNode(mut_filenode.MutableFileNode):
def __init__(self, *args, **kwargs):
mut_filenode.MutableFileNode.__init__(self, *args, **kwargs)
self._uri = uri.WriteableSSKFileURI(randutil.insecurerandstr(16), randutil.insecurerandstr(32))
class FakeDirectoryNode(dirnode.DirectoryNode):
def __init__(self, client):
dirnode.DirectoryNode.__init__(self, client)
mutfileuri = uri.WriteableSSKFileURI(randutil.insecurerandstr(16), randutil.insecurerandstr(32))
myuri = uri.DirectoryURI(mutfileuri).to_string()
self.init_from_uri(myuri)
children = [] # tuples of (k, v) (suitable for passing to dict()) children = [] # tuples of (k, v) (suitable for passing to dict())
packstr = None packstr = None
fakeclient = FakeClient()
testdirnode = dirnode.DirectoryNode(fakeclient) class ContainerNode:
testdirnode.init_from_uri(uri.DirectoryURI(uri.WriteableSSKFileURI(randutil.insecurerandstr(16), randutil.insecurerandstr(32))).to_string()) # dirnodes sit on top of a "container" filenode, from which it extracts a
# writekey
def __init__(self):
self._writekey = randutil.insecurerandstr(16)
self._fingerprint = randutil.insecurerandstr(32)
self._cap = uri.WriteableSSKFileURI(self._writekey, self._fingerprint)
def get_writekey(self):
return self._writekey
def get_uri(self):
return self._cap.to_string()
def is_readonly(self):
return False
class FakeNodeMaker:
def create_from_cap(self, writecap, readcap=None):
return None
nodemaker = FakeNodeMaker()
testdirnode = dirnode.DirectoryNode(ContainerNode(), nodemaker, uploader=None)
def random_unicode(l): def random_unicode(l):
while True: while True:
@ -53,16 +36,28 @@ def random_unicode(l):
except UnicodeDecodeError: except UnicodeDecodeError:
pass pass
encoding_parameters = {"k": 3, "n": 10}
def random_fsnode(): def random_fsnode():
coin = random.randrange(0, 3) coin = random.randrange(0, 3)
if coin == 0: if coin == 0:
return immut_filenode.FileNode(uri.CHKFileURI(randutil.insecurerandstr(16), randutil.insecurerandstr(32), random.randrange(1, 5), random.randrange(6, 15), random.randrange(99, 1000000000000)).to_string(), None, None, None, None, None) cap = uri.CHKFileURI(randutil.insecurerandstr(16),
randutil.insecurerandstr(32),
random.randrange(1, 5),
random.randrange(6, 15),
random.randrange(99, 1000000000000))
return immut_filenode.FileNode(cap, None, None, None, None, None)
elif coin == 1: elif coin == 1:
encoding_parameters = {"k": 3, "n": 10} cap = uri.WriteableSSKFileURI(randutil.insecurerandstr(16),
return FakeMutableFileNode(None, None, encoding_parameters, None) randutil.insecurerandstr(32))
n = mut_filenode.MutableFileNode(None, None, encoding_parameters, None)
return n.init_from_cap(cap)
else: else:
assert coin == 2 assert coin == 2
return FakeDirectoryNode(fakeclient) cap = uri.WriteableSSKFileURI(randutil.insecurerandstr(16),
randutil.insecurerandstr(32))
n = mut_filenode.MutableFileNode(None, None, encoding_parameters, None)
n.init_from_cap(cap)
return dirnode.DirectoryNode(n, nodemaker, uploader=None)
def random_metadata(): def random_metadata():
d = {} d = {}
@ -78,7 +73,8 @@ def random_child():
def init_for_pack(N): def init_for_pack(N):
for i in xrange(len(children), N): for i in xrange(len(children), N):
children.append((random_unicode(random.randrange(1, 9)), random_child())) name = random_unicode(random.randrange(1, 9))
children.append( (name, random_child()) )
def init_for_unpack(N): def init_for_unpack(N):
global packstr global packstr
@ -86,7 +82,7 @@ def init_for_unpack(N):
packstr = pack(N) packstr = pack(N)
def pack(N): def pack(N):
return testdirnode._pack_contents(dirnode.CachingDict(children[:N])) return testdirnode._pack_contents(dict(children[:N]))
def unpack(N): def unpack(N):
return testdirnode._unpack_contents(packstr) return testdirnode._unpack_contents(packstr)
@ -97,9 +93,12 @@ def unpack_and_repack(N):
PROF_FILE_NAME="bench_dirnode.prof" PROF_FILE_NAME="bench_dirnode.prof"
def run_benchmarks(profile=False): def run_benchmarks(profile=False):
for (func, initfunc) in [(unpack, init_for_unpack), (pack, init_for_pack), (unpack_and_repack, init_for_unpack)]: for (initfunc, func) in [(init_for_unpack, unpack),
(init_for_pack, pack),
(init_for_unpack, unpack_and_repack)]:
print "benchmarking %s" % (func,) print "benchmarking %s" % (func,)
benchutil.bench(unpack_and_repack, initfunc=init_for_unpack, TOPXP=12)#, profile=profile, profresults=PROF_FILE_NAME) benchutil.bench(unpack_and_repack, initfunc=init_for_unpack,
TOPXP=12)#, profile=profile, profresults=PROF_FILE_NAME)
def print_stats(): def print_stats():
s = hotshot.stats.load(PROF_FILE_NAME) s = hotshot.stats.load(PROF_FILE_NAME)

View File

@ -39,8 +39,8 @@ class FakeCHKFileNode:
bad_shares = {} bad_shares = {}
def __init__(self, filecap): def __init__(self, filecap):
precondition(isinstance(filecap, str), filecap) precondition(isinstance(filecap, uri.CHKFileURI), filecap)
self.my_uri = uri.CHKFileURI.init_from_string(filecap) self.my_uri = filecap
self.storage_index = self.my_uri.storage_index self.storage_index = self.my_uri.storage_index
def get_uri(self): def get_uri(self):
@ -130,18 +130,19 @@ class FakeCHKFileNode:
d.addCallback(_got) d.addCallback(_got)
return d return d
def make_chk_file_cap(size):
return uri.CHKFileURI(key=os.urandom(16),
uri_extension_hash=os.urandom(32),
needed_shares=3,
total_shares=10,
size=size)
def make_chk_file_uri(size): def make_chk_file_uri(size):
u = uri.CHKFileURI(key=os.urandom(16), return make_chk_file_cap(size).to_string()
uri_extension_hash=os.urandom(32),
needed_shares=3,
total_shares=10,
size=size)
return u.to_string()
def create_chk_filenode(contents): def create_chk_filenode(contents):
filecap = make_chk_file_uri(len(contents)) filecap = make_chk_file_cap(len(contents))
n = FakeCHKFileNode(filecap) n = FakeCHKFileNode(filecap)
FakeCHKFileNode.all_contents[filecap] = contents FakeCHKFileNode.all_contents[filecap.to_string()] = contents
return n return n
@ -156,7 +157,7 @@ class FakeMutableFileNode:
def __init__(self, storage_broker, secret_holder, def __init__(self, storage_broker, secret_holder,
default_encoding_parameters, history): default_encoding_parameters, history):
self.init_from_uri(make_mutable_file_uri()) self.init_from_cap(make_mutable_file_cap())
def create(self, contents, key_generator=None, keysize=None): def create(self, contents, key_generator=None, keysize=None):
initial_contents = self._get_initial_contents(contents) initial_contents = self._get_initial_contents(contents)
if len(initial_contents) > self.MUTABLE_SIZELIMIT: if len(initial_contents) > self.MUTABLE_SIZELIMIT:
@ -173,15 +174,16 @@ class FakeMutableFileNode:
assert callable(contents), "%s should be callable, not %s" % \ assert callable(contents), "%s should be callable, not %s" % \
(contents, type(contents)) (contents, type(contents))
return contents(self) return contents(self)
def init_from_uri(self, filecap): def init_from_cap(self, filecap):
assert isinstance(filecap, str) assert isinstance(filecap, (uri.WriteableSSKFileURI,
if filecap.startswith("URI:SSK:"): uri.ReadonlySSKFileURI))
self.my_uri = uri.WriteableSSKFileURI.init_from_string(filecap) self.my_uri = filecap
else:
assert filecap.startswith("URI:SSK-RO:")
self.my_uri = uri.ReadonlySSKFileURI.init_from_string(filecap)
self.storage_index = self.my_uri.storage_index self.storage_index = self.my_uri.storage_index
return self return self
def get_cap(self):
return self.my_uri
def get_readcap(self):
return self.my_uri.get_readonly()
def get_uri(self): def get_uri(self):
return self.my_uri.to_string() return self.my_uri.to_string()
def get_readonly(self): def get_readonly(self):
@ -296,9 +298,12 @@ class FakeMutableFileNode:
data = self.all_contents[self.storage_index] data = self.all_contents[self.storage_index]
return defer.succeed(data) return defer.succeed(data)
def make_mutable_file_uri(): def make_mutable_file_cap():
return uri.WriteableSSKFileURI(writekey=os.urandom(16), return uri.WriteableSSKFileURI(writekey=os.urandom(16),
fingerprint=os.urandom(32)).to_string() fingerprint=os.urandom(32))
def make_mutable_file_uri():
return make_mutable_file_cap().to_string()
def make_verifier_uri(): def make_verifier_uri():
return uri.SSKVerifierURI(storage_index=os.urandom(16), return uri.SSKVerifierURI(storage_index=os.urandom(16),
fingerprint=os.urandom(32)).to_string() fingerprint=os.urandom(32)).to_string()

View File

@ -1032,7 +1032,7 @@ class UCWEingNodeMaker(NodeMaker):
n = UCWEingMutableFileNode(self.storage_broker, self.secret_holder, n = UCWEingMutableFileNode(self.storage_broker, self.secret_holder,
self.default_encoding_parameters, self.default_encoding_parameters,
self.history) self.history)
return n.init_from_uri(cap) return n.init_from_cap(cap)
class Deleter(GridTestMixin, unittest.TestCase): class Deleter(GridTestMixin, unittest.TestCase):

View File

@ -32,12 +32,14 @@ class Node(unittest.TestCase):
size=1000) size=1000)
c = FakeClient() c = FakeClient()
cf = cachedir.CacheFile("none") cf = cachedir.CacheFile("none")
fn1 = filenode.FileNode(u.to_string(), None, None, None, None, cf) fn1 = filenode.FileNode(u, None, None, None, None, cf)
fn2 = filenode.FileNode(u.to_string(), None, None, None, None, cf) fn2 = filenode.FileNode(u, None, None, None, None, cf)
self.failUnlessEqual(fn1, fn2) self.failUnlessEqual(fn1, fn2)
self.failIfEqual(fn1, "I am not a filenode") self.failIfEqual(fn1, "I am not a filenode")
self.failIfEqual(fn1, NotANode()) self.failIfEqual(fn1, NotANode())
self.failUnlessEqual(fn1.get_uri(), u.to_string()) self.failUnlessEqual(fn1.get_uri(), u.to_string())
self.failUnlessEqual(fn1.get_cap(), u)
self.failUnlessEqual(fn1.get_readcap(), u)
self.failUnlessEqual(fn1.is_readonly(), True) self.failUnlessEqual(fn1.is_readonly(), True)
self.failUnlessEqual(fn1.is_mutable(), False) self.failUnlessEqual(fn1.is_mutable(), False)
self.failUnlessEqual(fn1.get_readonly_uri(), u.to_string()) self.failUnlessEqual(fn1.get_readonly_uri(), u.to_string())
@ -54,12 +56,14 @@ class Node(unittest.TestCase):
DATA = "I am a short file." DATA = "I am a short file."
u = uri.LiteralFileURI(data=DATA) u = uri.LiteralFileURI(data=DATA)
c = None c = None
fn1 = filenode.LiteralFileNode(u.to_string()) fn1 = filenode.LiteralFileNode(u)
fn2 = filenode.LiteralFileNode(u.to_string()) fn2 = filenode.LiteralFileNode(u)
self.failUnlessEqual(fn1, fn2) self.failUnlessEqual(fn1, fn2)
self.failIfEqual(fn1, "I am not a filenode") self.failIfEqual(fn1, "I am not a filenode")
self.failIfEqual(fn1, NotANode()) self.failIfEqual(fn1, NotANode())
self.failUnlessEqual(fn1.get_uri(), u.to_string()) self.failUnlessEqual(fn1.get_uri(), u.to_string())
self.failUnlessEqual(fn1.get_cap(), u)
self.failUnlessEqual(fn1.get_readcap(), u)
self.failUnlessEqual(fn1.is_readonly(), True) self.failUnlessEqual(fn1.is_readonly(), True)
self.failUnlessEqual(fn1.is_mutable(), False) self.failUnlessEqual(fn1.is_mutable(), False)
self.failUnlessEqual(fn1.get_readonly_uri(), u.to_string()) self.failUnlessEqual(fn1.get_readonly_uri(), u.to_string())
@ -99,7 +103,7 @@ class Node(unittest.TestCase):
u = uri.WriteableSSKFileURI("\x00"*16, "\x00"*32) u = uri.WriteableSSKFileURI("\x00"*16, "\x00"*32)
n = MutableFileNode(None, None, client.get_encoding_parameters(), n = MutableFileNode(None, None, client.get_encoding_parameters(),
None).init_from_uri(u.to_string()) None).init_from_cap(u)
self.failUnlessEqual(n.get_writekey(), wk) self.failUnlessEqual(n.get_writekey(), wk)
self.failUnlessEqual(n.get_readkey(), rk) self.failUnlessEqual(n.get_readkey(), rk)
@ -112,11 +116,13 @@ class Node(unittest.TestCase):
self.failUnlessEqual(n.get_uri(), u.to_string()) self.failUnlessEqual(n.get_uri(), u.to_string())
self.failUnlessEqual(n.get_readonly_uri(), u.get_readonly().to_string()) self.failUnlessEqual(n.get_readonly_uri(), u.get_readonly().to_string())
self.failUnlessEqual(n.get_cap(), u)
self.failUnlessEqual(n.get_readcap(), u.get_readonly())
self.failUnlessEqual(n.is_mutable(), True) self.failUnlessEqual(n.is_mutable(), True)
self.failUnlessEqual(n.is_readonly(), False) self.failUnlessEqual(n.is_readonly(), False)
n2 = MutableFileNode(None, None, client.get_encoding_parameters(), n2 = MutableFileNode(None, None, client.get_encoding_parameters(),
None).init_from_uri(u.to_string()) None).init_from_cap(u)
self.failUnlessEqual(n, n2) self.failUnlessEqual(n, n2)
self.failIfEqual(n, "not even the right type") self.failIfEqual(n, "not even the right type")
self.failIfEqual(n, u) # not the right class self.failIfEqual(n, u) # not the right class
@ -128,6 +134,8 @@ class Node(unittest.TestCase):
self.failUnless(isinstance(nro, MutableFileNode)) self.failUnless(isinstance(nro, MutableFileNode))
self.failUnlessEqual(nro.get_readonly(), nro) self.failUnlessEqual(nro.get_readonly(), nro)
self.failUnlessEqual(nro.get_cap(), u.get_readonly())
self.failUnlessEqual(nro.get_readcap(), u.get_readonly())
nro_u = nro.get_uri() nro_u = nro.get_uri()
self.failUnlessEqual(nro_u, nro.get_readonly_uri()) self.failUnlessEqual(nro_u, nro.get_readonly_uri())
self.failUnlessEqual(nro_u, u.get_readonly().to_string()) self.failUnlessEqual(nro_u, u.get_readonly().to_string())
@ -143,7 +151,7 @@ class LiteralChecker(unittest.TestCase):
def test_literal_filenode(self): def test_literal_filenode(self):
DATA = "I am a short file." DATA = "I am a short file."
u = uri.LiteralFileURI(data=DATA) u = uri.LiteralFileURI(data=DATA)
fn1 = filenode.LiteralFileNode(u.to_string()) fn1 = filenode.LiteralFileNode(u)
d = fn1.check(Monitor()) d = fn1.check(Monitor())
def _check_checker_results(cr): def _check_checker_results(cr):

View File

@ -135,6 +135,7 @@ class SystemTest(SystemTestMixin, unittest.TestCase):
log.msg("upload finished: uri is %s" % (theuri,)) log.msg("upload finished: uri is %s" % (theuri,))
self.uri = theuri self.uri = theuri
assert isinstance(self.uri, str), self.uri assert isinstance(self.uri, str), self.uri
self.cap = uri.from_string(self.uri)
self.downloader = self.clients[1].downloader self.downloader = self.clients[1].downloader
d.addCallback(_upload_done) d.addCallback(_upload_done)
@ -151,7 +152,7 @@ class SystemTest(SystemTestMixin, unittest.TestCase):
def _download_to_data(res): def _download_to_data(res):
log.msg("DOWNLOADING") log.msg("DOWNLOADING")
return self.downloader.download_to_data(self.uri) return self.downloader.download_to_data(self.cap)
d.addCallback(_download_to_data) d.addCallback(_download_to_data)
def _download_to_data_done(data): def _download_to_data_done(data):
log.msg("download finished") log.msg("download finished")
@ -160,7 +161,7 @@ class SystemTest(SystemTestMixin, unittest.TestCase):
target_filename = os.path.join(self.basedir, "download.target") target_filename = os.path.join(self.basedir, "download.target")
def _download_to_filename(res): def _download_to_filename(res):
return self.downloader.download_to_filename(self.uri, return self.downloader.download_to_filename(self.cap,
target_filename) target_filename)
d.addCallback(_download_to_filename) d.addCallback(_download_to_filename)
def _download_to_filename_done(res): def _download_to_filename_done(res):
@ -171,7 +172,7 @@ class SystemTest(SystemTestMixin, unittest.TestCase):
target_filename2 = os.path.join(self.basedir, "download.target2") target_filename2 = os.path.join(self.basedir, "download.target2")
def _download_to_filehandle(res): def _download_to_filehandle(res):
fh = open(target_filename2, "wb") fh = open(target_filename2, "wb")
return self.downloader.download_to_filehandle(self.uri, fh) return self.downloader.download_to_filehandle(self.cap, fh)
d.addCallback(_download_to_filehandle) d.addCallback(_download_to_filehandle)
def _download_to_filehandle_done(fh): def _download_to_filehandle_done(fh):
fh.close() fh.close()
@ -182,7 +183,7 @@ class SystemTest(SystemTestMixin, unittest.TestCase):
consumer = GrabEverythingConsumer() consumer = GrabEverythingConsumer()
ct = download.ConsumerAdapter(consumer) ct = download.ConsumerAdapter(consumer)
d.addCallback(lambda res: d.addCallback(lambda res:
self.downloader.download(self.uri, ct)) self.downloader.download(self.cap, ct))
def _download_to_consumer_done(ign): def _download_to_consumer_done(ign):
self.failUnlessEqual(consumer.contents, DATA) self.failUnlessEqual(consumer.contents, DATA)
d.addCallback(_download_to_consumer_done) d.addCallback(_download_to_consumer_done)
@ -228,7 +229,7 @@ class SystemTest(SystemTestMixin, unittest.TestCase):
baduri = self.mangle_uri(self.uri) baduri = self.mangle_uri(self.uri)
log.msg("about to download non-existent URI", level=log.UNUSUAL, log.msg("about to download non-existent URI", level=log.UNUSUAL,
facility="tahoe.tests") facility="tahoe.tests")
d1 = self.downloader.download_to_data(baduri) d1 = self.downloader.download_to_data(uri.from_string(baduri))
def _baduri_should_fail(res): def _baduri_should_fail(res):
log.msg("finished downloading non-existend URI", log.msg("finished downloading non-existend URI",
level=log.UNUSUAL, facility="tahoe.tests") level=log.UNUSUAL, facility="tahoe.tests")
@ -253,8 +254,8 @@ class SystemTest(SystemTestMixin, unittest.TestCase):
u = upload.Data(HELPER_DATA, convergence=convergence) u = upload.Data(HELPER_DATA, convergence=convergence)
d = self.extra_node.upload(u) d = self.extra_node.upload(u)
def _uploaded(results): def _uploaded(results):
uri = results.uri cap = uri.from_string(results.uri)
return self.downloader.download_to_data(uri) return self.downloader.download_to_data(cap)
d.addCallback(_uploaded) d.addCallback(_uploaded)
def _check(newdata): def _check(newdata):
self.failUnlessEqual(newdata, HELPER_DATA) self.failUnlessEqual(newdata, HELPER_DATA)
@ -267,8 +268,8 @@ class SystemTest(SystemTestMixin, unittest.TestCase):
u.debug_stash_RemoteEncryptedUploadable = True u.debug_stash_RemoteEncryptedUploadable = True
d = self.extra_node.upload(u) d = self.extra_node.upload(u)
def _uploaded(results): def _uploaded(results):
uri = results.uri cap = uri.from_string(results.uri)
return self.downloader.download_to_data(uri) return self.downloader.download_to_data(cap)
d.addCallback(_uploaded) d.addCallback(_uploaded)
def _check(newdata): def _check(newdata):
self.failUnlessEqual(newdata, HELPER_DATA) self.failUnlessEqual(newdata, HELPER_DATA)
@ -355,7 +356,7 @@ class SystemTest(SystemTestMixin, unittest.TestCase):
d.addCallback(lambda res: self.extra_node.upload(u2)) d.addCallback(lambda res: self.extra_node.upload(u2))
def _uploaded(results): def _uploaded(results):
uri = results.uri cap = uri.from_string(results.uri)
log.msg("Second upload complete", level=log.NOISY, log.msg("Second upload complete", level=log.NOISY,
facility="tahoe.test.test_system") facility="tahoe.test.test_system")
@ -383,7 +384,7 @@ class SystemTest(SystemTestMixin, unittest.TestCase):
"resumption saved us some work even though we were using random keys:" "resumption saved us some work even though we were using random keys:"
" read %d bytes out of %d total" % " read %d bytes out of %d total" %
(bytes_sent, len(DATA))) (bytes_sent, len(DATA)))
return self.downloader.download_to_data(uri) return self.downloader.download_to_data(cap)
d.addCallback(_uploaded) d.addCallback(_uploaded)
def _check(newdata): def _check(newdata):

View File

@ -279,7 +279,7 @@ class NewDirnode(unittest.TestCase):
self.failIf(IFileURI.providedBy(u1)) self.failIf(IFileURI.providedBy(u1))
self.failUnless(IDirnodeURI.providedBy(u1)) self.failUnless(IDirnodeURI.providedBy(u1))
self.failUnless("DirectoryURI" in str(u1)) self.failUnless("DirectoryURI" in str(u1))
u1_filenode = u1.get_filenode_uri() u1_filenode = u1.get_filenode_cap()
self.failUnless(u1_filenode.is_mutable()) self.failUnless(u1_filenode.is_mutable())
self.failIf(u1_filenode.is_readonly()) self.failIf(u1_filenode.is_readonly())
u1a = IDirnodeURI(u1.to_string()) u1a = IDirnodeURI(u1.to_string())
@ -302,7 +302,7 @@ class NewDirnode(unittest.TestCase):
u3n = u3._filenode_uri u3n = u3._filenode_uri
self.failUnless(u3n.is_readonly()) self.failUnless(u3n.is_readonly())
self.failUnless(u3n.is_mutable()) self.failUnless(u3n.is_mutable())
u3_filenode = u3.get_filenode_uri() u3_filenode = u3.get_filenode_cap()
self.failUnless(u3_filenode.is_mutable()) self.failUnless(u3_filenode.is_mutable())
self.failUnless(u3_filenode.is_readonly()) self.failUnless(u3_filenode.is_readonly())
@ -318,7 +318,7 @@ class NewDirnode(unittest.TestCase):
self.failUnless(IDirnodeURI.providedBy(u4)) self.failUnless(IDirnodeURI.providedBy(u4))
u4_verifier = u4.get_verify_cap() u4_verifier = u4.get_verify_cap()
u4_verifier_filenode = u4_verifier.get_filenode_uri() u4_verifier_filenode = u4_verifier.get_filenode_cap()
self.failUnless(isinstance(u4_verifier_filenode, uri.SSKVerifierURI)) self.failUnless(isinstance(u4_verifier_filenode, uri.SSKVerifierURI))
verifiers = [u1.get_verify_cap(), u2.get_verify_cap(), verifiers = [u1.get_verify_cap(), u2.get_verify_cap(),
@ -353,7 +353,7 @@ class NewDirnode(unittest.TestCase):
self.failIf(IFileURI.providedBy(u1)) self.failIf(IFileURI.providedBy(u1))
self.failUnless(IDirnodeURI.providedBy(u1)) self.failUnless(IDirnodeURI.providedBy(u1))
self.failUnless("DirectoryURI" in str(u1)) self.failUnless("DirectoryURI" in str(u1))
u1_filenode = u1.get_filenode_uri() u1_filenode = u1.get_filenode_cap()
self.failIf(u1_filenode.is_mutable()) self.failIf(u1_filenode.is_mutable())
self.failUnless(u1_filenode.is_readonly()) self.failUnless(u1_filenode.is_readonly())
self.failUnlessEqual(u1_filenode.to_string(), fncap) self.failUnlessEqual(u1_filenode.to_string(), fncap)
@ -375,7 +375,7 @@ class NewDirnode(unittest.TestCase):
self.failUnless(isinstance(u2_verifier, self.failUnless(isinstance(u2_verifier,
uri.ImmutableDirectoryURIVerifier), u2_verifier) uri.ImmutableDirectoryURIVerifier), u2_verifier)
self.failUnless(IVerifierURI.providedBy(u2_verifier)) self.failUnless(IVerifierURI.providedBy(u2_verifier))
u2_verifier_fileuri = u2_verifier.get_filenode_uri() u2_verifier_fileuri = u2_verifier.get_filenode_cap()
self.failUnless(IVerifierURI.providedBy(u2_verifier_fileuri)) self.failUnless(IVerifierURI.providedBy(u2_verifier_fileuri))
self.failUnlessEqual(u2_verifier_fileuri.to_string(), self.failUnlessEqual(u2_verifier_fileuri.to_string(),
fnuri.get_verify_cap().to_string()) fnuri.get_verify_cap().to_string())

View File

@ -43,7 +43,7 @@ class FakeNodeMaker(NodeMaker):
def _create_immutable(self, cap): def _create_immutable(self, cap):
return FakeCHKFileNode(cap) return FakeCHKFileNode(cap)
def _create_mutable(self, cap): def _create_mutable(self, cap):
return FakeMutableFileNode(None, None, None, None).init_from_uri(cap) return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
def create_mutable_file(self, contents="", keysize=None): def create_mutable_file(self, contents="", keysize=None):
n = FakeMutableFileNode(None, None, None, None) n = FakeMutableFileNode(None, None, None, None)
return n.create(contents) return n.create(contents)

View File

@ -374,7 +374,7 @@ class _DirectoryBaseURI(_BaseURI):
def abbrev_si(self): def abbrev_si(self):
return base32.b2a(self._filenode_uri.storage_index)[:5] return base32.b2a(self._filenode_uri.storage_index)[:5]
def get_filenode_uri(self): def get_filenode_cap(self):
return self._filenode_uri return self._filenode_uri
def is_mutable(self): def is_mutable(self):
@ -475,7 +475,7 @@ class DirectoryURIVerifier(_DirectoryBaseURI):
filenode_uri = IVerifierURI(filenode_uri) filenode_uri = IVerifierURI(filenode_uri)
self._filenode_uri = filenode_uri self._filenode_uri = filenode_uri
def get_filenode_uri(self): def get_filenode_cap(self):
return self._filenode_uri return self._filenode_uri
class ImmutableDirectoryURIVerifier(DirectoryURIVerifier): class ImmutableDirectoryURIVerifier(DirectoryURIVerifier):