ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2024-12-24 07:06:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Tests now pass on Python 3 too.

Browse Source
This commit is contained in:
Itamar Turner-Trauring 2020-08-19 12:15:39 -04:00
parent ff582c5129
commit 35ac5a62e7
2 changed files with 12 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/allmydata/storage/crawler.py
View File

@ -318,8 +318,7 @@ class ShareCrawler(service.MultiService):
try: try:
buckets = os.listdir(prefixdir) buckets = os.listdir(prefixdir)
buckets.sort() buckets.sort()
except EnvironmentError as e: except EnvironmentError:
print(e)
buckets = [] buckets = []
self.bucket_cache = (i, buckets) self.bucket_cache = (i, buckets)
self.process_prefixdir(cycle, prefix, prefixdir, self.process_prefixdir(cycle, prefix, prefixdir,
@ -361,7 +360,8 @@ class ShareCrawler(service.MultiService):
""" """
for bucket in buckets: for bucket in buckets:
if bucket <= self.state["last-complete-bucket"]: last_complete = self.state["last-complete-bucket"]
if last_complete is not None and bucket <= last_complete:
continue continue
self.process_bucket(cycle, prefix, prefixdir, bucket) self.process_bucket(cycle, prefix, prefixdir, bucket)
self.state["last-complete-bucket"] = bucket self.state["last-complete-bucket"] = bucket

10
src/allmydata/test/test_crawler.py
View File

@ -3,7 +3,7 @@ from __future__ import division
from __future__ import absolute_import from __future__ import absolute_import
from __future__ import unicode_literals from __future__ import unicode_literals
from future.utils import PY2 from future.utils import PY2, PY3, native_str
if PY2: if PY2:
# Don't use future bytes, since it breaks tests. # Don't use future bytes, since it breaks tests.
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min # noqa: F401
@ -30,6 +30,10 @@ class BucketEnumeratingCrawler(ShareCrawler):
self.all_buckets = [] self.all_buckets = []
self.finished_d = defer.Deferred() self.finished_d = defer.Deferred()
def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32): def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32):
if PY3:
# Bucket _inputs_ are bytes, and that's what we will compare this
# to:
storage_index_b32 = storage_index_b32.encode("ascii")
self.all_buckets.append(storage_index_b32) self.all_buckets.append(storage_index_b32)
def finished_cycle(self, cycle): def finished_cycle(self, cycle):
eventually(self.finished_d.callback, None) eventually(self.finished_d.callback, None)
@ -44,6 +48,10 @@ class PacedCrawler(ShareCrawler):
self.finished_d = defer.Deferred() self.finished_d = defer.Deferred()
self.yield_cb = None self.yield_cb = None
def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32): def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32):
if PY3:
# Bucket _inputs_ are bytes, and that's what we will compare this
# to:
storage_index_b32 = storage_index_b32.encode("ascii")
self.all_buckets.append(storage_index_b32) self.all_buckets.append(storage_index_b32)
self.countdown -= 1 self.countdown -= 1
if self.countdown == 0: if self.countdown == 0: