diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 3f48a127d..75dd723ce 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -318,8 +318,7 @@ class ShareCrawler(service.MultiService): try: buckets = os.listdir(prefixdir) buckets.sort() - except EnvironmentError as e: - print(e) + except EnvironmentError: buckets = [] self.bucket_cache = (i, buckets) self.process_prefixdir(cycle, prefix, prefixdir, @@ -361,7 +360,8 @@ class ShareCrawler(service.MultiService): """ for bucket in buckets: - if bucket <= self.state["last-complete-bucket"]: + last_complete = self.state["last-complete-bucket"] + if last_complete is not None and bucket <= last_complete: continue self.process_bucket(cycle, prefix, prefixdir, bucket) self.state["last-complete-bucket"] = bucket diff --git a/src/allmydata/test/test_crawler.py b/src/allmydata/test/test_crawler.py index 1e1e79adb..cfab04c31 100644 --- a/src/allmydata/test/test_crawler.py +++ b/src/allmydata/test/test_crawler.py @@ -3,7 +3,7 @@ from __future__ import division from __future__ import absolute_import from __future__ import unicode_literals -from future.utils import PY2 +from future.utils import PY2, PY3, native_str if PY2: # Don't use future bytes, since it breaks tests. from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 @@ -30,6 +30,10 @@ class BucketEnumeratingCrawler(ShareCrawler): self.all_buckets = [] self.finished_d = defer.Deferred() def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32): + if PY3: + # Bucket _inputs_ are bytes, and that's what we will compare this + # to: + storage_index_b32 = storage_index_b32.encode("ascii") self.all_buckets.append(storage_index_b32) def finished_cycle(self, cycle): eventually(self.finished_d.callback, None) @@ -44,6 +48,10 @@ class PacedCrawler(ShareCrawler): self.finished_d = defer.Deferred() self.yield_cb = None def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32): + if PY3: + # Bucket _inputs_ are bytes, and that's what we will compare this + # to: + storage_index_b32 = storage_index_b32.encode("ascii") self.all_buckets.append(storage_index_b32) self.countdown -= 1 if self.countdown == 0: