mirror of
https://github.com/tahoe-lafs/tahoe-lafs.git
synced 2024-12-23 14:52:26 +00:00
IServer refactoring: pass IServer instances around, instead of peerids
refs #1363 This collapses 88 small incremental changes (each of which passes all tests) into one big patch. The development process for the long path started with adding some temporary scaffolding, changing one method at a time, then removing the scaffolding. The individual pieces are as follows, in reverse chronological order (the first patch is at the end of this comment): commit 9bbe4174fd0d98a6cf47a8ef96e85d9ef34b2f9a Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 16:05:00 2011 -0400 immutable/downloader/status.py: correct comment src/allmydata/immutable/downloader/status.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) commit 72146a7c7c91eac2f7c3ceb801eb7a1721376889 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 15:46:20 2011 -0400 remove temporary ServerMap._storage_broker src/allmydata/mutable/checker.py | 2 +- src/allmydata/mutable/filenode.py | 2 +- src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/servermap.py | 5 ++--- src/allmydata/test/test_mutable.py | 8 ++++---- 5 files changed, 9 insertions(+), 10 deletions(-) commit d703096b41632c47d76414b12672e076a422ff5c Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 15:37:05 2011 -0400 remove temporary storage_broker.get_server_for_id() src/allmydata/storage_client.py | 3 --- src/allmydata/test/no_network.py | 13 ------------- 2 files changed, 0 insertions(+), 16 deletions(-) commit 620cc5d80882ef6f7decfd26af8a6c7c1ddf80d1 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 12:50:06 2011 -0400 API of Retrieve._try_to_validate_privkey(), trying to remove reader.server src/allmydata/mutable/retrieve.py | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) commit 92f43f856f4a8b36c207d1b190ed8699b5a4ecb4 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 12:48:08 2011 -0400 API of Retrieve._validate_block(), trying to remove reader.server src/allmydata/mutable/retrieve.py | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) commit 572d5070761861a2190349d1ed8d85dbc25698a5 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 12:36:58 2011 -0400 API of Retrieve._mark_bad_share(), trying to remove reader.server src/allmydata/mutable/retrieve.py | 21 +++++++++------------ 1 files changed, 9 insertions(+), 12 deletions(-) commit a793ff00c0de1e2eec7b46288fdf388c7a2bec89 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 12:06:13 2011 -0400 remove now-unused get_rref_for_serverid() src/allmydata/mutable/servermap.py | 3 --- 1 files changed, 0 insertions(+), 3 deletions(-) commit 1b9827cc9366bf90b93297fdd6832f2ad0480ce7 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 12:03:09 2011 -0400 Retrieve: stop adding .serverid attributes to readers src/allmydata/mutable/retrieve.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) commit 5d4e9d491b19e49d2e443a1dfff2c672842c36ef Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 12:03:34 2011 -0400 return value of Retrieve(verify=True) src/allmydata/mutable/checker.py | 11 ++++++----- src/allmydata/mutable/retrieve.py | 3 +-- 2 files changed, 7 insertions(+), 7 deletions(-) commit e9ab7978c384e1f677cb7779dc449b1044face82 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 11:54:23 2011 -0400 Retrieve._bad_shares (but not return value, used by Verifier) src/allmydata/mutable/retrieve.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) commit 2d91926de233ec5c881f30e36b4a30ad92ab42a9 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 11:51:23 2011 -0400 Publish: stop adding .serverid attributes to writers src/allmydata/mutable/publish.py | 9 ++------- 1 files changed, 2 insertions(+), 7 deletions(-) commit 47c7a0105dec7cbf4f7e0a3ce800bbb85b15df4a Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 11:56:33 2011 -0400 API of get_write_enabler() src/allmydata/mutable/filenode.py | 7 ++++--- src/allmydata/mutable/publish.py | 4 ++-- src/allmydata/test/no_network.py | 3 +++ 3 files changed, 9 insertions(+), 5 deletions(-) commit 9196a5c6590fdbfd660325ea8358b345887d3db0 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 11:46:24 2011 -0400 API of get_(renewal|cancel)_secret() src/allmydata/mutable/filenode.py | 14 ++++++++------ src/allmydata/mutable/publish.py | 8 ++++---- src/allmydata/mutable/servermap.py | 5 ++--- 3 files changed, 14 insertions(+), 13 deletions(-) commit de7c1552f8c163eff5b6d820b5fb3b21c1b47cb5 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 11:41:52 2011 -0400 API of CorruptShareError. Also comment out some related+unused test_web.py code src/allmydata/mutable/common.py | 13 +++++-------- src/allmydata/mutable/retrieve.py | 10 +++++----- src/allmydata/mutable/servermap.py | 8 +++----- src/allmydata/test/common.py | 13 ++++++++----- 4 files changed, 21 insertions(+), 23 deletions(-) commit 2c1c314046b620c16f1e66d030c150d768b7d01e Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 12:01:46 2011 -0400 API of ServerMap.mark_bad_share() src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/retrieve.py | 6 +++--- src/allmydata/mutable/servermap.py | 6 ++---- src/allmydata/test/test_mutable.py | 3 +-- 4 files changed, 7 insertions(+), 10 deletions(-) commit 1bed349030779fd0c378ae4e821384f953c6f6ff Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 11:11:17 2011 -0400 API+name of ServerMap.shares_on_server() : only for tests, so debug_ prefix src/allmydata/mutable/servermap.py | 7 ++----- src/allmydata/test/test_mutable.py | 6 +++--- 2 files changed, 5 insertions(+), 8 deletions(-) commit 2d32e448677d6b818692e801045d4115b29abf21 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 11:07:10 2011 -0400 API of ServerMap.all_servers_for_version() src/allmydata/mutable/servermap.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) commit 48f3204d1889c3e7179578125c4bdef515af3d6a Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 11:04:50 2011 -0400 internals of ServerMap methods that use make_versionmap(), remove temp copy src/allmydata/mutable/servermap.py | 28 +++++++++---------------- 1 files changed, 10 insertions(+), 18 deletions(-) commit 5c3da77b6c777a145bd5ddfaa4db849dc9495548 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 11:01:28 2011 -0400 API of ServerMap.make_versionmap() src/allmydata/mutable/checker.py | 4 ++-- src/allmydata/mutable/retrieve.py | 5 ++--- src/allmydata/mutable/servermap.py | 4 ++-- src/allmydata/test/test_mutable.py | 7 ++++--- 4 files changed, 10 insertions(+), 10 deletions(-) commit b6882ece49afb4c507d118af2db346fa329209dc Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 10:53:38 2011 -0400 make a copy of ServerMap.make_versionmap() (_make_versionmap2) for internal use src/allmydata/mutable/servermap.py | 18 +++++++++++++----- 1 files changed, 13 insertions(+), 5 deletions(-) commit 963f8e63faf32b950eb1b8103cd2ff16fe8f0151 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:45:58 2011 -0400 API of RetrieveStatus.add_problem() src/allmydata/mutable/retrieve.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit 4976d29ffae565a048851601c29013bbae2976d8 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:45:05 2011 -0400 API of RetrieveStatus.add_fetch_timing() src/allmydata/mutable/retrieve.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit d057d3bbba72663ee148a8b916bc2d52be2e3982 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:44:04 2011 -0400 API of Retrieve.notify_server_corruption() src/allmydata/mutable/retrieve.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit 8a2a81e46671c860610e0e96d6add1a57551f22d Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:42:32 2011 -0400 remove unused _outstanding_queries src/allmydata/mutable/retrieve.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) commit 56d12cc9968d03ccd53764455c671122c4f391d1 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:40:57 2011 -0400 change Retrieve.remaining_sharemap src/allmydata/mutable/retrieve.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) commit 4f0b7af4821f43290bfc70f2b1fc30149ad81281 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 10:40:18 2011 -0400 accessor for PublishStatus._problems src/allmydata/mutable/publish.py | 4 +++- src/allmydata/web/status.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) commit 627087cf66d0b8cc519f4d551a967a7bd9b6a741 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 10:36:39 2011 -0400 accessor for RetrieveStatus._problems src/allmydata/mutable/retrieve.py | 8 ++++++-- src/allmydata/web/status.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) commit ca7dea81f03801b1c7353fc00ecba689268109cf Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:35:32 2011 -0400 add .server to "reader", so we can get at it later src/allmydata/mutable/retrieve.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit 6ef516e24908ec195af084a7550d1921a5e983b0 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:32:32 2011 -0400 temporarily give Retrieve a _storage_broker, so it can map serverids to servers src/allmydata/mutable/checker.py | 3 ++- src/allmydata/mutable/filenode.py | 6 ++++-- src/allmydata/mutable/retrieve.py | 5 +++-- src/allmydata/test/test_mutable.py | 4 ++-- 4 files changed, 11 insertions(+), 7 deletions(-) commit afe08e4dd3f4ff9ff7e8a2a8d28b181e3625bcc9 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:21:51 2011 -0400 mutable/retrieve.py: s/peer/server/ src/allmydata/mutable/retrieve.py | 82 +++++++++++++------------- src/allmydata/test/test_mutable.py | 6 +- 2 files changed, 44 insertions(+), 44 deletions(-) commit 910afcb5d7f274880f68dd6cdb5b05f2bbc29adc Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:16:01 2011 -0400 web.status.PublishStatusPage: add comment, I think .problems isn't exercised src/allmydata/web/status.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) commit 311466dd8c931bbba40d590ade867704282e7f1a Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:48:16 2011 -0400 API of PublishStatus.add_per_server_time() src/allmydata/mutable/publish.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit 2df5faa1b6cbfbaded520d2320305a62fe961118 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:46:37 2011 -0400 more simplifications src/allmydata/mutable/publish.py | 4 +--- 1 files changed, 1 insertions(+), 3 deletions(-) commit 6ac4544a3da385f2aad9392f906b90192f4f919a Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:44:08 2011 -0400 API of ServerMap.version_on_server() src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/servermap.py | 4 ++-- src/allmydata/test/test_mutable.py | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) commit 3e187e322511072e4683329df6b2c6c733a66dba Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:16:32 2011 -0400 API of ServerMap.make_sharemap() src/allmydata/mutable/servermap.py | 4 ++-- src/allmydata/test/test_mutable.py | 7 ++++--- src/allmydata/web/status.py | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) commit 318feed8437bdd8d4943c6569d38f7b54b6313cc Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:36:19 2011 -0400 small cleanups src/allmydata/mutable/publish.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) commit bd459ed5714e1db5a7163935c54b7b0b56db8349 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:33:39 2011 -0400 API of ServerMap.add_new_share() src/allmydata/mutable/publish.py | 4 ++-- src/allmydata/mutable/servermap.py | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) commit f2804fb6ed11d80088e0da8ed48e6c2922f2ffef Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:30:26 2011 -0400 API of ServerMap.get_bad_shares() src/allmydata/mutable/publish.py | 3 +-- src/allmydata/mutable/servermap.py | 9 ++++----- 2 files changed, 5 insertions(+), 7 deletions(-) commit 965074a47b3ce1431cb46d9a233840afcf9105f5 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:26:58 2011 -0400 more small cleanups src/allmydata/mutable/publish.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit 38020da34f034f8889947dd3dc05e087ffff7106 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:18:47 2011 -0400 change Publish.bad_share_checkstrings src/allmydata/mutable/publish.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit 5efebcbd2ee0c2f299ea86f7591d856c0f265304 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:16:31 2011 -0400 change internals of Publish.update_goal() src/allmydata/mutable/publish.py | 8 +++----- 1 files changed, 3 insertions(+), 5 deletions(-) commit e91b55ff4c2a69165b71f2c7b217ac319ff4c527 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:11:42 2011 -0400 get rid of Publish.connections src/allmydata/mutable/publish.py | 27 +++++---------------------- 1 files changed, 5 insertions(+), 22 deletions(-) commit 64e9a53b3229ebe2f9ebf7ed502d539311d0e037 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:05:32 2011 -0400 change Publish.bad_servers src/allmydata/mutable/publish.py | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) commit b85a934bef315a06bcfe00c9c12a3627fed2b918 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:03:07 2011 -0400 Publish.bad_servers: fix bug, this should be a set of serverids, not writers src/allmydata/mutable/publish.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) commit 605ea15ec15ed671513819003ccd211cdb9761e0 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:00:21 2011 -0400 change .placed src/allmydata/mutable/publish.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit f7aba37b1b345d5b6d5cb16e3b3f6f3c1afb658e Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 22:59:22 2011 -0400 temporarily stash IServer as .server on the "writer" object src/allmydata/mutable/publish.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) commit f9b551d788e7db1f187fce5ab98ab5d5fe4e1c36 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 22:48:18 2011 -0400 change Publish.goal and API of log_goal() to use IServer, not serverid src/allmydata/mutable/publish.py | 48 ++++++++++++++-------------- 1 files changed, 24 insertions(+), 24 deletions(-) commit 75f20616558e4900b8b1f685dd99aa838de6d452 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 15:27:02 2011 -0400 API of ServerMap.get_known_shares() src/allmydata/mutable/publish.py | 16 ++++++++++------ src/allmydata/mutable/servermap.py | 7 ++----- 2 files changed, 12 insertions(+), 11 deletions(-) commit 1c38c9d37bb08221b4418762234b1a62397b3b4b Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 15:20:29 2011 -0400 Publish.full_serverlist src/allmydata/mutable/publish.py | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) commit b6cbd215a04b9cde31a7d92a97a7f048622b16f1 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 15:12:31 2011 -0400 API of ServerMap.all_servers() src/allmydata/mutable/servermap.py | 19 ++++++------------- 1 files changed, 6 insertions(+), 13 deletions(-) commit e63cd0315fae65357b1727ec6d5ff3c6e0d27c98 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 15:10:18 2011 -0400 remove ServerMap.connections, set_rref_for_serverid() src/allmydata/mutable/servermap.py | 11 +---------- 1 files changed, 1 insertions(+), 10 deletions(-) commit 4df52db2f80eb12eefa5d57103c24893cde89553 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 15:04:06 2011 -0400 API of ServerMap.mark_server_reachable() src/allmydata/mutable/servermap.py | 7 ++----- 1 files changed, 2 insertions(+), 5 deletions(-) commit 69c715bde77944dc25181b3dbbeb042c816f9a1b Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 15:03:21 2011 -0400 API of ServerMap.mark_server_unreachable() src/allmydata/mutable/servermap.py | 9 +++------ 1 files changed, 3 insertions(+), 6 deletions(-) commit 3d784d60eec1c508858e3a617e4411ffbcc3c1fa Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 15:02:03 2011 -0400 API of status.set_privkey_from() src/allmydata/mutable/servermap.py | 7 +++---- 1 files changed, 3 insertions(+), 4 deletions(-) commit 544ed3ea29bed7e66da7fd29ca3f6f076f27a9e6 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 15:01:15 2011 -0400 API of status.add_per_server_time() src/allmydata/mutable/servermap.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) commit fffe5008b6320bd1e04c3c68389a2bf2ee383fa8 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:59:02 2011 -0400 remove unused .versionmap src/allmydata/mutable/servermap.py | 7 ------- 1 files changed, 0 insertions(+), 7 deletions(-) commit 2816562e090d2294179db3588dafcca18de1bc2b Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:57:51 2011 -0400 remove serverid from all log messages. Also one unused lambda. src/allmydata/mutable/servermap.py | 30 +++++++++++++------------- 1 files changed, 15 insertions(+), 15 deletions(-) commit 28fa6b1a2738fa98c1f1dbd3d0e01ae98912d11f Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:54:30 2011 -0400 removed unused _readers src/allmydata/mutable/servermap.py | 3 --- 1 files changed, 0 insertions(+), 3 deletions(-) commit a8e4ed3d645ab592d1add6a1e69b6d1ebfb77817 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:54:16 2011 -0400 remove unused _sharemap src/allmydata/mutable/servermap.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) commit 3f072e55cf1d0700f9fffe23f8f3a475725df588 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:49:03 2011 -0400 _must_query src/allmydata/mutable/servermap.py | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) commit c599a059b8df3f5785e4bf89fb6ecc6d8dcd708b Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:48:05 2011 -0400 _queries_outstanding src/allmydata/mutable/servermap.py | 16 +++++++--------- 1 files changed, 7 insertions(+), 9 deletions(-) commit 7743759f98ac2c07926b2fdbd80bf52dfab33085 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:46:17 2011 -0400 _empty_servers src/allmydata/mutable/servermap.py | 5 ++--- 1 files changed, 2 insertions(+), 3 deletions(-) commit 6bb1825916828a713a32cdf7f7411fa3ea2e1e5d Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:45:39 2011 -0400 _good_servers src/allmydata/mutable/servermap.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) commit 1768fab1b51d8dd93ecabbaaabfadfa20cf6c3d4 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:44:59 2011 -0400 _bad_servers src/allmydata/mutable/servermap.py | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) commit dccbaef30f0ba714c746bf6d4a1a803c36e17b65 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:41:54 2011 -0400 API of _try_to_set_pubkey() src/allmydata/mutable/servermap.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) commit 0481ea70042ba3575f15eac7fd0780f8ece580cc Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:35:02 2011 -0400 API of notify_server_corruption() src/allmydata/mutable/servermap.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit bea9cba18fb3b9c11bb22f18356a263ecec7351e Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:34:09 2011 -0400 API of _got_signature_one_share() src/allmydata/mutable/servermap.py | 9 +++++---- 1 files changed, 5 insertions(+), 4 deletions(-) commit 1520123583cf78650706e114b15bb5b0ac1f4a14 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:32:33 2011 -0400 API of _try_to_validate_privkey() src/allmydata/mutable/servermap.py | 9 +++++---- 1 files changed, 5 insertions(+), 4 deletions(-) commit 938852c9c8519c7a078f58a9b1f4dd8ec8b6715e Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:31:48 2011 -0400 API and internals of _add_lease_failed() src/allmydata/mutable/servermap.py | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) commit 3843dba367e3c19e176a622ab853cb51d2472ddf Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:30:37 2011 -0400 API of _privkey_query_failed() src/allmydata/mutable/servermap.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit 2219a710e1633cd57d0ca0786490de87b3e19ba7 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:29:43 2011 -0400 fix bug in call to _privkey_query_failed, unrelated to refactoring src/allmydata/mutable/servermap.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) commit ae615bec7d0d1b269710b6902797b12f9592ad62 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:27:17 2011 -0400 API of _got_corrupt_share() src/allmydata/mutable/servermap.py | 17 +++++++++-------- 1 files changed, 9 insertions(+), 8 deletions(-) commit cb51c95a6f4e077278157a77dab060c8c1ad7a81 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:23:16 2011 -0400 API of _got_results() src/allmydata/mutable/servermap.py | 9 +++++---- 1 files changed, 5 insertions(+), 4 deletions(-) commit bac9154fe0af18f226999a58ffc2362d8cf4b802 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:19:19 2011 -0400 API of _query_failed() src/allmydata/mutable/servermap.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit fdc29a8ca95d4b5c503e5382b9e5d4d02141ba12 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:17:20 2011 -0400 API of _do_read() src/allmydata/mutable/servermap.py | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) commit e7e9e338f28d004aa4d423d11c65f1e271ac7322 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:20:21 2011 -0400 API of _do_query() src/allmydata/mutable/servermap.py | 15 +++++++-------- 1 files changed, 7 insertions(+), 8 deletions(-) commit 330625b9dac4cdbe72a11464a893065b9aeed453 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:43:05 2011 -0400 next step: first batch of updates to ServermapUpdater updates: most method-local variables in update() API of _build_initial_querylist() API of _send_initial_requests() .full_serverlist .extra_servers src/allmydata/mutable/servermap.py | 39 ++++++++++++++------------ 1 files changed, 21 insertions(+), 18 deletions(-) commit 4aadc584fa7dcb2daa86b048c81dee0049ba26d9 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 15:07:00 2011 -0400 internal change: index _bad_shares with IServer src/allmydata/mutable/servermap.py | 20 ++++++++++---------- 1 files changed, 10 insertions(+), 10 deletions(-) commit 16d4e6fa82a9907dbdc92094213387c6a4164e41 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 18:20:47 2011 +0100 internal change: index _known_shares with IServer instead of serverid callers are unchanged src/allmydata/mutable/servermap.py | 42 +++++++++++++++---------- 1 files changed, 25 insertions(+), 17 deletions(-) commit ceeb5f4938cc814a0c75d1b8f4018aed965c2176 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 18:11:43 2011 +0100 accessors and name cleanup for servermap.Servermap.last_update_mode/time src/allmydata/mutable/filenode.py | 6 +++--- src/allmydata/mutable/publish.py | 4 ++-- src/allmydata/mutable/servermap.py | 17 +++++++++++------ 3 files changed, 16 insertions(+), 11 deletions(-) commit 8d3cbda82661c0a7e5c3d3b65cf7a5d5ab7e32c0 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 18:11:14 2011 +0100 accessors and name cleanup for servermap.Servermap.problems src/allmydata/mutable/servermap.py | 21 +++++++++++++-------- src/allmydata/test/test_mutable.py | 6 +++--- 2 files changed, 16 insertions(+), 11 deletions(-) commit 348f57988f79389db0aab7672e6eaa9a6d8e3219 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 18:10:41 2011 +0100 accessors and name cleanup for servermap.Servermap.bad_shares src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/servermap.py | 30 ++++++++++++++----------- 2 files changed, 18 insertions(+), 14 deletions(-) commit 520c9368134673cdf76c653c5e1bb91c2ab5d51e Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 18:10:05 2011 +0100 accessors and name cleanup for servermap.Servermap.servermap . src/allmydata/mutable/publish.py | 14 +++++---- src/allmydata/mutable/servermap.py | 38 ++++++++++++++----------- 2 files changed, 29 insertions(+), 23 deletions(-) commit b8b8dc38287a91dbdf494426ac801d9381ce5841 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 18:08:02 2011 +0100 fix reachable_servers src/allmydata/mutable/checker.py | 3 ++- src/allmydata/mutable/publish.py | 4 +++- src/allmydata/mutable/servermap.py | 12 ++++++++++-- 3 files changed, 15 insertions(+), 4 deletions(-) commit cb0cfd1adfefad357c187aaaf690c3df68b622bc Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 18:06:03 2011 +0100 fix Servermap.unreachable_servers src/allmydata/mutable/servermap.py | 11 ++++++++--- 1 files changed, 8 insertions(+), 3 deletions(-) commit 2d9ea79b94bd4db674d40386fda90825785ac495 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 18:03:48 2011 +0100 give ServerMap a StorageFarmBroker, temporary this makes it possible for the ServerMap to accept bare serverids and still build data structures with IServers src/allmydata/mutable/checker.py | 2 +- src/allmydata/mutable/filenode.py | 2 +- src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/servermap.py | 5 +++-- src/allmydata/test/test_mutable.py | 8 ++++---- 5 files changed, 10 insertions(+), 9 deletions(-) commit 718d1aeff6fded893f65397806d22ece928b0dd4 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 13:43:30 2011 -0400 add StorageFarmBroker.get_server_for_id(), temporary helper This will go away once we're passing IServers everywhere. src/allmydata/storage_client.py | 2 ++ src/allmydata/test/no_network.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 0 deletions(-) commit ece20231d7fda0d503704842a4aa068dfbc2e54e Author: Brian Warner <warner@lothar.com> Date: Sun Oct 2 01:11:50 2011 +0100 add proper accessors for Servermap.connections, to make refactoring easier src/allmydata/mutable/publish.py | 6 +++--- src/allmydata/mutable/retrieve.py | 10 +++++----- src/allmydata/mutable/servermap.py | 17 +++++++++++------ 3 files changed, 19 insertions(+), 14 deletions(-) commit 3b943d6bf302ff702668081a612fc4fe2604cf9c Author: Brian Warner <warner@lothar.com> Date: Fri Sep 23 10:34:30 2011 -0700 mutable/servermap.py and neighbors: s/peer/server/ src/allmydata/mutable/checker.py | 22 +- src/allmydata/mutable/publish.py | 204 +++++++------- src/allmydata/mutable/servermap.py | 402 +++++++++++++------------- src/allmydata/test/test_mutable.py | 18 +- 4 files changed, 323 insertions(+), 323 deletions(-) IServer refactoring: pass IServer instances around, instead of peerids refs #1363 This collapses 88 small incremental changes (each of which passes all tests) into one big patch. The development process for the long path started with adding some temporary scaffolding, changing one method at a time, then removing the scaffolding. The individual pieces are as follows, in reverse chronological order (the first patch is at the end of this comment): commit 9bbe4174fd0d98a6cf47a8ef96e85d9ef34b2f9a Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 16:05:00 2011 -0400 immutable/downloader/status.py: correct comment src/allmydata/immutable/downloader/status.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) commit 72146a7c7c91eac2f7c3ceb801eb7a1721376889 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 15:46:20 2011 -0400 remove temporary ServerMap._storage_broker src/allmydata/mutable/checker.py | 2 +- src/allmydata/mutable/filenode.py | 2 +- src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/servermap.py | 5 ++--- src/allmydata/test/test_mutable.py | 8 ++++---- 5 files changed, 9 insertions(+), 10 deletions(-) commit d703096b41632c47d76414b12672e076a422ff5c Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 15:37:05 2011 -0400 remove temporary storage_broker.get_server_for_id() src/allmydata/storage_client.py | 3 --- src/allmydata/test/no_network.py | 13 ------------- 2 files changed, 0 insertions(+), 16 deletions(-) commit 620cc5d80882ef6f7decfd26af8a6c7c1ddf80d1 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 12:50:06 2011 -0400 API of Retrieve._try_to_validate_privkey(), trying to remove reader.server src/allmydata/mutable/retrieve.py | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) commit 92f43f856f4a8b36c207d1b190ed8699b5a4ecb4 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 12:48:08 2011 -0400 API of Retrieve._validate_block(), trying to remove reader.server src/allmydata/mutable/retrieve.py | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) commit 572d5070761861a2190349d1ed8d85dbc25698a5 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 12:36:58 2011 -0400 API of Retrieve._mark_bad_share(), trying to remove reader.server src/allmydata/mutable/retrieve.py | 21 +++++++++------------ 1 files changed, 9 insertions(+), 12 deletions(-) commit a793ff00c0de1e2eec7b46288fdf388c7a2bec89 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 12:06:13 2011 -0400 remove now-unused get_rref_for_serverid() src/allmydata/mutable/servermap.py | 3 --- 1 files changed, 0 insertions(+), 3 deletions(-) commit 1b9827cc9366bf90b93297fdd6832f2ad0480ce7 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 12:03:09 2011 -0400 Retrieve: stop adding .serverid attributes to readers src/allmydata/mutable/retrieve.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) commit 5d4e9d491b19e49d2e443a1dfff2c672842c36ef Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 12:03:34 2011 -0400 return value of Retrieve(verify=True) src/allmydata/mutable/checker.py | 11 ++++++----- src/allmydata/mutable/retrieve.py | 3 +-- 2 files changed, 7 insertions(+), 7 deletions(-) commit e9ab7978c384e1f677cb7779dc449b1044face82 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 11:54:23 2011 -0400 Retrieve._bad_shares (but not return value, used by Verifier) src/allmydata/mutable/retrieve.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) commit 2d91926de233ec5c881f30e36b4a30ad92ab42a9 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 11:51:23 2011 -0400 Publish: stop adding .serverid attributes to writers src/allmydata/mutable/publish.py | 9 ++------- 1 files changed, 2 insertions(+), 7 deletions(-) commit 47c7a0105dec7cbf4f7e0a3ce800bbb85b15df4a Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 11:56:33 2011 -0400 API of get_write_enabler() src/allmydata/mutable/filenode.py | 7 ++++--- src/allmydata/mutable/publish.py | 4 ++-- src/allmydata/test/no_network.py | 3 +++ 3 files changed, 9 insertions(+), 5 deletions(-) commit 9196a5c6590fdbfd660325ea8358b345887d3db0 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 11:46:24 2011 -0400 API of get_(renewal|cancel)_secret() src/allmydata/mutable/filenode.py | 14 ++++++++------ src/allmydata/mutable/publish.py | 8 ++++---- src/allmydata/mutable/servermap.py | 5 ++--- 3 files changed, 14 insertions(+), 13 deletions(-) commit de7c1552f8c163eff5b6d820b5fb3b21c1b47cb5 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 11:41:52 2011 -0400 API of CorruptShareError. Also comment out some related+unused test_web.py code src/allmydata/mutable/common.py | 13 +++++-------- src/allmydata/mutable/retrieve.py | 10 +++++----- src/allmydata/mutable/servermap.py | 8 +++----- src/allmydata/test/common.py | 13 ++++++++----- 4 files changed, 21 insertions(+), 23 deletions(-) commit 2c1c314046b620c16f1e66d030c150d768b7d01e Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 12:01:46 2011 -0400 API of ServerMap.mark_bad_share() src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/retrieve.py | 6 +++--- src/allmydata/mutable/servermap.py | 6 ++---- src/allmydata/test/test_mutable.py | 3 +-- 4 files changed, 7 insertions(+), 10 deletions(-) commit 1bed349030779fd0c378ae4e821384f953c6f6ff Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 11:11:17 2011 -0400 API+name of ServerMap.shares_on_server() : only for tests, so debug_ prefix src/allmydata/mutable/servermap.py | 7 ++----- src/allmydata/test/test_mutable.py | 6 +++--- 2 files changed, 5 insertions(+), 8 deletions(-) commit 2d32e448677d6b818692e801045d4115b29abf21 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 11:07:10 2011 -0400 API of ServerMap.all_servers_for_version() src/allmydata/mutable/servermap.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) commit 48f3204d1889c3e7179578125c4bdef515af3d6a Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 11:04:50 2011 -0400 internals of ServerMap methods that use make_versionmap(), remove temp copy src/allmydata/mutable/servermap.py | 28 +++++++++---------------- 1 files changed, 10 insertions(+), 18 deletions(-) commit 5c3da77b6c777a145bd5ddfaa4db849dc9495548 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 11:01:28 2011 -0400 API of ServerMap.make_versionmap() src/allmydata/mutable/checker.py | 4 ++-- src/allmydata/mutable/retrieve.py | 5 ++--- src/allmydata/mutable/servermap.py | 4 ++-- src/allmydata/test/test_mutable.py | 7 ++++--- 4 files changed, 10 insertions(+), 10 deletions(-) commit b6882ece49afb4c507d118af2db346fa329209dc Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 10:53:38 2011 -0400 make a copy of ServerMap.make_versionmap() (_make_versionmap2) for internal use src/allmydata/mutable/servermap.py | 18 +++++++++++++----- 1 files changed, 13 insertions(+), 5 deletions(-) commit 963f8e63faf32b950eb1b8103cd2ff16fe8f0151 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:45:58 2011 -0400 API of RetrieveStatus.add_problem() src/allmydata/mutable/retrieve.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit 4976d29ffae565a048851601c29013bbae2976d8 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:45:05 2011 -0400 API of RetrieveStatus.add_fetch_timing() src/allmydata/mutable/retrieve.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit d057d3bbba72663ee148a8b916bc2d52be2e3982 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:44:04 2011 -0400 API of Retrieve.notify_server_corruption() src/allmydata/mutable/retrieve.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit 8a2a81e46671c860610e0e96d6add1a57551f22d Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:42:32 2011 -0400 remove unused _outstanding_queries src/allmydata/mutable/retrieve.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) commit 56d12cc9968d03ccd53764455c671122c4f391d1 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:40:57 2011 -0400 change Retrieve.remaining_sharemap src/allmydata/mutable/retrieve.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) commit 4f0b7af4821f43290bfc70f2b1fc30149ad81281 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 10:40:18 2011 -0400 accessor for PublishStatus._problems src/allmydata/mutable/publish.py | 4 +++- src/allmydata/web/status.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) commit 627087cf66d0b8cc519f4d551a967a7bd9b6a741 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 10:36:39 2011 -0400 accessor for RetrieveStatus._problems src/allmydata/mutable/retrieve.py | 8 ++++++-- src/allmydata/web/status.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) commit ca7dea81f03801b1c7353fc00ecba689268109cf Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:35:32 2011 -0400 add .server to "reader", so we can get at it later src/allmydata/mutable/retrieve.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit 6ef516e24908ec195af084a7550d1921a5e983b0 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:32:32 2011 -0400 temporarily give Retrieve a _storage_broker, so it can map serverids to servers src/allmydata/mutable/checker.py | 3 ++- src/allmydata/mutable/filenode.py | 6 ++++-- src/allmydata/mutable/retrieve.py | 5 +++-- src/allmydata/test/test_mutable.py | 4 ++-- 4 files changed, 11 insertions(+), 7 deletions(-) commit afe08e4dd3f4ff9ff7e8a2a8d28b181e3625bcc9 Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:21:51 2011 -0400 mutable/retrieve.py: s/peer/server/ src/allmydata/mutable/retrieve.py | 82 +++++++++++++------------- src/allmydata/test/test_mutable.py | 6 +- 2 files changed, 44 insertions(+), 44 deletions(-) commit 910afcb5d7f274880f68dd6cdb5b05f2bbc29adc Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:16:01 2011 -0400 web.status.PublishStatusPage: add comment, I think .problems isn't exercised src/allmydata/web/status.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) commit 311466dd8c931bbba40d590ade867704282e7f1a Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:48:16 2011 -0400 API of PublishStatus.add_per_server_time() src/allmydata/mutable/publish.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit 2df5faa1b6cbfbaded520d2320305a62fe961118 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:46:37 2011 -0400 more simplifications src/allmydata/mutable/publish.py | 4 +--- 1 files changed, 1 insertions(+), 3 deletions(-) commit 6ac4544a3da385f2aad9392f906b90192f4f919a Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:44:08 2011 -0400 API of ServerMap.version_on_server() src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/servermap.py | 4 ++-- src/allmydata/test/test_mutable.py | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) commit 3e187e322511072e4683329df6b2c6c733a66dba Author: Brian Warner <warner@lothar.com> Date: Tue Oct 4 00:16:32 2011 -0400 API of ServerMap.make_sharemap() src/allmydata/mutable/servermap.py | 4 ++-- src/allmydata/test/test_mutable.py | 7 ++++--- src/allmydata/web/status.py | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) commit 318feed8437bdd8d4943c6569d38f7b54b6313cc Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:36:19 2011 -0400 small cleanups src/allmydata/mutable/publish.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) commit bd459ed5714e1db5a7163935c54b7b0b56db8349 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:33:39 2011 -0400 API of ServerMap.add_new_share() src/allmydata/mutable/publish.py | 4 ++-- src/allmydata/mutable/servermap.py | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) commit f2804fb6ed11d80088e0da8ed48e6c2922f2ffef Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:30:26 2011 -0400 API of ServerMap.get_bad_shares() src/allmydata/mutable/publish.py | 3 +-- src/allmydata/mutable/servermap.py | 9 ++++----- 2 files changed, 5 insertions(+), 7 deletions(-) commit 965074a47b3ce1431cb46d9a233840afcf9105f5 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:26:58 2011 -0400 more small cleanups src/allmydata/mutable/publish.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit 38020da34f034f8889947dd3dc05e087ffff7106 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:18:47 2011 -0400 change Publish.bad_share_checkstrings src/allmydata/mutable/publish.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit 5efebcbd2ee0c2f299ea86f7591d856c0f265304 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:16:31 2011 -0400 change internals of Publish.update_goal() src/allmydata/mutable/publish.py | 8 +++----- 1 files changed, 3 insertions(+), 5 deletions(-) commit e91b55ff4c2a69165b71f2c7b217ac319ff4c527 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:11:42 2011 -0400 get rid of Publish.connections src/allmydata/mutable/publish.py | 27 +++++---------------------- 1 files changed, 5 insertions(+), 22 deletions(-) commit 64e9a53b3229ebe2f9ebf7ed502d539311d0e037 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:05:32 2011 -0400 change Publish.bad_servers src/allmydata/mutable/publish.py | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) commit b85a934bef315a06bcfe00c9c12a3627fed2b918 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:03:07 2011 -0400 Publish.bad_servers: fix bug, this should be a set of serverids, not writers src/allmydata/mutable/publish.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) commit 605ea15ec15ed671513819003ccd211cdb9761e0 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 23:00:21 2011 -0400 change .placed src/allmydata/mutable/publish.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit f7aba37b1b345d5b6d5cb16e3b3f6f3c1afb658e Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 22:59:22 2011 -0400 temporarily stash IServer as .server on the "writer" object src/allmydata/mutable/publish.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) commit f9b551d788e7db1f187fce5ab98ab5d5fe4e1c36 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 22:48:18 2011 -0400 change Publish.goal and API of log_goal() to use IServer, not serverid src/allmydata/mutable/publish.py | 48 ++++++++++++++-------------- 1 files changed, 24 insertions(+), 24 deletions(-) commit 75f20616558e4900b8b1f685dd99aa838de6d452 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 15:27:02 2011 -0400 API of ServerMap.get_known_shares() src/allmydata/mutable/publish.py | 16 ++++++++++------ src/allmydata/mutable/servermap.py | 7 ++----- 2 files changed, 12 insertions(+), 11 deletions(-) commit 1c38c9d37bb08221b4418762234b1a62397b3b4b Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 15:20:29 2011 -0400 Publish.full_serverlist src/allmydata/mutable/publish.py | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) commit b6cbd215a04b9cde31a7d92a97a7f048622b16f1 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 15:12:31 2011 -0400 API of ServerMap.all_servers() src/allmydata/mutable/servermap.py | 19 ++++++------------- 1 files changed, 6 insertions(+), 13 deletions(-) commit e63cd0315fae65357b1727ec6d5ff3c6e0d27c98 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 15:10:18 2011 -0400 remove ServerMap.connections, set_rref_for_serverid() src/allmydata/mutable/servermap.py | 11 +---------- 1 files changed, 1 insertions(+), 10 deletions(-) commit 4df52db2f80eb12eefa5d57103c24893cde89553 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 15:04:06 2011 -0400 API of ServerMap.mark_server_reachable() src/allmydata/mutable/servermap.py | 7 ++----- 1 files changed, 2 insertions(+), 5 deletions(-) commit 69c715bde77944dc25181b3dbbeb042c816f9a1b Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 15:03:21 2011 -0400 API of ServerMap.mark_server_unreachable() src/allmydata/mutable/servermap.py | 9 +++------ 1 files changed, 3 insertions(+), 6 deletions(-) commit 3d784d60eec1c508858e3a617e4411ffbcc3c1fa Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 15:02:03 2011 -0400 API of status.set_privkey_from() src/allmydata/mutable/servermap.py | 7 +++---- 1 files changed, 3 insertions(+), 4 deletions(-) commit 544ed3ea29bed7e66da7fd29ca3f6f076f27a9e6 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 15:01:15 2011 -0400 API of status.add_per_server_time() src/allmydata/mutable/servermap.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) commit fffe5008b6320bd1e04c3c68389a2bf2ee383fa8 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:59:02 2011 -0400 remove unused .versionmap src/allmydata/mutable/servermap.py | 7 ------- 1 files changed, 0 insertions(+), 7 deletions(-) commit 2816562e090d2294179db3588dafcca18de1bc2b Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:57:51 2011 -0400 remove serverid from all log messages. Also one unused lambda. src/allmydata/mutable/servermap.py | 30 +++++++++++++------------- 1 files changed, 15 insertions(+), 15 deletions(-) commit 28fa6b1a2738fa98c1f1dbd3d0e01ae98912d11f Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:54:30 2011 -0400 removed unused _readers src/allmydata/mutable/servermap.py | 3 --- 1 files changed, 0 insertions(+), 3 deletions(-) commit a8e4ed3d645ab592d1add6a1e69b6d1ebfb77817 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:54:16 2011 -0400 remove unused _sharemap src/allmydata/mutable/servermap.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) commit 3f072e55cf1d0700f9fffe23f8f3a475725df588 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:49:03 2011 -0400 _must_query src/allmydata/mutable/servermap.py | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) commit c599a059b8df3f5785e4bf89fb6ecc6d8dcd708b Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:48:05 2011 -0400 _queries_outstanding src/allmydata/mutable/servermap.py | 16 +++++++--------- 1 files changed, 7 insertions(+), 9 deletions(-) commit 7743759f98ac2c07926b2fdbd80bf52dfab33085 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:46:17 2011 -0400 _empty_servers src/allmydata/mutable/servermap.py | 5 ++--- 1 files changed, 2 insertions(+), 3 deletions(-) commit 6bb1825916828a713a32cdf7f7411fa3ea2e1e5d Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:45:39 2011 -0400 _good_servers src/allmydata/mutable/servermap.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) commit 1768fab1b51d8dd93ecabbaaabfadfa20cf6c3d4 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:44:59 2011 -0400 _bad_servers src/allmydata/mutable/servermap.py | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) commit dccbaef30f0ba714c746bf6d4a1a803c36e17b65 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:41:54 2011 -0400 API of _try_to_set_pubkey() src/allmydata/mutable/servermap.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) commit 0481ea70042ba3575f15eac7fd0780f8ece580cc Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:35:02 2011 -0400 API of notify_server_corruption() src/allmydata/mutable/servermap.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) commit bea9cba18fb3b9c11bb22f18356a263ecec7351e Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:34:09 2011 -0400 API of _got_signature_one_share() src/allmydata/mutable/servermap.py | 9 +++++---- 1 files changed, 5 insertions(+), 4 deletions(-) commit 1520123583cf78650706e114b15bb5b0ac1f4a14 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:32:33 2011 -0400 API of _try_to_validate_privkey() src/allmydata/mutable/servermap.py | 9 +++++---- 1 files changed, 5 insertions(+), 4 deletions(-) commit 938852c9c8519c7a078f58a9b1f4dd8ec8b6715e Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:31:48 2011 -0400 API and internals of _add_lease_failed() src/allmydata/mutable/servermap.py | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) commit 3843dba367e3c19e176a622ab853cb51d2472ddf Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:30:37 2011 -0400 API of _privkey_query_failed() src/allmydata/mutable/servermap.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit 2219a710e1633cd57d0ca0786490de87b3e19ba7 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:29:43 2011 -0400 fix bug in call to _privkey_query_failed, unrelated to refactoring src/allmydata/mutable/servermap.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) commit ae615bec7d0d1b269710b6902797b12f9592ad62 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:27:17 2011 -0400 API of _got_corrupt_share() src/allmydata/mutable/servermap.py | 17 +++++++++-------- 1 files changed, 9 insertions(+), 8 deletions(-) commit cb51c95a6f4e077278157a77dab060c8c1ad7a81 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:23:16 2011 -0400 API of _got_results() src/allmydata/mutable/servermap.py | 9 +++++---- 1 files changed, 5 insertions(+), 4 deletions(-) commit bac9154fe0af18f226999a58ffc2362d8cf4b802 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:19:19 2011 -0400 API of _query_failed() src/allmydata/mutable/servermap.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) commit fdc29a8ca95d4b5c503e5382b9e5d4d02141ba12 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:17:20 2011 -0400 API of _do_read() src/allmydata/mutable/servermap.py | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) commit e7e9e338f28d004aa4d423d11c65f1e271ac7322 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:20:21 2011 -0400 API of _do_query() src/allmydata/mutable/servermap.py | 15 +++++++-------- 1 files changed, 7 insertions(+), 8 deletions(-) commit 330625b9dac4cdbe72a11464a893065b9aeed453 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 14:43:05 2011 -0400 next step: first batch of updates to ServermapUpdater updates: most method-local variables in update() API of _build_initial_querylist() API of _send_initial_requests() .full_serverlist .extra_servers src/allmydata/mutable/servermap.py | 39 ++++++++++++++------------ 1 files changed, 21 insertions(+), 18 deletions(-) commit 4aadc584fa7dcb2daa86b048c81dee0049ba26d9 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 15:07:00 2011 -0400 internal change: index _bad_shares with IServer src/allmydata/mutable/servermap.py | 20 ++++++++++---------- 1 files changed, 10 insertions(+), 10 deletions(-) commit 16d4e6fa82a9907dbdc92094213387c6a4164e41 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 18:20:47 2011 +0100 internal change: index _known_shares with IServer instead of serverid callers are unchanged src/allmydata/mutable/servermap.py | 42 +++++++++++++++---------- 1 files changed, 25 insertions(+), 17 deletions(-) commit ceeb5f4938cc814a0c75d1b8f4018aed965c2176 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 18:11:43 2011 +0100 accessors and name cleanup for servermap.Servermap.last_update_mode/time src/allmydata/mutable/filenode.py | 6 +++--- src/allmydata/mutable/publish.py | 4 ++-- src/allmydata/mutable/servermap.py | 17 +++++++++++------ 3 files changed, 16 insertions(+), 11 deletions(-) commit 8d3cbda82661c0a7e5c3d3b65cf7a5d5ab7e32c0 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 18:11:14 2011 +0100 accessors and name cleanup for servermap.Servermap.problems src/allmydata/mutable/servermap.py | 21 +++++++++++++-------- src/allmydata/test/test_mutable.py | 6 +++--- 2 files changed, 16 insertions(+), 11 deletions(-) commit 348f57988f79389db0aab7672e6eaa9a6d8e3219 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 18:10:41 2011 +0100 accessors and name cleanup for servermap.Servermap.bad_shares src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/servermap.py | 30 ++++++++++++++----------- 2 files changed, 18 insertions(+), 14 deletions(-) commit 520c9368134673cdf76c653c5e1bb91c2ab5d51e Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 18:10:05 2011 +0100 accessors and name cleanup for servermap.Servermap.servermap . src/allmydata/mutable/publish.py | 14 +++++---- src/allmydata/mutable/servermap.py | 38 ++++++++++++++----------- 2 files changed, 29 insertions(+), 23 deletions(-) commit b8b8dc38287a91dbdf494426ac801d9381ce5841 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 18:08:02 2011 +0100 fix reachable_servers src/allmydata/mutable/checker.py | 3 ++- src/allmydata/mutable/publish.py | 4 +++- src/allmydata/mutable/servermap.py | 12 ++++++++++-- 3 files changed, 15 insertions(+), 4 deletions(-) commit cb0cfd1adfefad357c187aaaf690c3df68b622bc Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 18:06:03 2011 +0100 fix Servermap.unreachable_servers src/allmydata/mutable/servermap.py | 11 ++++++++--- 1 files changed, 8 insertions(+), 3 deletions(-) commit 2d9ea79b94bd4db674d40386fda90825785ac495 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 18:03:48 2011 +0100 give ServerMap a StorageFarmBroker, temporary this makes it possible for the ServerMap to accept bare serverids and still build data structures with IServers src/allmydata/mutable/checker.py | 2 +- src/allmydata/mutable/filenode.py | 2 +- src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/servermap.py | 5 +++-- src/allmydata/test/test_mutable.py | 8 ++++---- 5 files changed, 10 insertions(+), 9 deletions(-) commit 718d1aeff6fded893f65397806d22ece928b0dd4 Author: Brian Warner <warner@lothar.com> Date: Mon Oct 3 13:43:30 2011 -0400 add StorageFarmBroker.get_server_for_id(), temporary helper This will go away once we're passing IServers everywhere. src/allmydata/storage_client.py | 2 ++ src/allmydata/test/no_network.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 0 deletions(-) commit ece20231d7fda0d503704842a4aa068dfbc2e54e Author: Brian Warner <warner@lothar.com> Date: Sun Oct 2 01:11:50 2011 +0100 add proper accessors for Servermap.connections, to make refactoring easier src/allmydata/mutable/publish.py | 6 +++--- src/allmydata/mutable/retrieve.py | 10 +++++----- src/allmydata/mutable/servermap.py | 17 +++++++++++------ 3 files changed, 19 insertions(+), 14 deletions(-) commit 3b943d6bf302ff702668081a612fc4fe2604cf9c Author: Brian Warner <warner@lothar.com> Date: Fri Sep 23 10:34:30 2011 -0700 mutable/servermap.py and neighbors: s/peer/server/ src/allmydata/mutable/checker.py | 22 +- src/allmydata/mutable/publish.py | 204 +++++++------- src/allmydata/mutable/servermap.py | 402 +++++++++++++------------- src/allmydata/test/test_mutable.py | 18 +- 4 files changed, 323 insertions(+), 323 deletions(-)
This commit is contained in:
Brian Warner
parent
ac49656723
commit
5bf1ffbc87
@ -125,7 +125,7 @@ class DownloadStatus:
|
||||
# response_length (None until success)
|
||||
self.block_requests = []
|
||||
|
||||
self.known_shares = [] # (serverid, shnum)
|
||||
self.known_shares = [] # (server, shnum)
|
||||
self.problems = []
|
||||
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
|
||||
from allmydata.uri import from_string
|
||||
from allmydata.util import base32, idlib, log
|
||||
from allmydata.util import base32, log
|
||||
from allmydata.check_results import CheckAndRepairResults, CheckResults
|
||||
|
||||
from allmydata.mutable.common import MODE_CHECK, CorruptShareError
|
||||
@ -14,7 +14,7 @@ class MutableChecker:
|
||||
self._storage_broker = storage_broker
|
||||
self._history = history
|
||||
self._monitor = monitor
|
||||
self.bad_shares = [] # list of (nodeid,shnum,failure)
|
||||
self.bad_shares = [] # list of (server,shnum,failure)
|
||||
self._storage_index = self._node.get_storage_index()
|
||||
self.results = CheckResults(from_string(node.get_uri()), self._storage_index)
|
||||
self.need_repair = False
|
||||
@ -91,7 +91,8 @@ class MutableChecker:
|
||||
if not self.best_version:
|
||||
return
|
||||
|
||||
r = Retrieve(self._node, servermap, self.best_version, verify=True)
|
||||
r = Retrieve(self._node, self._storage_broker, servermap,
|
||||
self.best_version, verify=True)
|
||||
d = r.download()
|
||||
d.addCallback(self._process_bad_shares)
|
||||
return d
|
||||
@ -110,7 +111,7 @@ class MutableChecker:
|
||||
counters["count-shares-good"] = num_distinct_shares
|
||||
counters["count-shares-needed"] = k
|
||||
counters["count-shares-expected"] = N
|
||||
good_hosts = smap.all_peers_for_version(version)
|
||||
good_hosts = smap.all_servers_for_version(version)
|
||||
counters["count-good-share-hosts"] = len(good_hosts)
|
||||
vmap = smap.make_versionmap()
|
||||
counters["count-wrong-shares"] = sum([len(shares)
|
||||
@ -170,7 +171,7 @@ class MutableChecker:
|
||||
report.append("Unhealthy: best version has only %d shares "
|
||||
"(encoding is %d-of-%d)" % (s, k, N))
|
||||
summary.append("%d shares (enc %d-of-%d)" % (s, k, N))
|
||||
hosts = smap.all_peers_for_version(best_version)
|
||||
hosts = smap.all_servers_for_version(best_version)
|
||||
needs_rebalancing = bool( len(hosts) < N )
|
||||
elif unrecoverable:
|
||||
healthy = False
|
||||
@ -193,21 +194,22 @@ class MutableChecker:
|
||||
data["list-corrupt-shares"] = locators = []
|
||||
report.append("Corrupt Shares:")
|
||||
summary.append("Corrupt Shares:")
|
||||
for (peerid, shnum, f) in sorted(self.bad_shares):
|
||||
locators.append( (peerid, self._storage_index, shnum) )
|
||||
s = "%s-sh%d" % (idlib.shortnodeid_b2a(peerid), shnum)
|
||||
for (server, shnum, f) in sorted(self.bad_shares):
|
||||
serverid = server.get_serverid()
|
||||
locators.append( (serverid, self._storage_index, shnum) )
|
||||
s = "%s-sh%d" % (server.get_name(), shnum)
|
||||
if f.check(CorruptShareError):
|
||||
ft = f.value.reason
|
||||
else:
|
||||
ft = str(f)
|
||||
report.append(" %s: %s" % (s, ft))
|
||||
summary.append(s)
|
||||
p = (peerid, self._storage_index, shnum, f)
|
||||
p = (serverid, self._storage_index, shnum, f)
|
||||
r.problems.append(p)
|
||||
msg = ("CorruptShareError during mutable verify, "
|
||||
"peerid=%(peerid)s, si=%(si)s, shnum=%(shnum)d, "
|
||||
"serverid=%(serverid)s, si=%(si)s, shnum=%(shnum)d, "
|
||||
"where=%(where)s")
|
||||
log.msg(format=msg, peerid=idlib.nodeid_b2a(peerid),
|
||||
log.msg(format=msg, serverid=server.get_name(),
|
||||
si=base32.b2a(self._storage_index),
|
||||
shnum=shnum,
|
||||
where=ft,
|
||||
@ -218,13 +220,14 @@ class MutableChecker:
|
||||
|
||||
sharemap = {}
|
||||
for verinfo in vmap:
|
||||
for (shnum, peerid, timestamp) in vmap[verinfo]:
|
||||
for (shnum, server, timestamp) in vmap[verinfo]:
|
||||
shareid = "%s-sh%d" % (smap.summarize_version(verinfo), shnum)
|
||||
if shareid not in sharemap:
|
||||
sharemap[shareid] = []
|
||||
sharemap[shareid].append(peerid)
|
||||
sharemap[shareid].append(server.get_serverid())
|
||||
data["sharemap"] = sharemap
|
||||
data["servers-responding"] = list(smap.reachable_peers)
|
||||
data["servers-responding"] = [s.get_serverid() for s in
|
||||
list(smap.get_reachable_servers())]
|
||||
|
||||
r.set_healthy(healthy)
|
||||
r.set_recoverable(bool(recoverable))
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
|
||||
from allmydata.util import idlib
|
||||
from allmydata.util.spans import DataSpans
|
||||
|
||||
MODE_CHECK = "MODE_CHECK" # query all peers
|
||||
@ -42,16 +41,14 @@ class NotEnoughServersError(Exception):
|
||||
self.first_error = first_error
|
||||
|
||||
class CorruptShareError(Exception):
|
||||
def __init__(self, peerid, shnum, reason):
|
||||
self.args = (peerid, shnum, reason)
|
||||
self.peerid = peerid
|
||||
def __init__(self, server, shnum, reason):
|
||||
self.args = (server, shnum, reason)
|
||||
self.server = server
|
||||
self.shnum = shnum
|
||||
self.reason = reason
|
||||
def __str__(self):
|
||||
short_peerid = idlib.nodeid_b2a(self.peerid)[:8]
|
||||
return "<CorruptShareError peerid=%s shnum[%d]: %s" % (short_peerid,
|
||||
self.shnum,
|
||||
self.reason)
|
||||
return "<CorruptShareError server=%s shnum[%d]: %s" % \
|
||||
(self.server.get_name(), self.shnum, self.reason)
|
||||
|
||||
class UnknownVersionError(Exception):
|
||||
"""The share we received was of a version we don't recognize."""
|
||||
|
||||
@ -185,19 +185,22 @@ class MutableFileNode:
|
||||
def _read_from_cache(self, verinfo, shnum, offset, length):
|
||||
return self._cache.read(verinfo, shnum, offset, length)
|
||||
|
||||
def get_write_enabler(self, peerid):
|
||||
assert len(peerid) == 20
|
||||
return hashutil.ssk_write_enabler_hash(self._writekey, peerid)
|
||||
def get_renewal_secret(self, peerid):
|
||||
assert len(peerid) == 20
|
||||
def get_write_enabler(self, server):
|
||||
seed = server.get_foolscap_write_enabler_seed()
|
||||
assert len(seed) == 20
|
||||
return hashutil.ssk_write_enabler_hash(self._writekey, seed)
|
||||
def get_renewal_secret(self, server):
|
||||
crs = self._secret_holder.get_renewal_secret()
|
||||
frs = hashutil.file_renewal_secret_hash(crs, self._storage_index)
|
||||
return hashutil.bucket_renewal_secret_hash(frs, peerid)
|
||||
def get_cancel_secret(self, peerid):
|
||||
assert len(peerid) == 20
|
||||
lease_seed = server.get_lease_seed()
|
||||
assert len(lease_seed) == 20
|
||||
return hashutil.bucket_renewal_secret_hash(frs, lease_seed)
|
||||
def get_cancel_secret(self, server):
|
||||
ccs = self._secret_holder.get_cancel_secret()
|
||||
fcs = hashutil.file_cancel_secret_hash(ccs, self._storage_index)
|
||||
return hashutil.bucket_cancel_secret_hash(fcs, peerid)
|
||||
lease_seed = server.get_lease_seed()
|
||||
assert len(lease_seed) == 20
|
||||
return hashutil.bucket_cancel_secret_hash(fcs, lease_seed)
|
||||
|
||||
def get_writekey(self):
|
||||
return self._writekey
|
||||
@ -387,7 +390,7 @@ class MutableFileNode:
|
||||
recoverable version that I can find in there.
|
||||
"""
|
||||
# XXX: wording ^^^^
|
||||
if servermap and servermap.last_update_mode == mode:
|
||||
if servermap and servermap.get_last_update()[0] == mode:
|
||||
d = defer.succeed(servermap)
|
||||
else:
|
||||
d = self._get_servermap(mode)
|
||||
@ -780,7 +783,7 @@ class MutableFileVersion:
|
||||
|
||||
def _overwrite(self, new_contents):
|
||||
assert IMutableUploadable.providedBy(new_contents)
|
||||
assert self._servermap.last_update_mode == MODE_WRITE
|
||||
assert self._servermap.get_last_update()[0] == MODE_WRITE
|
||||
|
||||
return self._upload(new_contents)
|
||||
|
||||
@ -874,7 +877,7 @@ class MutableFileVersion:
|
||||
I attempt to apply a modifier to the contents of the mutable
|
||||
file.
|
||||
"""
|
||||
assert self._servermap.last_update_mode != MODE_READ
|
||||
assert self._servermap.get_last_update()[0] != MODE_READ
|
||||
|
||||
# download_to_data is serialized, so we have to call this to
|
||||
# avoid deadlock.
|
||||
@ -974,7 +977,8 @@ class MutableFileVersion:
|
||||
"""
|
||||
I am the serialized companion of read.
|
||||
"""
|
||||
r = Retrieve(self._node, self._servermap, self._version, fetch_privkey)
|
||||
r = Retrieve(self._node, self._storage_broker, self._servermap,
|
||||
self._version, fetch_privkey)
|
||||
if self._history:
|
||||
self._history.notify_retrieve(r.get_status())
|
||||
d = r.download(consumer, offset, size)
|
||||
@ -1122,7 +1126,8 @@ class MutableFileVersion:
|
||||
used by the new uploadable. I return a Deferred that fires with
|
||||
the segments.
|
||||
"""
|
||||
r = Retrieve(self._node, self._servermap, self._version)
|
||||
r = Retrieve(self._node, self._storage_broker, self._servermap,
|
||||
self._version)
|
||||
# decode: takes in our blocks and salts from the servermap,
|
||||
# returns a Deferred that fires with the corresponding plaintext
|
||||
# segments. Does not download -- simply takes advantage of
|
||||
|
||||
@ -8,7 +8,7 @@ from twisted.internet import defer
|
||||
from twisted.python import failure
|
||||
from allmydata.interfaces import IPublishStatus, SDMF_VERSION, MDMF_VERSION, \
|
||||
IMutableUploadable
|
||||
from allmydata.util import base32, hashutil, mathutil, idlib, log
|
||||
from allmydata.util import base32, hashutil, mathutil, log
|
||||
from allmydata.util.dictutil import DictOfSets
|
||||
from allmydata import hashtree, codec
|
||||
from allmydata.storage.server import si_b2a
|
||||
@ -39,7 +39,7 @@ class PublishStatus:
|
||||
self.timings["encrypt"] = 0.0
|
||||
self.timings["encode"] = 0.0
|
||||
self.servermap = None
|
||||
self.problems = {}
|
||||
self._problems = {}
|
||||
self.active = True
|
||||
self.storage_index = None
|
||||
self.helper = False
|
||||
@ -50,10 +50,11 @@ class PublishStatus:
|
||||
self.counter = self.statusid_counter.next()
|
||||
self.started = time.time()
|
||||
|
||||
def add_per_server_time(self, peerid, elapsed):
|
||||
if peerid not in self.timings["send_per_server"]:
|
||||
self.timings["send_per_server"][peerid] = []
|
||||
self.timings["send_per_server"][peerid].append(elapsed)
|
||||
def add_per_server_time(self, server, elapsed):
|
||||
serverid = server.get_serverid()
|
||||
if serverid not in self.timings["send_per_server"]:
|
||||
self.timings["send_per_server"][serverid] = []
|
||||
self.timings["send_per_server"][serverid].append(elapsed)
|
||||
def accumulate_encode_time(self, elapsed):
|
||||
self.timings["encode"] += elapsed
|
||||
def accumulate_encrypt_time(self, elapsed):
|
||||
@ -79,6 +80,8 @@ class PublishStatus:
|
||||
return self.active
|
||||
def get_counter(self):
|
||||
return self.counter
|
||||
def get_problems(self):
|
||||
return self._problems
|
||||
|
||||
def set_storage_index(self, si):
|
||||
self.storage_index = si
|
||||
@ -152,8 +155,8 @@ class Publish:
|
||||
it on their own.
|
||||
"""
|
||||
# How this works:
|
||||
# 1: Make peer assignments. We'll assign each share that we know
|
||||
# about on the grid to that peer that currently holds that
|
||||
# 1: Make server assignments. We'll assign each share that we know
|
||||
# about on the grid to that server that currently holds that
|
||||
# share, and will not place any new shares.
|
||||
# 2: Setup encoding parameters. Most of these will stay the same
|
||||
# -- datalength will change, as will some of the offsets.
|
||||
@ -183,9 +186,9 @@ class Publish:
|
||||
|
||||
# first, which servers will we publish to? We require that the
|
||||
# servermap was updated in MODE_WRITE, so we can depend upon the
|
||||
# peerlist computed by that process instead of computing our own.
|
||||
# serverlist computed by that process instead of computing our own.
|
||||
assert self._servermap
|
||||
assert self._servermap.last_update_mode in (MODE_WRITE, MODE_CHECK)
|
||||
assert self._servermap.get_last_update()[0] in (MODE_WRITE, MODE_CHECK)
|
||||
# we will push a version that is one larger than anything present
|
||||
# in the grid, according to the servermap.
|
||||
self._new_seqnum = self._servermap.highest_seqnum() + 1
|
||||
@ -210,10 +213,9 @@ class Publish:
|
||||
self._encprivkey = self._node.get_encprivkey()
|
||||
|
||||
sb = self._storage_broker
|
||||
full_peerlist = [(s.get_serverid(), s.get_rref())
|
||||
for s in sb.get_servers_for_psi(self._storage_index)]
|
||||
self.full_peerlist = full_peerlist # for use later, immutable
|
||||
self.bad_peers = set() # peerids who have errbacked/refused requests
|
||||
full_serverlist = list(sb.get_servers_for_psi(self._storage_index))
|
||||
self.full_serverlist = full_serverlist # for use later, immutable
|
||||
self.bad_servers = set() # servers who have errbacked/refused requests
|
||||
|
||||
# This will set self.segment_size, self.num_segments, and
|
||||
# self.fec. TODO: Does it know how to do the offset? Probably
|
||||
@ -229,7 +231,7 @@ class Publish:
|
||||
# we keep track of three tables. The first is our goal: which share
|
||||
# we want to see on which servers. This is initially populated by the
|
||||
# existing servermap.
|
||||
self.goal = set() # pairs of (peerid, shnum) tuples
|
||||
self.goal = set() # pairs of (server, shnum) tuples
|
||||
|
||||
# the number of outstanding queries: those that are in flight and
|
||||
# may or may not be delivered, accepted, or acknowledged. This is
|
||||
@ -240,12 +242,7 @@ class Publish:
|
||||
# the third is a table of successes: share which have actually been
|
||||
# placed. These are populated when responses come back with success.
|
||||
# When self.placed == self.goal, we're done.
|
||||
self.placed = set() # (peerid, shnum) tuples
|
||||
|
||||
# we also keep a mapping from peerid to RemoteReference. Each time we
|
||||
# pull a connection out of the full peerlist, we add it to this for
|
||||
# use later.
|
||||
self.connections = {}
|
||||
self.placed = set() # (server, shnum) tuples
|
||||
|
||||
self.bad_share_checkstrings = {}
|
||||
|
||||
@ -256,27 +253,24 @@ class Publish:
|
||||
# try to update each existing share in place. Since we're
|
||||
# updating, we ignore damaged and missing shares -- callers must
|
||||
# do a repair to repair and recreate these.
|
||||
for (peerid, shnum) in self._servermap.servermap:
|
||||
self.goal.add( (peerid, shnum) )
|
||||
self.connections[peerid] = self._servermap.connections[peerid]
|
||||
self.goal = set(self._servermap.get_known_shares())
|
||||
self.writers = {}
|
||||
|
||||
# SDMF files are updated differently.
|
||||
self._version = MDMF_VERSION
|
||||
writer_class = MDMFSlotWriteProxy
|
||||
|
||||
# For each (peerid, shnum) in self.goal, we make a
|
||||
# write proxy for that peer. We'll use this to write
|
||||
# shares to the peer.
|
||||
for key in self.goal:
|
||||
peerid, shnum = key
|
||||
write_enabler = self._node.get_write_enabler(peerid)
|
||||
renew_secret = self._node.get_renewal_secret(peerid)
|
||||
cancel_secret = self._node.get_cancel_secret(peerid)
|
||||
# For each (server, shnum) in self.goal, we make a
|
||||
# write proxy for that server. We'll use this to write
|
||||
# shares to the server.
|
||||
for (server,shnum) in self.goal:
|
||||
write_enabler = self._node.get_write_enabler(server)
|
||||
renew_secret = self._node.get_renewal_secret(server)
|
||||
cancel_secret = self._node.get_cancel_secret(server)
|
||||
secrets = (write_enabler, renew_secret, cancel_secret)
|
||||
|
||||
self.writers[shnum] = writer_class(shnum,
|
||||
self.connections[peerid],
|
||||
server.get_rref(),
|
||||
self._storage_index,
|
||||
secrets,
|
||||
self._new_seqnum,
|
||||
@ -284,9 +278,10 @@ class Publish:
|
||||
self.total_shares,
|
||||
self.segment_size,
|
||||
self.datalength)
|
||||
self.writers[shnum].peerid = peerid
|
||||
assert (peerid, shnum) in self._servermap.servermap
|
||||
old_versionid, old_timestamp = self._servermap.servermap[key]
|
||||
self.writers[shnum].server = server
|
||||
known_shares = self._servermap.get_known_shares()
|
||||
assert (server, shnum) in known_shares
|
||||
old_versionid, old_timestamp = known_shares[(server,shnum)]
|
||||
(old_seqnum, old_root_hash, old_salt, old_segsize,
|
||||
old_datalength, old_k, old_N, old_prefix,
|
||||
old_offsets_tuple) = old_versionid
|
||||
@ -375,9 +370,9 @@ class Publish:
|
||||
|
||||
# first, which servers will we publish to? We require that the
|
||||
# servermap was updated in MODE_WRITE, so we can depend upon the
|
||||
# peerlist computed by that process instead of computing our own.
|
||||
# serverlist computed by that process instead of computing our own.
|
||||
if self._servermap:
|
||||
assert self._servermap.last_update_mode in (MODE_WRITE, MODE_CHECK)
|
||||
assert self._servermap.get_last_update()[0] in (MODE_WRITE, MODE_CHECK)
|
||||
# we will push a version that is one larger than anything present
|
||||
# in the grid, according to the servermap.
|
||||
self._new_seqnum = self._servermap.highest_seqnum() + 1
|
||||
@ -408,10 +403,9 @@ class Publish:
|
||||
self._encprivkey = self._node.get_encprivkey()
|
||||
|
||||
sb = self._storage_broker
|
||||
full_peerlist = [(s.get_serverid(), s.get_rref())
|
||||
for s in sb.get_servers_for_psi(self._storage_index)]
|
||||
self.full_peerlist = full_peerlist # for use later, immutable
|
||||
self.bad_peers = set() # peerids who have errbacked/refused requests
|
||||
full_serverlist = list(sb.get_servers_for_psi(self._storage_index))
|
||||
self.full_serverlist = full_serverlist # for use later, immutable
|
||||
self.bad_servers = set() # servers who have errbacked/refused requests
|
||||
|
||||
# This will set self.segment_size, self.num_segments, and
|
||||
# self.fec.
|
||||
@ -426,7 +420,7 @@ class Publish:
|
||||
# we keep track of three tables. The first is our goal: which share
|
||||
# we want to see on which servers. This is initially populated by the
|
||||
# existing servermap.
|
||||
self.goal = set() # pairs of (peerid, shnum) tuples
|
||||
self.goal = set() # pairs of (server, shnum) tuples
|
||||
|
||||
# the number of outstanding queries: those that are in flight and
|
||||
# may or may not be delivered, accepted, or acknowledged. This is
|
||||
@ -437,12 +431,7 @@ class Publish:
|
||||
# the third is a table of successes: share which have actually been
|
||||
# placed. These are populated when responses come back with success.
|
||||
# When self.placed == self.goal, we're done.
|
||||
self.placed = set() # (peerid, shnum) tuples
|
||||
|
||||
# we also keep a mapping from peerid to RemoteReference. Each time we
|
||||
# pull a connection out of the full peerlist, we add it to this for
|
||||
# use later.
|
||||
self.connections = {}
|
||||
self.placed = set() # (server, shnum) tuples
|
||||
|
||||
self.bad_share_checkstrings = {}
|
||||
|
||||
@ -451,18 +440,16 @@ class Publish:
|
||||
|
||||
# we use the servermap to populate the initial goal: this way we will
|
||||
# try to update each existing share in place.
|
||||
for (peerid, shnum) in self._servermap.servermap:
|
||||
self.goal.add( (peerid, shnum) )
|
||||
self.connections[peerid] = self._servermap.connections[peerid]
|
||||
self.goal = set(self._servermap.get_known_shares())
|
||||
|
||||
# then we add in all the shares that were bad (corrupted, bad
|
||||
# signatures, etc). We want to replace these.
|
||||
for key, old_checkstring in self._servermap.bad_shares.items():
|
||||
(peerid, shnum) = key
|
||||
self.goal.add(key)
|
||||
self.bad_share_checkstrings[key] = old_checkstring
|
||||
self.connections[peerid] = self._servermap.connections[peerid]
|
||||
for key, old_checkstring in self._servermap.get_bad_shares().items():
|
||||
(server, shnum) = key
|
||||
self.goal.add( (server,shnum) )
|
||||
self.bad_share_checkstrings[(server,shnum)] = old_checkstring
|
||||
|
||||
# TODO: Make this part do peer selection.
|
||||
# TODO: Make this part do server selection.
|
||||
self.update_goal()
|
||||
self.writers = {}
|
||||
if self._version == MDMF_VERSION:
|
||||
@ -470,18 +457,17 @@ class Publish:
|
||||
else:
|
||||
writer_class = SDMFSlotWriteProxy
|
||||
|
||||
# For each (peerid, shnum) in self.goal, we make a
|
||||
# write proxy for that peer. We'll use this to write
|
||||
# shares to the peer.
|
||||
for key in self.goal:
|
||||
peerid, shnum = key
|
||||
write_enabler = self._node.get_write_enabler(peerid)
|
||||
renew_secret = self._node.get_renewal_secret(peerid)
|
||||
cancel_secret = self._node.get_cancel_secret(peerid)
|
||||
# For each (server, shnum) in self.goal, we make a
|
||||
# write proxy for that server. We'll use this to write
|
||||
# shares to the server.
|
||||
for (server,shnum) in self.goal:
|
||||
write_enabler = self._node.get_write_enabler(server)
|
||||
renew_secret = self._node.get_renewal_secret(server)
|
||||
cancel_secret = self._node.get_cancel_secret(server)
|
||||
secrets = (write_enabler, renew_secret, cancel_secret)
|
||||
|
||||
self.writers[shnum] = writer_class(shnum,
|
||||
self.connections[peerid],
|
||||
server.get_rref(),
|
||||
self._storage_index,
|
||||
secrets,
|
||||
self._new_seqnum,
|
||||
@ -489,17 +475,18 @@ class Publish:
|
||||
self.total_shares,
|
||||
self.segment_size,
|
||||
self.datalength)
|
||||
self.writers[shnum].peerid = peerid
|
||||
if (peerid, shnum) in self._servermap.servermap:
|
||||
old_versionid, old_timestamp = self._servermap.servermap[key]
|
||||
self.writers[shnum].server = server
|
||||
known_shares = self._servermap.get_known_shares()
|
||||
if (server, shnum) in known_shares:
|
||||
old_versionid, old_timestamp = known_shares[(server,shnum)]
|
||||
(old_seqnum, old_root_hash, old_salt, old_segsize,
|
||||
old_datalength, old_k, old_N, old_prefix,
|
||||
old_offsets_tuple) = old_versionid
|
||||
self.writers[shnum].set_checkstring(old_seqnum,
|
||||
old_root_hash,
|
||||
old_salt)
|
||||
elif (peerid, shnum) in self.bad_share_checkstrings:
|
||||
old_checkstring = self.bad_share_checkstrings[(peerid, shnum)]
|
||||
elif (server, shnum) in self.bad_share_checkstrings:
|
||||
old_checkstring = self.bad_share_checkstrings[(server, shnum)]
|
||||
self.writers[shnum].set_checkstring(old_checkstring)
|
||||
|
||||
# Our remote shares will not have a complete checkstring until
|
||||
@ -897,9 +884,8 @@ class Publish:
|
||||
|
||||
def log_goal(self, goal, message=""):
|
||||
logmsg = [message]
|
||||
for (shnum, peerid) in sorted([(s,p) for (p,s) in goal]):
|
||||
logmsg.append("sh%d to [%s]" % (shnum,
|
||||
idlib.shortnodeid_b2a(peerid)))
|
||||
for (shnum, server) in sorted([(s,p) for (p,s) in goal]):
|
||||
logmsg.append("sh%d to [%s]" % (shnum, server.get_name()))
|
||||
self.log("current goal: %s" % (", ".join(logmsg)), level=log.NOISY)
|
||||
self.log("we are planning to push new seqnum=#%d" % self._new_seqnum,
|
||||
level=log.NOISY)
|
||||
@ -909,56 +895,57 @@ class Publish:
|
||||
if True:
|
||||
self.log_goal(self.goal, "before update: ")
|
||||
|
||||
# first, remove any bad peers from our goal
|
||||
self.goal = set([ (peerid, shnum)
|
||||
for (peerid, shnum) in self.goal
|
||||
if peerid not in self.bad_peers ])
|
||||
# first, remove any bad servers from our goal
|
||||
self.goal = set([ (server, shnum)
|
||||
for (server, shnum) in self.goal
|
||||
if server not in self.bad_servers ])
|
||||
|
||||
# find the homeless shares:
|
||||
homefull_shares = set([shnum for (peerid, shnum) in self.goal])
|
||||
homefull_shares = set([shnum for (server, shnum) in self.goal])
|
||||
homeless_shares = set(range(self.total_shares)) - homefull_shares
|
||||
homeless_shares = sorted(list(homeless_shares))
|
||||
# place them somewhere. We prefer unused servers at the beginning of
|
||||
# the available peer list.
|
||||
# the available server list.
|
||||
|
||||
if not homeless_shares:
|
||||
return
|
||||
|
||||
# if an old share X is on a node, put the new share X there too.
|
||||
# TODO: 1: redistribute shares to achieve one-per-peer, by copying
|
||||
# shares from existing peers to new (less-crowded) ones. The
|
||||
# TODO: 1: redistribute shares to achieve one-per-server, by copying
|
||||
# shares from existing servers to new (less-crowded) ones. The
|
||||
# old shares must still be updated.
|
||||
# TODO: 2: move those shares instead of copying them, to reduce future
|
||||
# update work
|
||||
|
||||
# this is a bit CPU intensive but easy to analyze. We create a sort
|
||||
# order for each peerid. If the peerid is marked as bad, we don't
|
||||
# order for each server. If the server is marked as bad, we don't
|
||||
# even put them in the list. Then we care about the number of shares
|
||||
# which have already been assigned to them. After that we care about
|
||||
# their permutation order.
|
||||
old_assignments = DictOfSets()
|
||||
for (peerid, shnum) in self.goal:
|
||||
old_assignments.add(peerid, shnum)
|
||||
for (server, shnum) in self.goal:
|
||||
old_assignments.add(server, shnum)
|
||||
|
||||
peerlist = []
|
||||
for i, (peerid, ss) in enumerate(self.full_peerlist):
|
||||
if peerid in self.bad_peers:
|
||||
serverlist = []
|
||||
for i, server in enumerate(self.full_serverlist):
|
||||
serverid = server.get_serverid()
|
||||
if server in self.bad_servers:
|
||||
continue
|
||||
entry = (len(old_assignments.get(peerid, [])), i, peerid, ss)
|
||||
peerlist.append(entry)
|
||||
peerlist.sort()
|
||||
entry = (len(old_assignments.get(server, [])), i, serverid, server)
|
||||
serverlist.append(entry)
|
||||
serverlist.sort()
|
||||
|
||||
if not peerlist:
|
||||
if not serverlist:
|
||||
raise NotEnoughServersError("Ran out of non-bad servers, "
|
||||
"first_error=%s" %
|
||||
str(self._first_write_error),
|
||||
self._first_write_error)
|
||||
|
||||
# we then index this peerlist with an integer, because we may have to
|
||||
# wrap. We update the goal as we go.
|
||||
# we then index this serverlist with an integer, because we may have
|
||||
# to wrap. We update the goal as we go.
|
||||
i = 0
|
||||
for shnum in homeless_shares:
|
||||
(ignored1, ignored2, peerid, ss) = peerlist[i]
|
||||
(ignored1, ignored2, ignored3, server) = serverlist[i]
|
||||
# if we are forced to send a share to a server that already has
|
||||
# one, we may have two write requests in flight, and the
|
||||
# servermap (which was computed before either request was sent)
|
||||
@ -967,10 +954,9 @@ class Publish:
|
||||
# this, otherwise it would cause the publish to fail with an
|
||||
# UncoordinatedWriteError. See #546 for details of the trouble
|
||||
# this used to cause.
|
||||
self.goal.add( (peerid, shnum) )
|
||||
self.connections[peerid] = ss
|
||||
self.goal.add( (server, shnum) )
|
||||
i += 1
|
||||
if i >= len(peerlist):
|
||||
if i >= len(serverlist):
|
||||
i = 0
|
||||
if True:
|
||||
self.log_goal(self.goal, "after update: ")
|
||||
@ -985,25 +971,25 @@ class Publish:
|
||||
# bother checking it.
|
||||
return
|
||||
|
||||
peerid = writer.peerid
|
||||
server = writer.server
|
||||
lp = self.log("_got_write_answer from %s, share %d" %
|
||||
(idlib.shortnodeid_b2a(peerid), writer.shnum))
|
||||
(server.get_name(), writer.shnum))
|
||||
|
||||
now = time.time()
|
||||
elapsed = now - started
|
||||
|
||||
self._status.add_per_server_time(peerid, elapsed)
|
||||
self._status.add_per_server_time(server, elapsed)
|
||||
|
||||
wrote, read_data = answer
|
||||
|
||||
surprise_shares = set(read_data.keys()) - set([writer.shnum])
|
||||
|
||||
# We need to remove from surprise_shares any shares that we are
|
||||
# knowingly also writing to that peer from other writers.
|
||||
# knowingly also writing to that server from other writers.
|
||||
|
||||
# TODO: Precompute this.
|
||||
known_shnums = [x.shnum for x in self.writers.values()
|
||||
if x.peerid == peerid]
|
||||
if x.server == server]
|
||||
surprise_shares -= set(known_shnums)
|
||||
self.log("found the following surprise shares: %s" %
|
||||
str(surprise_shares))
|
||||
@ -1024,7 +1010,7 @@ class Publish:
|
||||
if checkstring == self._checkstring:
|
||||
# they have the right share, somehow
|
||||
|
||||
if (peerid,shnum) in self.goal:
|
||||
if (server,shnum) in self.goal:
|
||||
# and we want them to have it, so we probably sent them a
|
||||
# copy in an earlier write. This is ok, and avoids the
|
||||
# #546 problem.
|
||||
@ -1039,7 +1025,7 @@ class Publish:
|
||||
|
||||
else:
|
||||
# the new shares are of a different version
|
||||
if peerid in self._servermap.reachable_peers:
|
||||
if server in self._servermap.get_reachable_servers():
|
||||
# we asked them about their shares, so we had knowledge
|
||||
# of what they used to have. Any surprising shares must
|
||||
# have come from someone else, so UCW.
|
||||
@ -1050,16 +1036,16 @@ class Publish:
|
||||
# mapupdate should have wokred harder and asked more
|
||||
# servers before concluding that it knew about them all.
|
||||
|
||||
# signal UCW, but make sure to ask this peer next time,
|
||||
# signal UCW, but make sure to ask this server next time,
|
||||
# so we'll remember to update it if/when we retry.
|
||||
surprised = True
|
||||
# TODO: ask this peer next time. I don't yet have a good
|
||||
# TODO: ask this server next time. I don't yet have a good
|
||||
# way to do this. Two insufficient possibilities are:
|
||||
#
|
||||
# self._servermap.add_new_share(peerid, shnum, verinfo, now)
|
||||
# self._servermap.add_new_share(server, shnum, verinfo, now)
|
||||
# but that requires fetching/validating/parsing the whole
|
||||
# version string, and all we have is the checkstring
|
||||
# self._servermap.mark_bad_share(peerid, shnum, checkstring)
|
||||
# self._servermap.mark_bad_share(server, shnum, checkstring)
|
||||
# that will make publish overwrite the share next time,
|
||||
# but it won't re-query the server, and it won't make
|
||||
# mapupdate search further
|
||||
@ -1090,17 +1076,17 @@ class Publish:
|
||||
# a way to tell these two apart (in fact, the storage server code
|
||||
# doesn't have the option of refusing our share).
|
||||
#
|
||||
# If the server is full, mark the peer as bad (so we don't ask
|
||||
# If the server is full, mark the server as bad (so we don't ask
|
||||
# them again), but don't set self.surprised. The loop() will find
|
||||
# a new server.
|
||||
#
|
||||
# If the testv failed, log it, set self.surprised, but don't
|
||||
# bother adding to self.bad_peers .
|
||||
# bother adding to self.bad_servers .
|
||||
|
||||
self.log("our testv failed, so the write did not happen",
|
||||
parent=lp, level=log.WEIRD, umid="8sc26g")
|
||||
self.surprised = True
|
||||
self.bad_peers.add(writer) # don't ask them again
|
||||
self.bad_servers.add(server) # don't ask them again
|
||||
# use the checkstring to add information to the log message
|
||||
unknown_format = False
|
||||
for (shnum,readv) in read_data.items():
|
||||
@ -1115,8 +1101,8 @@ class Publish:
|
||||
other_IV) = unpack_sdmf_checkstring(checkstring)
|
||||
else:
|
||||
unknown_format = True
|
||||
expected_version = self._servermap.version_on_peer(peerid,
|
||||
shnum)
|
||||
expected_version = self._servermap.version_on_server(server,
|
||||
shnum)
|
||||
if expected_version:
|
||||
(seqnum, root_hash, IV, segsize, datalength, k, N, prefix,
|
||||
offsets_tuple) = expected_version
|
||||
@ -1132,8 +1118,8 @@ class Publish:
|
||||
(other_seqnum, other_roothash)
|
||||
self.log(msg, parent=lp, level=log.NOISY)
|
||||
# if expected_version==None, then we didn't expect to see a
|
||||
# share on that peer, and the 'surprise_shares' clause above
|
||||
# will have logged it.
|
||||
# share on that server, and the 'surprise_shares' clause
|
||||
# above will have logged it.
|
||||
return
|
||||
|
||||
# and update the servermap
|
||||
@ -1142,9 +1128,9 @@ class Publish:
|
||||
# shares, and can safely execute these statements.
|
||||
if self.versioninfo:
|
||||
self.log("wrote successfully: adding new share to servermap")
|
||||
self._servermap.add_new_share(peerid, writer.shnum,
|
||||
self._servermap.add_new_share(server, writer.shnum,
|
||||
self.versioninfo, started)
|
||||
self.placed.add( (peerid, writer.shnum) )
|
||||
self.placed.add( (server, writer.shnum) )
|
||||
self._update_status()
|
||||
# the next method in the deferred chain will check to see if
|
||||
# we're done and successful.
|
||||
|
||||
@ -27,7 +27,7 @@ class RetrieveStatus:
|
||||
self.timings["decode"] = 0.0
|
||||
self.timings["decrypt"] = 0.0
|
||||
self.timings["cumulative_verify"] = 0.0
|
||||
self.problems = {}
|
||||
self._problems = {}
|
||||
self.active = True
|
||||
self.storage_index = None
|
||||
self.helper = False
|
||||
@ -56,11 +56,14 @@ class RetrieveStatus:
|
||||
return self.active
|
||||
def get_counter(self):
|
||||
return self.counter
|
||||
def get_problems(self):
|
||||
return self._problems
|
||||
|
||||
def add_fetch_timing(self, peerid, elapsed):
|
||||
if peerid not in self.timings["fetch_per_server"]:
|
||||
self.timings["fetch_per_server"][peerid] = []
|
||||
self.timings["fetch_per_server"][peerid].append(elapsed)
|
||||
def add_fetch_timing(self, server, elapsed):
|
||||
serverid = server.get_serverid()
|
||||
if serverid not in self.timings["fetch_per_server"]:
|
||||
self.timings["fetch_per_server"][serverid] = []
|
||||
self.timings["fetch_per_server"][serverid].append(elapsed)
|
||||
def accumulate_decode_time(self, elapsed):
|
||||
self.timings["decode"] += elapsed
|
||||
def accumulate_decrypt_time(self, elapsed):
|
||||
@ -79,6 +82,9 @@ class RetrieveStatus:
|
||||
self.progress = value
|
||||
def set_active(self, value):
|
||||
self.active = value
|
||||
def add_problem(self, server, f):
|
||||
serverid = server.get_serverid()
|
||||
self._problems[serverid] = f
|
||||
|
||||
class Marker:
|
||||
pass
|
||||
@ -91,16 +97,16 @@ class Retrieve:
|
||||
# will use a single ServerMap instance.
|
||||
implements(IPushProducer)
|
||||
|
||||
def __init__(self, filenode, servermap, verinfo, fetch_privkey=False,
|
||||
verify=False):
|
||||
def __init__(self, filenode, storage_broker, servermap, verinfo,
|
||||
fetch_privkey=False, verify=False):
|
||||
self._node = filenode
|
||||
assert self._node.get_pubkey()
|
||||
self._storage_broker = storage_broker
|
||||
self._storage_index = filenode.get_storage_index()
|
||||
assert self._node.get_readkey()
|
||||
self._last_failure = None
|
||||
prefix = si_b2a(self._storage_index)[:5]
|
||||
self._log_number = log.msg("Retrieve(%s): starting" % prefix)
|
||||
self._outstanding_queries = {} # maps (peerid,shnum) to start_time
|
||||
self._running = True
|
||||
self._decoding = False
|
||||
self._bad_shares = set()
|
||||
@ -239,11 +245,11 @@ class Retrieve:
|
||||
self.log("starting download")
|
||||
self._started_fetching = time.time()
|
||||
# The download process beyond this is a state machine.
|
||||
# _add_active_peers will select the peers that we want to use
|
||||
# _add_active_servers will select the servers that we want to use
|
||||
# for the download, and then attempt to start downloading. After
|
||||
# each segment, it will check for doneness, reacting to broken
|
||||
# peers and corrupt shares as necessary. If it runs out of good
|
||||
# peers before downloading all of the segments, _done_deferred
|
||||
# servers and corrupt shares as necessary. If it runs out of good
|
||||
# servers before downloading all of the segments, _done_deferred
|
||||
# will errback. Otherwise, it will eventually callback with the
|
||||
# contents of the mutable file.
|
||||
self.loop()
|
||||
@ -251,7 +257,7 @@ class Retrieve:
|
||||
|
||||
def loop(self):
|
||||
d = fireEventually(None) # avoid #237 recursion limit problem
|
||||
d.addCallback(lambda ign: self._activate_enough_peers())
|
||||
d.addCallback(lambda ign: self._activate_enough_servers())
|
||||
d.addCallback(lambda ign: self._download_current_segment())
|
||||
# when we're done, _download_current_segment will call _done. If we
|
||||
# aren't, it will call loop() again.
|
||||
@ -277,20 +283,19 @@ class Retrieve:
|
||||
shares = versionmap[self.verinfo]
|
||||
# this sharemap is consumed as we decide to send requests
|
||||
self.remaining_sharemap = DictOfSets()
|
||||
for (shnum, peerid, timestamp) in shares:
|
||||
self.remaining_sharemap.add(shnum, peerid)
|
||||
for (shnum, server, timestamp) in shares:
|
||||
self.remaining_sharemap.add(shnum, server)
|
||||
# If the servermap update fetched anything, it fetched at least 1
|
||||
# KiB, so we ask for that much.
|
||||
# TODO: Change the cache methods to allow us to fetch all of the
|
||||
# data that they have, then change this method to do that.
|
||||
any_cache = self._node._read_from_cache(self.verinfo, shnum,
|
||||
0, 1000)
|
||||
ss = self.servermap.connections[peerid]
|
||||
reader = MDMFSlotReadProxy(ss,
|
||||
reader = MDMFSlotReadProxy(server.get_rref(),
|
||||
self._storage_index,
|
||||
shnum,
|
||||
any_cache)
|
||||
reader.peerid = peerid
|
||||
reader.server = server
|
||||
self.readers[shnum] = reader
|
||||
assert len(self.remaining_sharemap) >= k
|
||||
|
||||
@ -436,7 +441,7 @@ class Retrieve:
|
||||
|
||||
self._current_segment = self._start_segment
|
||||
|
||||
def _activate_enough_peers(self):
|
||||
def _activate_enough_servers(self):
|
||||
"""
|
||||
I populate self._active_readers with enough active readers to
|
||||
retrieve the contents of this mutable file. I am called before
|
||||
@ -445,9 +450,9 @@ class Retrieve:
|
||||
"""
|
||||
# TODO: It would be cool to investigate other heuristics for
|
||||
# reader selection. For instance, the cost (in time the user
|
||||
# spends waiting for their file) of selecting a really slow peer
|
||||
# spends waiting for their file) of selecting a really slow server
|
||||
# that happens to have a primary share is probably more than
|
||||
# selecting a really fast peer that doesn't have a primary
|
||||
# selecting a really fast server that doesn't have a primary
|
||||
# share. Maybe the servermap could be extended to provide this
|
||||
# information; it could keep track of latency information while
|
||||
# it gathers more important data, and then this routine could
|
||||
@ -485,7 +490,7 @@ class Retrieve:
|
||||
else:
|
||||
new_shnums = []
|
||||
|
||||
self.log("adding %d new peers to the active list" % len(new_shnums))
|
||||
self.log("adding %d new servers to the active list" % len(new_shnums))
|
||||
for shnum in new_shnums:
|
||||
reader = self.readers[shnum]
|
||||
self._active_readers.append(reader)
|
||||
@ -496,7 +501,7 @@ class Retrieve:
|
||||
# segment decoding, then we'll take more drastic measures.
|
||||
if self._need_privkey and not self._node.is_readonly():
|
||||
d = reader.get_encprivkey()
|
||||
d.addCallback(self._try_to_validate_privkey, reader)
|
||||
d.addCallback(self._try_to_validate_privkey, reader, reader.server)
|
||||
# XXX: don't just drop the Deferred. We need error-reporting
|
||||
# but not flow-control here.
|
||||
assert len(self._active_readers) >= self._required_shares
|
||||
@ -528,7 +533,7 @@ class Retrieve:
|
||||
|
||||
def _remove_reader(self, reader):
|
||||
"""
|
||||
At various points, we will wish to remove a peer from
|
||||
At various points, we will wish to remove a server from
|
||||
consideration and/or use. These include, but are not necessarily
|
||||
limited to:
|
||||
|
||||
@ -556,18 +561,18 @@ class Retrieve:
|
||||
self._active_readers.remove(reader)
|
||||
# TODO: self.readers.remove(reader)?
|
||||
for shnum in list(self.remaining_sharemap.keys()):
|
||||
self.remaining_sharemap.discard(shnum, reader.peerid)
|
||||
self.remaining_sharemap.discard(shnum, reader.server)
|
||||
|
||||
|
||||
def _mark_bad_share(self, reader, f):
|
||||
def _mark_bad_share(self, server, shnum, reader, f):
|
||||
"""
|
||||
I mark the (peerid, shnum) encapsulated by my reader argument as
|
||||
a bad share, which means that it will not be used anywhere else.
|
||||
I mark the given (server, shnum) as a bad share, which means that it
|
||||
will not be used anywhere else.
|
||||
|
||||
There are several reasons to want to mark something as a bad
|
||||
share. These include:
|
||||
|
||||
- A connection error to the peer.
|
||||
- A connection error to the server.
|
||||
- A mismatched prefix (that is, a prefix that does not match
|
||||
our local conception of the version information string).
|
||||
- A failing block hash, salt hash, share hash, or other
|
||||
@ -576,21 +581,18 @@ class Retrieve:
|
||||
This method will ensure that readers that we wish to mark bad
|
||||
(for these reasons or other reasons) are not used for the rest
|
||||
of the download. Additionally, it will attempt to tell the
|
||||
remote peer (with no guarantee of success) that its share is
|
||||
remote server (with no guarantee of success) that its share is
|
||||
corrupt.
|
||||
"""
|
||||
self.log("marking share %d on server %s as bad" % \
|
||||
(reader.shnum, reader))
|
||||
(shnum, server.get_name()))
|
||||
prefix = self.verinfo[-2]
|
||||
self.servermap.mark_bad_share(reader.peerid,
|
||||
reader.shnum,
|
||||
prefix)
|
||||
self.servermap.mark_bad_share(server, shnum, prefix)
|
||||
self._remove_reader(reader)
|
||||
self._bad_shares.add((reader.peerid, reader.shnum, f))
|
||||
self._status.problems[reader.peerid] = f
|
||||
self._bad_shares.add((server, shnum, f))
|
||||
self._status.add_problem(server, f)
|
||||
self._last_failure = f
|
||||
self.notify_server_corruption(reader.peerid, reader.shnum,
|
||||
str(f.value))
|
||||
self.notify_server_corruption(server, shnum, str(f.value))
|
||||
|
||||
|
||||
def _download_current_segment(self):
|
||||
@ -632,7 +634,7 @@ class Retrieve:
|
||||
d = reader.get_block_and_salt(segnum)
|
||||
d2 = self._get_needed_hashes(reader, segnum)
|
||||
dl = defer.DeferredList([d, d2], consumeErrors=True)
|
||||
dl.addCallback(self._validate_block, segnum, reader, started)
|
||||
dl.addCallback(self._validate_block, segnum, reader, reader.server, started)
|
||||
dl.addErrback(self._validation_or_decoding_failed, [reader])
|
||||
ds.append(dl)
|
||||
dl = defer.DeferredList(ds)
|
||||
@ -722,20 +724,20 @@ class Retrieve:
|
||||
I am called when a block or a salt fails to correctly validate, or when
|
||||
the decryption or decoding operation fails for some reason. I react to
|
||||
this failure by notifying the remote server of corruption, and then
|
||||
removing the remote peer from further activity.
|
||||
removing the remote server from further activity.
|
||||
"""
|
||||
assert isinstance(readers, list)
|
||||
bad_shnums = [reader.shnum for reader in readers]
|
||||
|
||||
self.log("validation or decoding failed on share(s) %s, peer(s) %s "
|
||||
self.log("validation or decoding failed on share(s) %s, server(s) %s "
|
||||
", segment %d: %s" % \
|
||||
(bad_shnums, readers, self._current_segment, str(f)))
|
||||
for reader in readers:
|
||||
self._mark_bad_share(reader, f)
|
||||
self._mark_bad_share(reader.server, reader.shnum, reader, f)
|
||||
return
|
||||
|
||||
|
||||
def _validate_block(self, results, segnum, reader, started):
|
||||
def _validate_block(self, results, segnum, reader, server, started):
|
||||
"""
|
||||
I validate a block from one share on a remote server.
|
||||
"""
|
||||
@ -744,7 +746,7 @@ class Retrieve:
|
||||
self.log("validating share %d for segment %d" % (reader.shnum,
|
||||
segnum))
|
||||
elapsed = time.time() - started
|
||||
self._status.add_fetch_timing(reader.peerid, elapsed)
|
||||
self._status.add_fetch_timing(server, elapsed)
|
||||
self._set_current_status("validating blocks")
|
||||
# Did we fail to fetch either of the things that we were
|
||||
# supposed to? Fail if so.
|
||||
@ -757,7 +759,7 @@ class Retrieve:
|
||||
assert isinstance(results[0][1], failure.Failure)
|
||||
|
||||
f = results[0][1]
|
||||
raise CorruptShareError(reader.peerid,
|
||||
raise CorruptShareError(server,
|
||||
reader.shnum,
|
||||
"Connection error: %s" % str(f))
|
||||
|
||||
@ -777,7 +779,7 @@ class Retrieve:
|
||||
bht.set_hashes(blockhashes)
|
||||
except (hashtree.BadHashError, hashtree.NotEnoughHashesError, \
|
||||
IndexError), e:
|
||||
raise CorruptShareError(reader.peerid,
|
||||
raise CorruptShareError(server,
|
||||
reader.shnum,
|
||||
"block hash tree failure: %s" % e)
|
||||
|
||||
@ -791,7 +793,7 @@ class Retrieve:
|
||||
bht.set_hashes(leaves={segnum: blockhash})
|
||||
except (hashtree.BadHashError, hashtree.NotEnoughHashesError, \
|
||||
IndexError), e:
|
||||
raise CorruptShareError(reader.peerid,
|
||||
raise CorruptShareError(server,
|
||||
reader.shnum,
|
||||
"block hash tree failure: %s" % e)
|
||||
|
||||
@ -812,7 +814,7 @@ class Retrieve:
|
||||
leaves={reader.shnum: bht[0]})
|
||||
except (hashtree.BadHashError, hashtree.NotEnoughHashesError, \
|
||||
IndexError), e:
|
||||
raise CorruptShareError(reader.peerid,
|
||||
raise CorruptShareError(server,
|
||||
reader.shnum,
|
||||
"corrupt hashes: %s" % e)
|
||||
|
||||
@ -931,13 +933,13 @@ class Retrieve:
|
||||
return plaintext
|
||||
|
||||
|
||||
def notify_server_corruption(self, peerid, shnum, reason):
|
||||
ss = self.servermap.connections[peerid]
|
||||
ss.callRemoteOnly("advise_corrupt_share",
|
||||
"mutable", self._storage_index, shnum, reason)
|
||||
def notify_server_corruption(self, server, shnum, reason):
|
||||
rref = server.get_rref()
|
||||
rref.callRemoteOnly("advise_corrupt_share",
|
||||
"mutable", self._storage_index, shnum, reason)
|
||||
|
||||
|
||||
def _try_to_validate_privkey(self, enc_privkey, reader):
|
||||
def _try_to_validate_privkey(self, enc_privkey, reader, server):
|
||||
alleged_privkey_s = self._node._decrypt_privkey(enc_privkey)
|
||||
alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s)
|
||||
if alleged_writekey != self._node.get_writekey():
|
||||
@ -945,13 +947,13 @@ class Retrieve:
|
||||
(reader, reader.shnum),
|
||||
level=log.WEIRD, umid="YIw4tA")
|
||||
if self._verify:
|
||||
self.servermap.mark_bad_share(reader.peerid, reader.shnum,
|
||||
self.servermap.mark_bad_share(server, reader.shnum,
|
||||
self.verinfo[-2])
|
||||
e = CorruptShareError(reader.peerid,
|
||||
e = CorruptShareError(server,
|
||||
reader.shnum,
|
||||
"invalid privkey")
|
||||
f = failure.Failure(e)
|
||||
self._bad_shares.add((reader.peerid, reader.shnum, f))
|
||||
self._bad_shares.add((server, reader.shnum, f))
|
||||
return
|
||||
|
||||
# it's good
|
||||
@ -986,7 +988,7 @@ class Retrieve:
|
||||
self._node._populate_total_shares(N)
|
||||
|
||||
if self._verify:
|
||||
ret = list(self._bad_shares)
|
||||
ret = self._bad_shares
|
||||
self.log("done verifying, found %d bad shares" % len(ret))
|
||||
else:
|
||||
# TODO: upload status here?
|
||||
@ -997,14 +999,14 @@ class Retrieve:
|
||||
|
||||
def _raise_notenoughshareserror(self):
|
||||
"""
|
||||
I am called by _activate_enough_peers when there are not enough
|
||||
active peers left to complete the download. After making some
|
||||
I am called by _activate_enough_servers when there are not enough
|
||||
active servers left to complete the download. After making some
|
||||
useful logging statements, I throw an exception to that effect
|
||||
to the caller of this Retrieve object through
|
||||
self._done_deferred.
|
||||
"""
|
||||
|
||||
format = ("ran out of peers: "
|
||||
format = ("ran out of servers: "
|
||||
"have %(have)d of %(total)d segments "
|
||||
"found %(bad)d bad shares "
|
||||
"encoding %(k)d-of-%(n)d")
|
||||
|
||||
@ -6,7 +6,7 @@ from twisted.internet import defer
|
||||
from twisted.python import failure
|
||||
from foolscap.api import DeadReferenceError, RemoteException, eventually, \
|
||||
fireEventually
|
||||
from allmydata.util import base32, hashutil, idlib, log, deferredutil
|
||||
from allmydata.util import base32, hashutil, log, deferredutil
|
||||
from allmydata.util.dictutil import DictOfSets
|
||||
from allmydata.storage.server import si_b2a
|
||||
from allmydata.interfaces import IServermapUpdaterStatus
|
||||
@ -34,11 +34,12 @@ class UpdateStatus:
|
||||
self.started = time.time()
|
||||
self.finished = None
|
||||
|
||||
def add_per_server_time(self, peerid, op, sent, elapsed):
|
||||
def add_per_server_time(self, server, op, sent, elapsed):
|
||||
serverid = server.get_serverid()
|
||||
assert op in ("query", "late", "privkey")
|
||||
if peerid not in self.timings["per_server"]:
|
||||
self.timings["per_server"][peerid] = []
|
||||
self.timings["per_server"][peerid].append((op,sent,elapsed))
|
||||
if serverid not in self.timings["per_server"]:
|
||||
self.timings["per_server"][serverid] = []
|
||||
self.timings["per_server"][serverid].append((op,sent,elapsed))
|
||||
|
||||
def get_started(self):
|
||||
return self.started
|
||||
@ -69,8 +70,8 @@ class UpdateStatus:
|
||||
self.storage_index = si
|
||||
def set_mode(self, mode):
|
||||
self.mode = mode
|
||||
def set_privkey_from(self, peerid):
|
||||
self.privkey_from = peerid
|
||||
def set_privkey_from(self, server):
|
||||
self.privkey_from = server.get_serverid()
|
||||
def set_status(self, status):
|
||||
self.status = status
|
||||
def set_progress(self, value):
|
||||
@ -95,47 +96,53 @@ class ServerMap:
|
||||
has changed since I last retrieved this data'. This reduces the chances
|
||||
of clobbering a simultaneous (uncoordinated) write.
|
||||
|
||||
@ivar servermap: a dictionary, mapping a (peerid, shnum) tuple to a
|
||||
(versionid, timestamp) tuple. Each 'versionid' is a
|
||||
tuple of (seqnum, root_hash, IV, segsize, datalength,
|
||||
k, N, signed_prefix, offsets)
|
||||
@var _known_shares: a dictionary, mapping a (server, shnum) tuple to a
|
||||
(versionid, timestamp) tuple. Each 'versionid' is a
|
||||
tuple of (seqnum, root_hash, IV, segsize, datalength,
|
||||
k, N, signed_prefix, offsets)
|
||||
|
||||
@ivar connections: maps peerid to a RemoteReference
|
||||
|
||||
@ivar bad_shares: dict with keys of (peerid, shnum) tuples, describing
|
||||
shares that I should ignore (because a previous user of
|
||||
the servermap determined that they were invalid). The
|
||||
updater only locates a certain number of shares: if
|
||||
some of these turn out to have integrity problems and
|
||||
are unusable, the caller will need to mark those shares
|
||||
as bad, then re-update the servermap, then try again.
|
||||
The dict maps (peerid, shnum) tuple to old checkstring.
|
||||
@ivar _bad_shares: dict with keys of (server, shnum) tuples, describing
|
||||
shares that I should ignore (because a previous user
|
||||
of the servermap determined that they were invalid).
|
||||
The updater only locates a certain number of shares:
|
||||
if some of these turn out to have integrity problems
|
||||
and are unusable, the caller will need to mark those
|
||||
shares as bad, then re-update the servermap, then try
|
||||
again. The dict maps (server, shnum) tuple to old
|
||||
checkstring.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.servermap = {}
|
||||
self.connections = {}
|
||||
self.unreachable_peers = set() # peerids that didn't respond to queries
|
||||
self.reachable_peers = set() # peerids that did respond to queries
|
||||
self.problems = [] # mostly for debugging
|
||||
self.bad_shares = {} # maps (peerid,shnum) to old checkstring
|
||||
self.last_update_mode = None
|
||||
self.last_update_time = 0
|
||||
self._known_shares = {}
|
||||
self.unreachable_servers = set() # servers that didn't respond to queries
|
||||
self.reachable_servers = set() # servers that did respond to queries
|
||||
self._problems = [] # mostly for debugging
|
||||
self._bad_shares = {} # maps (server,shnum) to old checkstring
|
||||
self._last_update_mode = None
|
||||
self._last_update_time = 0
|
||||
self.update_data = {} # (verinfo,shnum) => data
|
||||
|
||||
def copy(self):
|
||||
s = ServerMap()
|
||||
s.servermap = self.servermap.copy() # tuple->tuple
|
||||
s.connections = self.connections.copy() # str->RemoteReference
|
||||
s.unreachable_peers = set(self.unreachable_peers)
|
||||
s.reachable_peers = set(self.reachable_peers)
|
||||
s.problems = self.problems[:]
|
||||
s.bad_shares = self.bad_shares.copy() # tuple->str
|
||||
s.last_update_mode = self.last_update_mode
|
||||
s.last_update_time = self.last_update_time
|
||||
s._known_shares = self._known_shares.copy() # tuple->tuple
|
||||
s.unreachable_servers = set(self.unreachable_servers)
|
||||
s.reachable_servers = set(self.reachable_servers)
|
||||
s._problems = self._problems[:]
|
||||
s._bad_shares = self._bad_shares.copy() # tuple->str
|
||||
s._last_update_mode = self._last_update_mode
|
||||
s._last_update_time = self._last_update_time
|
||||
return s
|
||||
|
||||
def mark_bad_share(self, peerid, shnum, checkstring):
|
||||
def get_reachable_servers(self):
|
||||
return self.reachable_servers
|
||||
|
||||
def mark_server_reachable(self, server):
|
||||
self.reachable_servers.add(server)
|
||||
|
||||
def mark_server_unreachable(self, server):
|
||||
self.unreachable_servers.add(server)
|
||||
|
||||
def mark_bad_share(self, server, shnum, checkstring):
|
||||
"""This share was found to be bad, either in the checkstring or
|
||||
signature (detected during mapupdate), or deeper in the share
|
||||
(detected at retrieve time). Remove it from our list of useful
|
||||
@ -144,70 +151,84 @@ class ServerMap:
|
||||
corrupted or badly signed) so that a repair operation can do the
|
||||
test-and-set using it as a reference.
|
||||
"""
|
||||
key = (peerid, shnum) # record checkstring
|
||||
self.bad_shares[key] = checkstring
|
||||
self.servermap.pop(key, None)
|
||||
key = (server, shnum) # record checkstring
|
||||
self._bad_shares[key] = checkstring
|
||||
self._known_shares.pop(key, None)
|
||||
|
||||
def add_new_share(self, peerid, shnum, verinfo, timestamp):
|
||||
def get_bad_shares(self):
|
||||
# key=(server,shnum) -> checkstring
|
||||
return self._bad_shares
|
||||
|
||||
def add_new_share(self, server, shnum, verinfo, timestamp):
|
||||
"""We've written a new share out, replacing any that was there
|
||||
before."""
|
||||
key = (peerid, shnum)
|
||||
self.bad_shares.pop(key, None)
|
||||
self.servermap[key] = (verinfo, timestamp)
|
||||
key = (server, shnum)
|
||||
self._bad_shares.pop(key, None)
|
||||
self._known_shares[key] = (verinfo, timestamp)
|
||||
|
||||
def add_problem(self, f):
|
||||
self._problems.append(f)
|
||||
def get_problems(self):
|
||||
return self._problems
|
||||
|
||||
def set_last_update(self, mode, when):
|
||||
self._last_update_mode = mode
|
||||
self._last_update_time = when
|
||||
def get_last_update(self):
|
||||
return (self._last_update_mode, self._last_update_time)
|
||||
|
||||
def dump(self, out=sys.stdout):
|
||||
print >>out, "servermap:"
|
||||
|
||||
for ( (peerid, shnum), (verinfo, timestamp) ) in self.servermap.items():
|
||||
for ( (server, shnum), (verinfo, timestamp) ) in self._known_shares.items():
|
||||
(seqnum, root_hash, IV, segsize, datalength, k, N, prefix,
|
||||
offsets_tuple) = verinfo
|
||||
print >>out, ("[%s]: sh#%d seq%d-%s %d-of-%d len%d" %
|
||||
(idlib.shortnodeid_b2a(peerid), shnum,
|
||||
(server.get_name(), shnum,
|
||||
seqnum, base32.b2a(root_hash)[:4], k, N,
|
||||
datalength))
|
||||
if self.problems:
|
||||
print >>out, "%d PROBLEMS" % len(self.problems)
|
||||
for f in self.problems:
|
||||
if self._problems:
|
||||
print >>out, "%d PROBLEMS" % len(self._problems)
|
||||
for f in self._problems:
|
||||
print >>out, str(f)
|
||||
return out
|
||||
|
||||
def all_peers(self):
|
||||
return set([peerid
|
||||
for (peerid, shnum)
|
||||
in self.servermap])
|
||||
def all_servers(self):
|
||||
return set([server for (server, shnum) in self._known_shares])
|
||||
|
||||
def all_peers_for_version(self, verinfo):
|
||||
"""Return a set of peerids that hold shares for the given version."""
|
||||
return set([peerid
|
||||
for ( (peerid, shnum), (verinfo2, timestamp) )
|
||||
in self.servermap.items()
|
||||
def all_servers_for_version(self, verinfo):
|
||||
"""Return a set of servers that hold shares for the given version."""
|
||||
return set([server
|
||||
for ( (server, shnum), (verinfo2, timestamp) )
|
||||
in self._known_shares.items()
|
||||
if verinfo == verinfo2])
|
||||
|
||||
def get_known_shares(self):
|
||||
# maps (server,shnum) to (versionid,timestamp)
|
||||
return self._known_shares
|
||||
|
||||
def make_sharemap(self):
|
||||
"""Return a dict that maps shnum to a set of peerds that hold it."""
|
||||
"""Return a dict that maps shnum to a set of servers that hold it."""
|
||||
sharemap = DictOfSets()
|
||||
for (peerid, shnum) in self.servermap:
|
||||
sharemap.add(shnum, peerid)
|
||||
for (server, shnum) in self._known_shares:
|
||||
sharemap.add(shnum, server)
|
||||
return sharemap
|
||||
|
||||
def make_versionmap(self):
|
||||
"""Return a dict that maps versionid to sets of (shnum, peerid,
|
||||
"""Return a dict that maps versionid to sets of (shnum, server,
|
||||
timestamp) tuples."""
|
||||
versionmap = DictOfSets()
|
||||
for ( (peerid, shnum), (verinfo, timestamp) ) in self.servermap.items():
|
||||
versionmap.add(verinfo, (shnum, peerid, timestamp))
|
||||
for ( (server, shnum), (verinfo, timestamp) ) in self._known_shares.items():
|
||||
versionmap.add(verinfo, (shnum, server, timestamp))
|
||||
return versionmap
|
||||
|
||||
def shares_on_peer(self, peerid):
|
||||
return set([shnum
|
||||
for (s_peerid, shnum)
|
||||
in self.servermap
|
||||
if s_peerid == peerid])
|
||||
def debug_shares_on_server(self, server): # used by tests
|
||||
return set([shnum for (s, shnum) in self._known_shares if s == server])
|
||||
|
||||
def version_on_peer(self, peerid, shnum):
|
||||
key = (peerid, shnum)
|
||||
if key in self.servermap:
|
||||
(verinfo, timestamp) = self.servermap[key]
|
||||
def version_on_server(self, server, shnum):
|
||||
key = (server, shnum)
|
||||
if key in self._known_shares:
|
||||
(verinfo, timestamp) = self._known_shares[key]
|
||||
return verinfo
|
||||
return None
|
||||
|
||||
@ -218,7 +239,7 @@ class ServerMap:
|
||||
all_shares = {}
|
||||
for verinfo, shares in versionmap.items():
|
||||
s = set()
|
||||
for (shnum, peerid, timestamp) in shares:
|
||||
for (shnum, server, timestamp) in shares:
|
||||
s.add(shnum)
|
||||
(seqnum, root_hash, IV, segsize, datalength, k, N, prefix,
|
||||
offsets_tuple) = verinfo
|
||||
@ -244,7 +265,7 @@ class ServerMap:
|
||||
bits = []
|
||||
for (verinfo, shares) in versionmap.items():
|
||||
vstr = self.summarize_version(verinfo)
|
||||
shnums = set([shnum for (shnum, peerid, timestamp) in shares])
|
||||
shnums = set([shnum for (shnum, server, timestamp) in shares])
|
||||
bits.append("%d*%s" % (len(shnums), vstr))
|
||||
return "/".join(bits)
|
||||
|
||||
@ -256,7 +277,7 @@ class ServerMap:
|
||||
for (verinfo, shares) in versionmap.items():
|
||||
(seqnum, root_hash, IV, segsize, datalength, k, N, prefix,
|
||||
offsets_tuple) = verinfo
|
||||
shnums = set([shnum for (shnum, peerid, timestamp) in shares])
|
||||
shnums = set([shnum for (shnum, server, timestamp) in shares])
|
||||
if len(shnums) >= k:
|
||||
# this one is recoverable
|
||||
recoverable_versions.add(verinfo)
|
||||
@ -272,7 +293,7 @@ class ServerMap:
|
||||
for (verinfo, shares) in versionmap.items():
|
||||
(seqnum, root_hash, IV, segsize, datalength, k, N, prefix,
|
||||
offsets_tuple) = verinfo
|
||||
shnums = set([shnum for (shnum, peerid, timestamp) in shares])
|
||||
shnums = set([shnum for (shnum, server, timestamp) in shares])
|
||||
if len(shnums) < k:
|
||||
unrecoverable_versions.add(verinfo)
|
||||
|
||||
@ -306,7 +327,7 @@ class ServerMap:
|
||||
for (verinfo, shares) in versionmap.items():
|
||||
(seqnum, root_hash, IV, segsize, datalength, k, N, prefix,
|
||||
offsets_tuple) = verinfo
|
||||
shnums = set([shnum for (shnum, peerid, timestamp) in shares])
|
||||
shnums = set([shnum for (shnum, server, timestamp) in shares])
|
||||
healths[verinfo] = (len(shnums),k)
|
||||
if len(shnums) < k:
|
||||
unrecoverable.add(verinfo)
|
||||
@ -445,29 +466,21 @@ class ServermapUpdater:
|
||||
# avoid re-checking the signatures for each share.
|
||||
self._valid_versions = set()
|
||||
|
||||
# self.versionmap maps verinfo tuples to sets of (shnum, peerid,
|
||||
# timestamp) tuples. This is used to figure out which versions might
|
||||
# be retrievable, and to make the eventual data download faster.
|
||||
self.versionmap = DictOfSets()
|
||||
|
||||
self._done_deferred = defer.Deferred()
|
||||
|
||||
# first, which peers should be talk to? Any that were in our old
|
||||
# first, which servers should be talk to? Any that were in our old
|
||||
# servermap, plus "enough" others.
|
||||
|
||||
self._queries_completed = 0
|
||||
|
||||
sb = self._storage_broker
|
||||
# All of the peers, permuted by the storage index, as usual.
|
||||
full_peerlist = [(s.get_serverid(), s.get_rref())
|
||||
for s in sb.get_servers_for_psi(self._storage_index)]
|
||||
self.full_peerlist = full_peerlist # for use later, immutable
|
||||
self.extra_peers = full_peerlist[:] # peers are removed as we use them
|
||||
self._good_peers = set() # peers who had some shares
|
||||
self._empty_peers = set() # peers who don't have any shares
|
||||
self._bad_peers = set() # peers to whom our queries failed
|
||||
self._readers = {} # peerid -> dict(sharewriters), filled in
|
||||
# after responses come in.
|
||||
# All of the servers, permuted by the storage index, as usual.
|
||||
full_serverlist = list(sb.get_servers_for_psi(self._storage_index))
|
||||
self.full_serverlist = full_serverlist # for use later, immutable
|
||||
self.extra_servers = full_serverlist[:] # servers are removed as we use them
|
||||
self._good_servers = set() # servers who had some shares
|
||||
self._empty_servers = set() # servers who don't have any shares
|
||||
self._bad_servers = set() # servers to whom our queries failed
|
||||
|
||||
k = self._node.get_required_shares()
|
||||
# For what cases can these conditions work?
|
||||
@ -478,23 +491,23 @@ class ServermapUpdater:
|
||||
if N is None:
|
||||
N = 10
|
||||
self.EPSILON = k
|
||||
# we want to send queries to at least this many peers (although we
|
||||
# we want to send queries to at least this many servers (although we
|
||||
# might not wait for all of their answers to come back)
|
||||
self.num_peers_to_query = k + self.EPSILON
|
||||
self.num_servers_to_query = k + self.EPSILON
|
||||
|
||||
if self.mode == MODE_CHECK:
|
||||
# We want to query all of the peers.
|
||||
initial_peers_to_query = dict(full_peerlist)
|
||||
must_query = set(initial_peers_to_query.keys())
|
||||
self.extra_peers = []
|
||||
# We want to query all of the servers.
|
||||
initial_servers_to_query = list(full_serverlist)
|
||||
must_query = set(initial_servers_to_query)
|
||||
self.extra_servers = []
|
||||
elif self.mode == MODE_WRITE:
|
||||
# we're planning to replace all the shares, so we want a good
|
||||
# chance of finding them all. We will keep searching until we've
|
||||
# seen epsilon that don't have a share.
|
||||
# We don't query all of the peers because that could take a while.
|
||||
self.num_peers_to_query = N + self.EPSILON
|
||||
initial_peers_to_query, must_query = self._build_initial_querylist()
|
||||
self.required_num_empty_peers = self.EPSILON
|
||||
# We don't query all of the servers because that could take a while.
|
||||
self.num_servers_to_query = N + self.EPSILON
|
||||
initial_servers_to_query, must_query = self._build_initial_querylist()
|
||||
self.required_num_empty_servers = self.EPSILON
|
||||
|
||||
# TODO: arrange to read lots of data from k-ish servers, to avoid
|
||||
# the extra round trip required to read large directories. This
|
||||
@ -502,55 +515,49 @@ class ServermapUpdater:
|
||||
# private key.
|
||||
|
||||
else: # MODE_READ, MODE_ANYTHING
|
||||
# 2k peers is good enough.
|
||||
initial_peers_to_query, must_query = self._build_initial_querylist()
|
||||
# 2*k servers is good enough.
|
||||
initial_servers_to_query, must_query = self._build_initial_querylist()
|
||||
|
||||
# this is a set of peers that we are required to get responses from:
|
||||
# they are peers who used to have a share, so we need to know where
|
||||
# they currently stand, even if that means we have to wait for a
|
||||
# silently-lost TCP connection to time out. We remove peers from this
|
||||
# set as we get responses.
|
||||
self._must_query = must_query
|
||||
# this is a set of servers that we are required to get responses
|
||||
# from: they are servers who used to have a share, so we need to know
|
||||
# where they currently stand, even if that means we have to wait for
|
||||
# a silently-lost TCP connection to time out. We remove servers from
|
||||
# this set as we get responses.
|
||||
self._must_query = set(must_query)
|
||||
|
||||
# now initial_peers_to_query contains the peers that we should ask,
|
||||
# self.must_query contains the peers that we must have heard from
|
||||
# before we can consider ourselves finished, and self.extra_peers
|
||||
# contains the overflow (peers that we should tap if we don't get
|
||||
# enough responses)
|
||||
# now initial_servers_to_query contains the servers that we should
|
||||
# ask, self.must_query contains the servers that we must have heard
|
||||
# from before we can consider ourselves finished, and
|
||||
# self.extra_servers contains the overflow (servers that we should
|
||||
# tap if we don't get enough responses)
|
||||
# I guess that self._must_query is a subset of
|
||||
# initial_peers_to_query?
|
||||
assert set(must_query).issubset(set(initial_peers_to_query))
|
||||
# initial_servers_to_query?
|
||||
assert must_query.issubset(initial_servers_to_query)
|
||||
|
||||
self._send_initial_requests(initial_peers_to_query)
|
||||
self._send_initial_requests(initial_servers_to_query)
|
||||
self._status.timings["initial_queries"] = time.time() - self._started
|
||||
return self._done_deferred
|
||||
|
||||
def _build_initial_querylist(self):
|
||||
initial_peers_to_query = {}
|
||||
must_query = set()
|
||||
for peerid in self._servermap.all_peers():
|
||||
ss = self._servermap.connections[peerid]
|
||||
# we send queries to everyone who was already in the sharemap
|
||||
initial_peers_to_query[peerid] = ss
|
||||
# and we must wait for responses from them
|
||||
must_query.add(peerid)
|
||||
# we send queries to everyone who was already in the sharemap
|
||||
initial_servers_to_query = set(self._servermap.all_servers())
|
||||
# and we must wait for responses from them
|
||||
must_query = set(initial_servers_to_query)
|
||||
|
||||
while ((self.num_peers_to_query > len(initial_peers_to_query))
|
||||
and self.extra_peers):
|
||||
(peerid, ss) = self.extra_peers.pop(0)
|
||||
initial_peers_to_query[peerid] = ss
|
||||
while ((self.num_servers_to_query > len(initial_servers_to_query))
|
||||
and self.extra_servers):
|
||||
initial_servers_to_query.add(self.extra_servers.pop(0))
|
||||
|
||||
return initial_peers_to_query, must_query
|
||||
return initial_servers_to_query, must_query
|
||||
|
||||
def _send_initial_requests(self, peerlist):
|
||||
self._status.set_status("Sending %d initial queries" % len(peerlist))
|
||||
def _send_initial_requests(self, serverlist):
|
||||
self._status.set_status("Sending %d initial queries" % len(serverlist))
|
||||
self._queries_outstanding = set()
|
||||
self._sharemap = DictOfSets() # shnum -> [(peerid, seqnum, R)..]
|
||||
for (peerid, ss) in peerlist.items():
|
||||
self._queries_outstanding.add(peerid)
|
||||
self._do_query(ss, peerid, self._storage_index, self._read_size)
|
||||
for server in serverlist:
|
||||
self._queries_outstanding.add(server)
|
||||
self._do_query(server, self._storage_index, self._read_size)
|
||||
|
||||
if not peerlist:
|
||||
if not serverlist:
|
||||
# there is nobody to ask, so we need to short-circuit the state
|
||||
# machine.
|
||||
d = defer.maybeDeferred(self._check_for_done, None)
|
||||
@ -561,18 +568,17 @@ class ServermapUpdater:
|
||||
# might produce a result.
|
||||
return None
|
||||
|
||||
def _do_query(self, ss, peerid, storage_index, readsize):
|
||||
self.log(format="sending query to [%(peerid)s], readsize=%(readsize)d",
|
||||
peerid=idlib.shortnodeid_b2a(peerid),
|
||||
def _do_query(self, server, storage_index, readsize):
|
||||
self.log(format="sending query to [%(name)s], readsize=%(readsize)d",
|
||||
name=server.get_name(),
|
||||
readsize=readsize,
|
||||
level=log.NOISY)
|
||||
self._servermap.connections[peerid] = ss
|
||||
started = time.time()
|
||||
self._queries_outstanding.add(peerid)
|
||||
d = self._do_read(ss, peerid, storage_index, [], [(0, readsize)])
|
||||
d.addCallback(self._got_results, peerid, readsize, (ss, storage_index),
|
||||
self._queries_outstanding.add(server)
|
||||
d = self._do_read(server, storage_index, [], [(0, readsize)])
|
||||
d.addCallback(self._got_results, server, readsize, storage_index,
|
||||
started)
|
||||
d.addErrback(self._query_failed, peerid)
|
||||
d.addErrback(self._query_failed, server)
|
||||
# errors that aren't handled by _query_failed (and errors caused by
|
||||
# _query_failed) get logged, but we still want to check for doneness.
|
||||
d.addErrback(log.err)
|
||||
@ -580,24 +586,25 @@ class ServermapUpdater:
|
||||
d.addCallback(self._check_for_done)
|
||||
return d
|
||||
|
||||
def _do_read(self, ss, peerid, storage_index, shnums, readv):
|
||||
def _do_read(self, server, storage_index, shnums, readv):
|
||||
ss = server.get_rref()
|
||||
if self._add_lease:
|
||||
# send an add-lease message in parallel. The results are handled
|
||||
# separately. This is sent before the slot_readv() so that we can
|
||||
# be sure the add_lease is retired by the time slot_readv comes
|
||||
# back (this relies upon our knowledge that the server code for
|
||||
# add_lease is synchronous).
|
||||
renew_secret = self._node.get_renewal_secret(peerid)
|
||||
cancel_secret = self._node.get_cancel_secret(peerid)
|
||||
renew_secret = self._node.get_renewal_secret(server)
|
||||
cancel_secret = self._node.get_cancel_secret(server)
|
||||
d2 = ss.callRemote("add_lease", storage_index,
|
||||
renew_secret, cancel_secret)
|
||||
# we ignore success
|
||||
d2.addErrback(self._add_lease_failed, peerid, storage_index)
|
||||
d2.addErrback(self._add_lease_failed, server, storage_index)
|
||||
d = ss.callRemote("slot_readv", storage_index, shnums, readv)
|
||||
return d
|
||||
|
||||
|
||||
def _got_corrupt_share(self, e, shnum, peerid, data, lp):
|
||||
def _got_corrupt_share(self, e, shnum, server, data, lp):
|
||||
"""
|
||||
I am called when a remote server returns a corrupt share in
|
||||
response to one of our queries. By corrupt, I mean a share
|
||||
@ -608,18 +615,18 @@ class ServermapUpdater:
|
||||
self.log(format="bad share: %(f_value)s", f_value=str(f),
|
||||
failure=f, parent=lp, level=log.WEIRD, umid="h5llHg")
|
||||
# Notify the server that its share is corrupt.
|
||||
self.notify_server_corruption(peerid, shnum, str(e))
|
||||
# By flagging this as a bad peer, we won't count any of
|
||||
# the other shares on that peer as valid, though if we
|
||||
self.notify_server_corruption(server, shnum, str(e))
|
||||
# By flagging this as a bad server, we won't count any of
|
||||
# the other shares on that server as valid, though if we
|
||||
# happen to find a valid version string amongst those
|
||||
# shares, we'll keep track of it so that we don't need
|
||||
# to validate the signature on those again.
|
||||
self._bad_peers.add(peerid)
|
||||
self._bad_servers.add(server)
|
||||
self._last_failure = f
|
||||
# XXX: Use the reader for this?
|
||||
checkstring = data[:SIGNED_PREFIX_LENGTH]
|
||||
self._servermap.mark_bad_share(peerid, shnum, checkstring)
|
||||
self._servermap.problems.append(f)
|
||||
self._servermap.mark_bad_share(server, shnum, checkstring)
|
||||
self._servermap.add_problem(f)
|
||||
|
||||
|
||||
def _cache_good_sharedata(self, verinfo, shnum, now, data):
|
||||
@ -635,30 +642,30 @@ class ServermapUpdater:
|
||||
self._node._add_to_cache(verinfo, shnum, 0, data)
|
||||
|
||||
|
||||
def _got_results(self, datavs, peerid, readsize, stuff, started):
|
||||
lp = self.log(format="got result from [%(peerid)s], %(numshares)d shares",
|
||||
peerid=idlib.shortnodeid_b2a(peerid),
|
||||
def _got_results(self, datavs, server, readsize, storage_index, started):
|
||||
lp = self.log(format="got result from [%(name)s], %(numshares)d shares",
|
||||
name=server.get_name(),
|
||||
numshares=len(datavs))
|
||||
ss = server.get_rref()
|
||||
now = time.time()
|
||||
elapsed = now - started
|
||||
def _done_processing(ignored=None):
|
||||
self._queries_outstanding.discard(peerid)
|
||||
self._servermap.reachable_peers.add(peerid)
|
||||
self._must_query.discard(peerid)
|
||||
self._queries_outstanding.discard(server)
|
||||
self._servermap.mark_server_reachable(server)
|
||||
self._must_query.discard(server)
|
||||
self._queries_completed += 1
|
||||
if not self._running:
|
||||
self.log("but we're not running, so we'll ignore it", parent=lp)
|
||||
_done_processing()
|
||||
self._status.add_per_server_time(peerid, "late", started, elapsed)
|
||||
self._status.add_per_server_time(server, "late", started, elapsed)
|
||||
return
|
||||
self._status.add_per_server_time(peerid, "query", started, elapsed)
|
||||
self._status.add_per_server_time(server, "query", started, elapsed)
|
||||
|
||||
if datavs:
|
||||
self._good_peers.add(peerid)
|
||||
self._good_servers.add(server)
|
||||
else:
|
||||
self._empty_peers.add(peerid)
|
||||
self._empty_servers.add(server)
|
||||
|
||||
ss, storage_index = stuff
|
||||
ds = []
|
||||
|
||||
for shnum,datav in datavs.items():
|
||||
@ -667,7 +674,6 @@ class ServermapUpdater:
|
||||
storage_index,
|
||||
shnum,
|
||||
data)
|
||||
self._readers.setdefault(peerid, dict())[shnum] = reader
|
||||
# our goal, with each response, is to validate the version
|
||||
# information and share data as best we can at this point --
|
||||
# we do this by validating the signature. To do this, we
|
||||
@ -677,11 +683,11 @@ class ServermapUpdater:
|
||||
if not self._node.get_pubkey():
|
||||
# fetch and set the public key.
|
||||
d = reader.get_verification_key()
|
||||
d.addCallback(lambda results, shnum=shnum, peerid=peerid:
|
||||
self._try_to_set_pubkey(results, peerid, shnum, lp))
|
||||
d.addCallback(lambda results, shnum=shnum:
|
||||
self._try_to_set_pubkey(results, server, shnum, lp))
|
||||
# XXX: Make self._pubkey_query_failed?
|
||||
d.addErrback(lambda error, shnum=shnum, peerid=peerid, data=data:
|
||||
self._got_corrupt_share(error, shnum, peerid, data, lp))
|
||||
d.addErrback(lambda error, shnum=shnum, data=data:
|
||||
self._got_corrupt_share(error, shnum, server, data, lp))
|
||||
else:
|
||||
# we already have the public key.
|
||||
d = defer.succeed(None)
|
||||
@ -695,16 +701,16 @@ class ServermapUpdater:
|
||||
# bytes of the share on the storage server, so we
|
||||
# shouldn't need to fetch anything at this step.
|
||||
d2 = reader.get_verinfo()
|
||||
d2.addErrback(lambda error, shnum=shnum, peerid=peerid, data=data:
|
||||
self._got_corrupt_share(error, shnum, peerid, data, lp))
|
||||
d2.addErrback(lambda error, shnum=shnum, data=data:
|
||||
self._got_corrupt_share(error, shnum, server, data, lp))
|
||||
# - Next, we need the signature. For an SDMF share, it is
|
||||
# likely that we fetched this when doing our initial fetch
|
||||
# to get the version information. In MDMF, this lives at
|
||||
# the end of the share, so unless the file is quite small,
|
||||
# we'll need to do a remote fetch to get it.
|
||||
d3 = reader.get_signature()
|
||||
d3.addErrback(lambda error, shnum=shnum, peerid=peerid, data=data:
|
||||
self._got_corrupt_share(error, shnum, peerid, data, lp))
|
||||
d3.addErrback(lambda error, shnum=shnum, data=data:
|
||||
self._got_corrupt_share(error, shnum, server, data, lp))
|
||||
# Once we have all three of these responses, we can move on
|
||||
# to validating the signature
|
||||
|
||||
@ -712,10 +718,10 @@ class ServermapUpdater:
|
||||
# fetch it here.
|
||||
if self._need_privkey:
|
||||
d4 = reader.get_encprivkey()
|
||||
d4.addCallback(lambda results, shnum=shnum, peerid=peerid:
|
||||
self._try_to_validate_privkey(results, peerid, shnum, lp))
|
||||
d4.addErrback(lambda error, shnum=shnum, peerid=peerid, data=data:
|
||||
self._privkey_query_failed(error, shnum, data, lp))
|
||||
d4.addCallback(lambda results, shnum=shnum:
|
||||
self._try_to_validate_privkey(results, server, shnum, lp))
|
||||
d4.addErrback(lambda error, shnum=shnum:
|
||||
self._privkey_query_failed(error, server, shnum, lp))
|
||||
else:
|
||||
d4 = defer.succeed(None)
|
||||
|
||||
@ -740,15 +746,15 @@ class ServermapUpdater:
|
||||
|
||||
dl = defer.DeferredList([d, d2, d3, d4, d5])
|
||||
dl.addBoth(self._turn_barrier)
|
||||
dl.addCallback(lambda results, shnum=shnum, peerid=peerid:
|
||||
self._got_signature_one_share(results, shnum, peerid, lp))
|
||||
dl.addCallback(lambda results, shnum=shnum:
|
||||
self._got_signature_one_share(results, shnum, server, lp))
|
||||
dl.addErrback(lambda error, shnum=shnum, data=data:
|
||||
self._got_corrupt_share(error, shnum, peerid, data, lp))
|
||||
dl.addCallback(lambda verinfo, shnum=shnum, peerid=peerid, data=data:
|
||||
self._cache_good_sharedata(verinfo, shnum, now, data))
|
||||
self._got_corrupt_share(error, shnum, server, data, lp))
|
||||
dl.addCallback(lambda verinfo, shnum=shnum, data=data:
|
||||
self._cache_good_sharedata(verinfo, shnum, now, data))
|
||||
ds.append(dl)
|
||||
# dl is a deferred list that will fire when all of the shares
|
||||
# that we found on this peer are done processing. When dl fires,
|
||||
# that we found on this server are done processing. When dl fires,
|
||||
# we know that processing is done, so we can decrement the
|
||||
# semaphore-like thing that we incremented earlier.
|
||||
dl = defer.DeferredList(ds, fireOnOneErrback=True)
|
||||
@ -775,31 +781,31 @@ class ServermapUpdater:
|
||||
return fireEventually(result)
|
||||
|
||||
|
||||
def _try_to_set_pubkey(self, pubkey_s, peerid, shnum, lp):
|
||||
def _try_to_set_pubkey(self, pubkey_s, server, shnum, lp):
|
||||
if self._node.get_pubkey():
|
||||
return # don't go through this again if we don't have to
|
||||
fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s)
|
||||
assert len(fingerprint) == 32
|
||||
if fingerprint != self._node.get_fingerprint():
|
||||
raise CorruptShareError(peerid, shnum,
|
||||
"pubkey doesn't match fingerprint")
|
||||
raise CorruptShareError(server, shnum,
|
||||
"pubkey doesn't match fingerprint")
|
||||
self._node._populate_pubkey(self._deserialize_pubkey(pubkey_s))
|
||||
assert self._node.get_pubkey()
|
||||
|
||||
|
||||
def notify_server_corruption(self, peerid, shnum, reason):
|
||||
ss = self._servermap.connections[peerid]
|
||||
ss.callRemoteOnly("advise_corrupt_share",
|
||||
"mutable", self._storage_index, shnum, reason)
|
||||
def notify_server_corruption(self, server, shnum, reason):
|
||||
rref = server.get_rref()
|
||||
rref.callRemoteOnly("advise_corrupt_share",
|
||||
"mutable", self._storage_index, shnum, reason)
|
||||
|
||||
|
||||
def _got_signature_one_share(self, results, shnum, peerid, lp):
|
||||
def _got_signature_one_share(self, results, shnum, server, lp):
|
||||
# It is our job to give versioninfo to our caller. We need to
|
||||
# raise CorruptShareError if the share is corrupt for any
|
||||
# reason, something that our caller will handle.
|
||||
self.log(format="_got_results: got shnum #%(shnum)d from peerid %(peerid)s",
|
||||
self.log(format="_got_results: got shnum #%(shnum)d from serverid %(name)s",
|
||||
shnum=shnum,
|
||||
peerid=idlib.shortnodeid_b2a(peerid),
|
||||
name=server.get_name(),
|
||||
level=log.NOISY,
|
||||
parent=lp)
|
||||
if not self._running:
|
||||
@ -840,14 +846,14 @@ class ServermapUpdater:
|
||||
assert self._node.get_pubkey()
|
||||
valid = self._node.get_pubkey().verify(prefix, signature[1])
|
||||
if not valid:
|
||||
raise CorruptShareError(peerid, shnum,
|
||||
raise CorruptShareError(server, shnum,
|
||||
"signature is invalid")
|
||||
|
||||
# ok, it's a valid verinfo. Add it to the list of validated
|
||||
# versions.
|
||||
self.log(" found valid version %d-%s from %s-sh%d: %d-%d/%d/%d"
|
||||
% (seqnum, base32.b2a(root_hash)[:4],
|
||||
idlib.shortnodeid_b2a(peerid), shnum,
|
||||
server.get_name(), shnum,
|
||||
k, n, segsize, datalen),
|
||||
parent=lp)
|
||||
self._valid_versions.add(verinfo)
|
||||
@ -857,8 +863,8 @@ class ServermapUpdater:
|
||||
# version info again, that its signature checks out and that
|
||||
# we're okay to skip the signature-checking step.
|
||||
|
||||
# (peerid, shnum) are bound in the method invocation.
|
||||
if (peerid, shnum) in self._servermap.bad_shares:
|
||||
# (server, shnum) are bound in the method invocation.
|
||||
if (server, shnum) in self._servermap.get_bad_shares():
|
||||
# we've been told that the rest of the data in this share is
|
||||
# unusable, so don't add it to the servermap.
|
||||
self.log("but we've been told this is a bad share",
|
||||
@ -867,9 +873,7 @@ class ServermapUpdater:
|
||||
|
||||
# Add the info to our servermap.
|
||||
timestamp = time.time()
|
||||
self._servermap.add_new_share(peerid, shnum, verinfo, timestamp)
|
||||
# and the versionmap
|
||||
self.versionmap.add(verinfo, (shnum, peerid, timestamp))
|
||||
self._servermap.add_new_share(server, shnum, verinfo, timestamp)
|
||||
|
||||
return verinfo
|
||||
|
||||
@ -914,7 +918,7 @@ class ServermapUpdater:
|
||||
return verifier
|
||||
|
||||
|
||||
def _try_to_validate_privkey(self, enc_privkey, peerid, shnum, lp):
|
||||
def _try_to_validate_privkey(self, enc_privkey, server, shnum, lp):
|
||||
"""
|
||||
Given a writekey from a remote server, I validate it against the
|
||||
writekey stored in my node. If it is valid, then I set the
|
||||
@ -924,22 +928,22 @@ class ServermapUpdater:
|
||||
alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s)
|
||||
if alleged_writekey != self._node.get_writekey():
|
||||
self.log("invalid privkey from %s shnum %d" %
|
||||
(idlib.nodeid_b2a(peerid)[:8], shnum),
|
||||
(server.get_name(), shnum),
|
||||
parent=lp, level=log.WEIRD, umid="aJVccw")
|
||||
return
|
||||
|
||||
# it's good
|
||||
self.log("got valid privkey from shnum %d on peerid %s" %
|
||||
(shnum, idlib.shortnodeid_b2a(peerid)),
|
||||
self.log("got valid privkey from shnum %d on serverid %s" %
|
||||
(shnum, server.get_name()),
|
||||
parent=lp)
|
||||
privkey = rsa.create_signing_key_from_string(alleged_privkey_s)
|
||||
self._node._populate_encprivkey(enc_privkey)
|
||||
self._node._populate_privkey(privkey)
|
||||
self._need_privkey = False
|
||||
self._status.set_privkey_from(peerid)
|
||||
self._status.set_privkey_from(server)
|
||||
|
||||
|
||||
def _add_lease_failed(self, f, peerid, storage_index):
|
||||
def _add_lease_failed(self, f, server, storage_index):
|
||||
# Older versions of Tahoe didn't handle the add-lease message very
|
||||
# well: <=1.1.0 throws a NameError because it doesn't implement
|
||||
# remote_add_lease(), 1.2.0/1.3.0 throw IndexError on unknown buckets
|
||||
@ -959,20 +963,20 @@ class ServermapUpdater:
|
||||
# this may ignore a bit too much, but that only hurts us
|
||||
# during debugging
|
||||
return
|
||||
self.log(format="error in add_lease from [%(peerid)s]: %(f_value)s",
|
||||
peerid=idlib.shortnodeid_b2a(peerid),
|
||||
self.log(format="error in add_lease from [%(name)s]: %(f_value)s",
|
||||
name=server.get_name(),
|
||||
f_value=str(f.value),
|
||||
failure=f,
|
||||
level=log.WEIRD, umid="iqg3mw")
|
||||
return
|
||||
# local errors are cause for alarm
|
||||
log.err(f,
|
||||
format="local error in add_lease to [%(peerid)s]: %(f_value)s",
|
||||
peerid=idlib.shortnodeid_b2a(peerid),
|
||||
format="local error in add_lease to [%(name)s]: %(f_value)s",
|
||||
name=server.get_name(),
|
||||
f_value=str(f.value),
|
||||
level=log.WEIRD, umid="ZWh6HA")
|
||||
|
||||
def _query_failed(self, f, peerid):
|
||||
def _query_failed(self, f, server):
|
||||
if not self._running:
|
||||
return
|
||||
level = log.WEIRD
|
||||
@ -981,20 +985,20 @@ class ServermapUpdater:
|
||||
self.log(format="error during query: %(f_value)s",
|
||||
f_value=str(f.value), failure=f,
|
||||
level=level, umid="IHXuQg")
|
||||
self._must_query.discard(peerid)
|
||||
self._queries_outstanding.discard(peerid)
|
||||
self._bad_peers.add(peerid)
|
||||
self._servermap.problems.append(f)
|
||||
# a peerid could be in both ServerMap.reachable_peers and
|
||||
# .unreachable_peers if they responded to our query, but then an
|
||||
self._must_query.discard(server)
|
||||
self._queries_outstanding.discard(server)
|
||||
self._bad_servers.add(server)
|
||||
self._servermap.add_problem(f)
|
||||
# a server could be in both ServerMap.reachable_servers and
|
||||
# .unreachable_servers if they responded to our query, but then an
|
||||
# exception was raised in _got_results.
|
||||
self._servermap.unreachable_peers.add(peerid)
|
||||
self._servermap.mark_server_unreachable(server)
|
||||
self._queries_completed += 1
|
||||
self._last_failure = f
|
||||
|
||||
|
||||
def _privkey_query_failed(self, f, peerid, shnum, lp):
|
||||
self._queries_outstanding.discard(peerid)
|
||||
def _privkey_query_failed(self, f, server, shnum, lp):
|
||||
self._queries_outstanding.discard(server)
|
||||
if not self._running:
|
||||
return
|
||||
level = log.WEIRD
|
||||
@ -1003,7 +1007,7 @@ class ServermapUpdater:
|
||||
self.log(format="error during privkey query: %(f_value)s",
|
||||
f_value=str(f.value), failure=f,
|
||||
parent=lp, level=level, umid="McoJ5w")
|
||||
self._servermap.problems.append(f)
|
||||
self._servermap.add_problem(f)
|
||||
self._last_failure = f
|
||||
|
||||
|
||||
@ -1014,13 +1018,13 @@ class ServermapUpdater:
|
||||
# return : keep waiting, no new queries
|
||||
lp = self.log(format=("_check_for_done, mode is '%(mode)s', "
|
||||
"%(outstanding)d queries outstanding, "
|
||||
"%(extra)d extra peers available, "
|
||||
"%(must)d 'must query' peers left, "
|
||||
"%(extra)d extra servers available, "
|
||||
"%(must)d 'must query' servers left, "
|
||||
"need_privkey=%(need_privkey)s"
|
||||
),
|
||||
mode=self.mode,
|
||||
outstanding=len(self._queries_outstanding),
|
||||
extra=len(self.extra_peers),
|
||||
extra=len(self.extra_servers),
|
||||
must=len(self._must_query),
|
||||
need_privkey=self._need_privkey,
|
||||
level=log.NOISY,
|
||||
@ -1031,17 +1035,17 @@ class ServermapUpdater:
|
||||
return
|
||||
|
||||
if self._must_query:
|
||||
# we are still waiting for responses from peers that used to have
|
||||
# we are still waiting for responses from servers that used to have
|
||||
# a share, so we must continue to wait. No additional queries are
|
||||
# required at this time.
|
||||
self.log("%d 'must query' peers left" % len(self._must_query),
|
||||
self.log("%d 'must query' servers left" % len(self._must_query),
|
||||
level=log.NOISY, parent=lp)
|
||||
return
|
||||
|
||||
if (not self._queries_outstanding and not self.extra_peers):
|
||||
# all queries have retired, and we have no peers left to ask. No
|
||||
if (not self._queries_outstanding and not self.extra_servers):
|
||||
# all queries have retired, and we have no servers left to ask. No
|
||||
# more progress can be made, therefore we are done.
|
||||
self.log("all queries are retired, no extra peers: done",
|
||||
self.log("all queries are retired, no extra servers: done",
|
||||
parent=lp)
|
||||
return self._done()
|
||||
|
||||
@ -1069,10 +1073,10 @@ class ServermapUpdater:
|
||||
# version, and we haven't seen any unrecoverable higher-seqnum'ed
|
||||
# versions, then we're done.
|
||||
|
||||
if self._queries_completed < self.num_peers_to_query:
|
||||
if self._queries_completed < self.num_servers_to_query:
|
||||
self.log(format="%(completed)d completed, %(query)d to query: need more",
|
||||
completed=self._queries_completed,
|
||||
query=self.num_peers_to_query,
|
||||
query=self.num_servers_to_query,
|
||||
level=log.NOISY, parent=lp)
|
||||
return self._send_more_queries(MAX_IN_FLIGHT)
|
||||
if not recoverable_versions:
|
||||
@ -1113,15 +1117,15 @@ class ServermapUpdater:
|
||||
states = []
|
||||
found_boundary = False
|
||||
|
||||
for i,(peerid,ss) in enumerate(self.full_peerlist):
|
||||
if peerid in self._bad_peers:
|
||||
for i,server in enumerate(self.full_serverlist):
|
||||
if server in self._bad_servers:
|
||||
# query failed
|
||||
states.append("x")
|
||||
#self.log("loop [%s]: x" % idlib.shortnodeid_b2a(peerid))
|
||||
elif peerid in self._empty_peers:
|
||||
#self.log("loop [%s]: x" % server.get_name()
|
||||
elif server in self._empty_servers:
|
||||
# no shares
|
||||
states.append("0")
|
||||
#self.log("loop [%s]: 0" % idlib.shortnodeid_b2a(peerid))
|
||||
#self.log("loop [%s]: 0" % server.get_name()
|
||||
if last_found != -1:
|
||||
num_not_found += 1
|
||||
if num_not_found >= self.EPSILON:
|
||||
@ -1131,16 +1135,16 @@ class ServermapUpdater:
|
||||
found_boundary = True
|
||||
break
|
||||
|
||||
elif peerid in self._good_peers:
|
||||
elif server in self._good_servers:
|
||||
# yes shares
|
||||
states.append("1")
|
||||
#self.log("loop [%s]: 1" % idlib.shortnodeid_b2a(peerid))
|
||||
#self.log("loop [%s]: 1" % server.get_name()
|
||||
last_found = i
|
||||
num_not_found = 0
|
||||
else:
|
||||
# not responded yet
|
||||
states.append("?")
|
||||
#self.log("loop [%s]: ?" % idlib.shortnodeid_b2a(peerid))
|
||||
#self.log("loop [%s]: ?" % server.get_name()
|
||||
last_not_responded = i
|
||||
num_not_responded += 1
|
||||
|
||||
@ -1166,7 +1170,7 @@ class ServermapUpdater:
|
||||
return self._send_more_queries(num_not_responded)
|
||||
|
||||
# if we hit here, we didn't find our boundary, so we're still
|
||||
# waiting for peers
|
||||
# waiting for servers
|
||||
self.log("no boundary yet, %s" % "".join(states), parent=lp,
|
||||
level=log.NOISY)
|
||||
return self._send_more_queries(MAX_IN_FLIGHT)
|
||||
@ -1187,18 +1191,17 @@ class ServermapUpdater:
|
||||
active_queries = len(self._queries_outstanding) + len(more_queries)
|
||||
if active_queries >= num_outstanding:
|
||||
break
|
||||
if not self.extra_peers:
|
||||
if not self.extra_servers:
|
||||
break
|
||||
more_queries.append(self.extra_peers.pop(0))
|
||||
more_queries.append(self.extra_servers.pop(0))
|
||||
|
||||
self.log(format="sending %(more)d more queries: %(who)s",
|
||||
more=len(more_queries),
|
||||
who=" ".join(["[%s]" % idlib.shortnodeid_b2a(peerid)
|
||||
for (peerid,ss) in more_queries]),
|
||||
who=" ".join(["[%s]" % s.get_name() for s in more_queries]),
|
||||
level=log.NOISY)
|
||||
|
||||
for (peerid, ss) in more_queries:
|
||||
self._do_query(ss, peerid, self._storage_index, self._read_size)
|
||||
for server in more_queries:
|
||||
self._do_query(server, self._storage_index, self._read_size)
|
||||
# we'll retrigger when those queries come back
|
||||
|
||||
def _done(self):
|
||||
@ -1214,8 +1217,7 @@ class ServermapUpdater:
|
||||
self._status.set_status("Finished")
|
||||
self._status.set_active(False)
|
||||
|
||||
self._servermap.last_update_mode = self.mode
|
||||
self._servermap.last_update_time = self._started
|
||||
self._servermap.set_last_update(self.mode, self._started)
|
||||
# the servermap will not be touched after this
|
||||
self.log("servermap: %s" % self._servermap.summarize_versions())
|
||||
|
||||
|
||||
@ -137,7 +137,6 @@ class StorageFarmBroker:
|
||||
return self.servers[serverid].get_nickname()
|
||||
return None
|
||||
|
||||
|
||||
class IServer(Interface):
|
||||
"""I live in the client, and represent a single server."""
|
||||
def start_connecting(tub, trigger_cb):
|
||||
|
||||
@ -14,7 +14,6 @@ from allmydata.interfaces import IMutableFileNode, IImmutableFileNode,\
|
||||
MDMF_VERSION
|
||||
from allmydata.check_results import CheckResults, CheckAndRepairResults, \
|
||||
DeepCheckResults, DeepCheckAndRepairResults
|
||||
from allmydata.mutable.common import CorruptShareError
|
||||
from allmydata.mutable.layout import unpack_header
|
||||
from allmydata.mutable.publish import MutableData
|
||||
from allmydata.storage.mutable import MutableShareFile
|
||||
@ -86,7 +85,10 @@ class FakeCHKFileNode:
|
||||
r.set_recoverable(True)
|
||||
data["count-shares-good"] = 9
|
||||
data["list-corrupt-shares"] = [(nodeid, self.storage_index, 0)]
|
||||
r.problems = failure.Failure(CorruptShareError(is_bad))
|
||||
# XXX: this whole 'is_bad' clause is unused. When a test is added
|
||||
# to take advantage of it, we must find a way to provide 'server'
|
||||
# (and IServer instance) to the CorruptShareError
|
||||
#r.problems = failure.Failure(CorruptShareError(server, 0, is_bad))
|
||||
else:
|
||||
r.set_healthy(True)
|
||||
r.set_recoverable(True)
|
||||
@ -301,9 +303,10 @@ class FakeMutableFileNode:
|
||||
r.set_healthy(False)
|
||||
r.set_recoverable(True)
|
||||
data["count-shares-good"] = 9
|
||||
r.problems = failure.Failure(CorruptShareError("peerid",
|
||||
0, # shnum
|
||||
is_bad))
|
||||
# XXX: this whole 'is_bad' clause is unused. When a test is added
|
||||
# to take advantage of it, we must find a way to provide 'server'
|
||||
# (and IServer instance) to the CorruptShareError
|
||||
#r.problems = failure.Failure(CorruptShareError(server, 0, is_bad))
|
||||
else:
|
||||
r.set_healthy(True)
|
||||
r.set_recoverable(True)
|
||||
|
||||
@ -131,6 +131,9 @@ class NoNetworkServer:
|
||||
return self.serverid
|
||||
def get_lease_seed(self):
|
||||
return self.serverid
|
||||
def get_foolscap_write_enabler_seed(self):
|
||||
return self.serverid
|
||||
|
||||
def get_name(self):
|
||||
return idlib.shortnodeid_b2a(self.serverid)
|
||||
def get_longname(self):
|
||||
|
||||
@ -1043,10 +1043,10 @@ class Servermap(unittest.TestCase, PublishMixin):
|
||||
self.failUnlessEqual(sm.recoverable_versions(), set([best]))
|
||||
self.failUnlessEqual(len(sm.shares_available()), 1)
|
||||
self.failUnlessEqual(sm.shares_available()[best], (num_shares, 3, 10))
|
||||
shnum, peerids = sm.make_sharemap().items()[0]
|
||||
peerid = list(peerids)[0]
|
||||
self.failUnlessEqual(sm.version_on_peer(peerid, shnum), best)
|
||||
self.failUnlessEqual(sm.version_on_peer(peerid, 666), None)
|
||||
shnum, servers = sm.make_sharemap().items()[0]
|
||||
server = list(servers)[0]
|
||||
self.failUnlessEqual(sm.version_on_server(server, shnum), best)
|
||||
self.failUnlessEqual(sm.version_on_server(server, 666), None)
|
||||
return sm
|
||||
|
||||
def test_basic(self):
|
||||
@ -1116,10 +1116,10 @@ class Servermap(unittest.TestCase, PublishMixin):
|
||||
# mark the first 5 shares as corrupt, then update the servermap.
|
||||
# The map should not have the marked shares it in any more, and
|
||||
# new shares should be found to replace the missing ones.
|
||||
for (shnum, peerid, timestamp) in shares:
|
||||
for (shnum, server, timestamp) in shares:
|
||||
if shnum < 5:
|
||||
self._corrupted.add( (peerid, shnum) )
|
||||
sm.mark_bad_share(peerid, shnum, "")
|
||||
self._corrupted.add( (server, shnum) )
|
||||
sm.mark_bad_share(server, shnum, "")
|
||||
return self.update_servermap(sm, MODE_WRITE)
|
||||
d.addCallback(_made_map)
|
||||
def _check_map(sm):
|
||||
@ -1127,10 +1127,10 @@ class Servermap(unittest.TestCase, PublishMixin):
|
||||
v = sm.best_recoverable_version()
|
||||
vm = sm.make_versionmap()
|
||||
shares = list(vm[v])
|
||||
for (peerid, shnum) in self._corrupted:
|
||||
peer_shares = sm.shares_on_peer(peerid)
|
||||
self.failIf(shnum in peer_shares,
|
||||
"%d was in %s" % (shnum, peer_shares))
|
||||
for (server, shnum) in self._corrupted:
|
||||
server_shares = sm.debug_shares_on_server(server)
|
||||
self.failIf(shnum in server_shares,
|
||||
"%d was in %s" % (shnum, server_shares))
|
||||
self.failUnlessEqual(len(shares), 5)
|
||||
d.addCallback(_check_map)
|
||||
return d
|
||||
@ -1280,7 +1280,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin):
|
||||
def do_download(self, servermap, version=None):
|
||||
if version is None:
|
||||
version = servermap.best_recoverable_version()
|
||||
r = Retrieve(self._fn, servermap, version)
|
||||
r = Retrieve(self._fn, self._storage_broker, servermap, version)
|
||||
c = consumer.MemoryConsumer()
|
||||
d = r.download(consumer=c)
|
||||
d.addCallback(lambda mc: "".join(mc.chunks))
|
||||
@ -1321,7 +1321,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin):
|
||||
shares.clear()
|
||||
d1 = self.shouldFail(NotEnoughSharesError,
|
||||
"test_all_shares_vanished",
|
||||
"ran out of peers",
|
||||
"ran out of servers",
|
||||
self.do_download, servermap)
|
||||
return d1
|
||||
d.addCallback(_remove_shares)
|
||||
@ -1336,7 +1336,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin):
|
||||
self.failUnlessEqual(servermap.best_recoverable_version(), None)
|
||||
self.failIf(servermap.recoverable_versions())
|
||||
self.failIf(servermap.unrecoverable_versions())
|
||||
self.failIf(servermap.all_peers())
|
||||
self.failIf(servermap.all_servers())
|
||||
d.addCallback(_check_servermap)
|
||||
return d
|
||||
|
||||
@ -1378,7 +1378,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin):
|
||||
# no recoverable versions == not succeeding. The problem
|
||||
# should be noted in the servermap's list of problems.
|
||||
if substring:
|
||||
allproblems = [str(f) for f in servermap.problems]
|
||||
allproblems = [str(f) for f in servermap.get_problems()]
|
||||
self.failUnlessIn(substring, "".join(allproblems))
|
||||
return servermap
|
||||
if should_succeed:
|
||||
@ -1502,7 +1502,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin):
|
||||
f = res[0]
|
||||
self.failUnless(f.check(NotEnoughSharesError))
|
||||
self.failUnless("uncoordinated write" in str(f))
|
||||
return self._test_corrupt_all(1, "ran out of peers",
|
||||
return self._test_corrupt_all(1, "ran out of servers",
|
||||
corrupt_early=False,
|
||||
failure_checker=_check)
|
||||
|
||||
@ -1540,11 +1540,11 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin):
|
||||
shnums_to_corrupt=range(0, N-k))
|
||||
d.addCallback(lambda res: self.make_servermap())
|
||||
def _do_retrieve(servermap):
|
||||
self.failUnless(servermap.problems)
|
||||
self.failUnless(servermap.get_problems())
|
||||
self.failUnless("pubkey doesn't match fingerprint"
|
||||
in str(servermap.problems[0]))
|
||||
in str(servermap.get_problems()[0]))
|
||||
ver = servermap.best_recoverable_version()
|
||||
r = Retrieve(self._fn, servermap, ver)
|
||||
r = Retrieve(self._fn, self._storage_broker, servermap, ver)
|
||||
c = consumer.MemoryConsumer()
|
||||
return r.download(c)
|
||||
d.addCallback(_do_retrieve)
|
||||
@ -2488,7 +2488,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
|
||||
d.addCallback(lambda res:
|
||||
self.shouldFail(NotEnoughSharesError,
|
||||
"test_retrieve_surprise",
|
||||
"ran out of peers: have 0 of 1",
|
||||
"ran out of servers: have 0 of 1",
|
||||
n.download_version,
|
||||
self.old_map,
|
||||
self.old_map.best_recoverable_version(),
|
||||
@ -2515,7 +2515,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
|
||||
# stash the old state of the file
|
||||
self.old_map = smap
|
||||
# now shut down one of the servers
|
||||
peer0 = list(smap.make_sharemap()[0])[0]
|
||||
peer0 = list(smap.make_sharemap()[0])[0].get_serverid()
|
||||
self.g.remove_server(peer0)
|
||||
# then modify the file, leaving the old map untouched
|
||||
log.msg("starting winning write")
|
||||
|
||||
@ -718,7 +718,7 @@ class RetrieveStatusPage(rend.Page, RateAndTimeMixin):
|
||||
return ctx.tag["Encoding: %s of %s" % (k, n)]
|
||||
|
||||
def render_problems(self, ctx, data):
|
||||
problems = data.problems
|
||||
problems = data.get_problems()
|
||||
if not problems:
|
||||
return ""
|
||||
l = T.ul()
|
||||
@ -814,15 +814,17 @@ class PublishStatusPage(rend.Page, RateAndTimeMixin):
|
||||
sharemap = servermap.make_sharemap()
|
||||
for shnum in sorted(sharemap.keys()):
|
||||
l[T.li["%d -> Placed on " % shnum,
|
||||
", ".join(["[%s]" % idlib.shortnodeid_b2a(peerid)
|
||||
for peerid in sharemap[shnum]])]]
|
||||
", ".join(["[%s]" % server.get_name()
|
||||
for server in sharemap[shnum]])]]
|
||||
return ctx.tag["Sharemap:", l]
|
||||
|
||||
def render_problems(self, ctx, data):
|
||||
problems = data.problems
|
||||
problems = data.get_problems()
|
||||
if not problems:
|
||||
return ""
|
||||
l = T.ul()
|
||||
# XXX: is this exercised? I don't think PublishStatus.problems is
|
||||
# ever populated
|
||||
for peerid in sorted(problems.keys()):
|
||||
peerid_s = idlib.shortnodeid_b2a(peerid)
|
||||
l[T.li["[%s]: %s" % (peerid_s, problems[peerid])]]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user