From 55221d4532fe051e01e67c21e53507feda5f7feb Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Thu, 9 Sep 2021 01:50:21 +0100 Subject: [PATCH 01/87] replaced testools.unittest.TestCase with common base case Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_checker.py | 5 +++-- src/allmydata/test/mutable/test_datahandle.py | 6 ++++-- src/allmydata/test/mutable/test_different_encoding.py | 5 +++-- src/allmydata/test/mutable/test_exceptions.py | 5 +++-- src/allmydata/test/mutable/test_filehandle.py | 6 ++++-- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/allmydata/test/mutable/test_checker.py b/src/allmydata/test/mutable/test_checker.py index 11ba776fd..6d9145d68 100644 --- a/src/allmydata/test/mutable/test_checker.py +++ b/src/allmydata/test/mutable/test_checker.py @@ -10,14 +10,15 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import AsyncTestCase from foolscap.api import flushEventualQueue from allmydata.monitor import Monitor from allmydata.mutable.common import CorruptShareError from .util import PublishMixin, corrupt, CheckerMixin -class Checker(unittest.TestCase, CheckerMixin, PublishMixin): +class Checker(AsyncTestCase, CheckerMixin, PublishMixin): def setUp(self): + super(Checker, self).setUp() return self.publish_one() diff --git a/src/allmydata/test/mutable/test_datahandle.py b/src/allmydata/test/mutable/test_datahandle.py index 1819cba01..53e2983d1 100644 --- a/src/allmydata/test/mutable/test_datahandle.py +++ b/src/allmydata/test/mutable/test_datahandle.py @@ -10,11 +10,13 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import SyncTestCase from allmydata.mutable.publish import MutableData -class DataHandle(unittest.TestCase): + +class DataHandle(SyncTestCase): def setUp(self): + super(DataHandle, self).setUp() self.test_data = b"Test Data" * 50000 self.uploadable = MutableData(self.test_data) diff --git a/src/allmydata/test/mutable/test_different_encoding.py b/src/allmydata/test/mutable/test_different_encoding.py index a5165532c..f1796d373 100644 --- a/src/allmydata/test/mutable/test_different_encoding.py +++ b/src/allmydata/test/mutable/test_different_encoding.py @@ -10,11 +10,12 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import AsyncTestCase from .util import FakeStorage, make_nodemaker -class DifferentEncoding(unittest.TestCase): +class DifferentEncoding(AsyncTestCase): def setUp(self): + super(DifferentEncoding, self).setUp() self._storage = s = FakeStorage() self.nodemaker = make_nodemaker(s) diff --git a/src/allmydata/test/mutable/test_exceptions.py b/src/allmydata/test/mutable/test_exceptions.py index 6a9b2b575..aa2b56b86 100644 --- a/src/allmydata/test/mutable/test_exceptions.py +++ b/src/allmydata/test/mutable/test_exceptions.py @@ -11,10 +11,11 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import SyncTestCase from allmydata.mutable.common import NeedMoreDataError, UncoordinatedWriteError -class Exceptions(unittest.TestCase): + +class Exceptions(SyncTestCase): def test_repr(self): nmde = NeedMoreDataError(100, 50, 100) self.failUnless("NeedMoreDataError" in repr(nmde), repr(nmde)) diff --git a/src/allmydata/test/mutable/test_filehandle.py b/src/allmydata/test/mutable/test_filehandle.py index 8db02f3fd..795f60654 100644 --- a/src/allmydata/test/mutable/test_filehandle.py +++ b/src/allmydata/test/mutable/test_filehandle.py @@ -12,11 +12,13 @@ if PY2: import os from io import BytesIO -from twisted.trial import unittest +from ..common import SyncTestCase from allmydata.mutable.publish import MutableFileHandle -class FileHandle(unittest.TestCase): + +class FileHandle(SyncTestCase): def setUp(self): + super(FileHandle, self).setUp() self.test_data = b"Test Data" * 50000 self.sio = BytesIO(self.test_data) self.uploadable = MutableFileHandle(self.sio) From bbbc8592f09d8f7bac1f434dab6b305f6b58ea54 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Thu, 9 Sep 2021 14:41:06 +0100 Subject: [PATCH 02/87] removed deprecated methods, already refactored mutable files Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_datahandle.py | 11 +-- src/allmydata/test/mutable/test_exceptions.py | 5 +- .../test/mutable/test_interoperability.py | 9 +-- .../test/mutable/test_multiple_encodings.py | 8 ++- .../test/mutable/test_multiple_versions.py | 38 ++++++----- src/allmydata/test/mutable/test_problems.py | 19 +++--- src/allmydata/test/mutable/test_repair.py | 67 ++++++++++--------- 7 files changed, 83 insertions(+), 74 deletions(-) diff --git a/src/allmydata/test/mutable/test_datahandle.py b/src/allmydata/test/mutable/test_datahandle.py index 53e2983d1..7aabcd8e1 100644 --- a/src/allmydata/test/mutable/test_datahandle.py +++ b/src/allmydata/test/mutable/test_datahandle.py @@ -12,6 +12,7 @@ if PY2: from ..common import SyncTestCase from allmydata.mutable.publish import MutableData +from testtools.matchers import Equals, HasLength class DataHandle(SyncTestCase): @@ -28,13 +29,13 @@ class DataHandle(SyncTestCase): data = b"".join(data) start = i end = i + chunk_size - self.failUnlessEqual(data, self.test_data[start:end]) + self.assertThat(data, Equals(self.test_data[start:end])) def test_datahandle_get_size(self): actual_size = len(self.test_data) size = self.uploadable.get_size() - self.failUnlessEqual(size, actual_size) + self.assertThat(size, Equals(actual_size)) def test_datahandle_get_size_out_of_order(self): @@ -42,14 +43,14 @@ class DataHandle(SyncTestCase): # disturbing the location of the seek pointer. chunk_size = 100 data = self.uploadable.read(chunk_size) - self.failUnlessEqual(b"".join(data), self.test_data[:chunk_size]) + self.assertThat(b"".join(data), Equals(self.test_data[:chunk_size])) # Now get the size. size = self.uploadable.get_size() - self.failUnlessEqual(size, len(self.test_data)) + self.assertThat(self.test_data, HasLength(size)) # Now get more data. We should be right where we left off. more_data = self.uploadable.read(chunk_size) start = chunk_size end = chunk_size * 2 - self.failUnlessEqual(b"".join(more_data), self.test_data[start:end]) + self.assertThat(b"".join(more_data), Equals(self.test_data[start:end])) diff --git a/src/allmydata/test/mutable/test_exceptions.py b/src/allmydata/test/mutable/test_exceptions.py index aa2b56b86..23674d036 100644 --- a/src/allmydata/test/mutable/test_exceptions.py +++ b/src/allmydata/test/mutable/test_exceptions.py @@ -18,6 +18,7 @@ from allmydata.mutable.common import NeedMoreDataError, UncoordinatedWriteError class Exceptions(SyncTestCase): def test_repr(self): nmde = NeedMoreDataError(100, 50, 100) - self.failUnless("NeedMoreDataError" in repr(nmde), repr(nmde)) + self.assertTrue("NeedMoreDataError" in repr(nmde), msg=repr(nmde)) + self.assertTrue("NeedMoreDataError" in repr(nmde), msg=repr(nmde)) ucwe = UncoordinatedWriteError() - self.failUnless("UncoordinatedWriteError" in repr(ucwe), repr(ucwe)) + self.assertTrue("UncoordinatedWriteError" in repr(ucwe), msg=repr(ucwe)) diff --git a/src/allmydata/test/mutable/test_interoperability.py b/src/allmydata/test/mutable/test_interoperability.py index 5d7414907..496da1d2a 100644 --- a/src/allmydata/test/mutable/test_interoperability.py +++ b/src/allmydata/test/mutable/test_interoperability.py @@ -11,14 +11,15 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os, base64 -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import HasLength from allmydata import uri from allmydata.storage.common import storage_index_to_dir from allmydata.util import fileutil from .. import common_util as testutil from ..no_network import GridTestMixin -class Interoperability(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): +class Interoperability(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): sdmf_old_shares = {} sdmf_old_shares[0] = b"VGFob2UgbXV0YWJsZSBjb250YWluZXIgdjEKdQlEA47ESLbTdKdpLJXCpBxd5OH239tl5hvAiz1dvGdE5rIOpf8cbfxbPcwNF+Y5dM92uBVbmV6KAAAAAAAAB/wAAAAAAAAJ0AAAAAFOWSw7jSx7WXzaMpdleJYXwYsRCV82jNA5oex9m2YhXSnb2POh+vvC1LE1NAfRc9GOb2zQG84Xdsx1Jub2brEeKkyt0sRIttN0p2kslcKkHF3k4fbf22XmAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABamJprL6ecrsOoFKdrXUmWveLq8nzEGDOjFnyK9detI3noX3uyK2MwSnFdAfyN0tuAwoAAAAAAAAAFQAAAAAAAAAVAAABjwAAAo8AAAMXAAADNwAAAAAAAAM+AAAAAAAAB/wwggEgMA0GCSqGSIb3DQEBAQUAA4IBDQAwggEIAoIBAQC1IkainlJF12IBXBQdpRK1zXB7a26vuEYqRmQM09YjC6sQjCs0F2ICk8n9m/2Kw4l16eIEboB2Au9pODCE+u/dEAakEFh4qidTMn61rbGUbsLK8xzuWNW22ezzz9/nPia0HDrulXt51/FYtfnnAuD1RJGXJv/8tDllE9FL/18TzlH4WuB6Fp8FTgv7QdbZAfWJHDGFIpVCJr1XxOCsSZNFJIqGwZnD2lsChiWw5OJDbKd8otqN1hIbfHyMyfMOJ/BzRzvZXaUt4Dv5nf93EmQDWClxShRwpuX/NkZ5B2K9OFonFTbOCexm/MjMAdCBqebKKaiHFkiknUCn9eJQpZ5bAgERgV50VKj+AVTDfgTpqfO2vfo4wrufi6ZBb8QV7hllhUFBjYogQ9C96dnS7skv0s+cqFuUjwMILr5/rsbEmEMGvl0T0ytyAbtlXuowEFVj/YORNknM4yjY72YUtEPTlMpk0Cis7aIgTvu5qWMPER26PMApZuRqiwRsGIkaJIvOVOTHHjFYe3/YzdMkc7OZtqRMfQLtwVl2/zKQQV8b/a9vaT6q3mRLRd4P3esaAFe/+7sR/t+9tmB+a8kxtKM6kmaVQJMbXJZ4aoHGfeLX0m35Rcvu2Bmph7QfSDjk/eaE3q55zYSoGWShmlhlw4Kwg84sMuhmcVhLvo0LovR8bKmbdgACtTh7+7gs/l5w1lOkgbF6w7rkXLNslK7L2KYF4SPFLUcABOOLy8EETxh7h7/z9d62EiPu9CNpRrCOLxUhn+JUS+DuAAhgcAb/adrQFrhlrRNoRpvjDuxmFebA4F0qCyqWssm61AAQ/EX4eC/1+hGOQ/h4EiKUkqxdsfzdcPlDvd11SGWZ0VHsUclZChTzuBAU2zLTXm+cG8IFhO50ly6Ey/DB44NtMKVaVzO0nU8DE0Wua7Lx6Bnad5n91qmHAnwSEJE5YIhQM634omd6cq9Wk4seJCUIn+ucoknrpxp0IR9QMxpKSMRHRUg2K8ZegnY3YqFunRZKCfsq9ufQEKgjZN12AFqi551KPBdn4/3V5HK6xTv0P4robSsE/BvuIfByvRf/W7ZrDx+CFC4EEcsBOACOZCrkhhqd5TkYKbe9RA+vs56+9N5qZGurkxcoKviiyEncxvTuShD65DK/6x6kMDMgQv/EdZDI3x9GtHTnRBYXwDGnPJ19w+q2zC3e2XarbxTGYQIPEC5mYx0gAA0sbjf018NGfwBhl6SB54iGsa8uLvR3jHv6OSRJgwxL6j7P0Ts4Hv2EtO12P0Lv21pwi3JC1O/WviSrKCvrQD5lMHL9Uym3hwFi2zu0mqwZvxOAbGy7kfOPXkLYKOHTZLthzKj3PsdjeceWBfYIvPGKYcd6wDr36d1aXSYS4IWeApTS2AQ2lu0DUcgSefAvsA8NkgOklvJY1cjTMSg6j6cxQo48Bvl8RAWGLbr4h2S/8KwDGxwLsSv0Gop/gnFc3GzCsmL0EkEyHHWkCA8YRXCghfW80KLDV495ff7yF5oiwK56GniqowZ3RG9Jxp5MXoJQgsLV1VMQFMAmsY69yz8eoxRH3wl9L0dMyndLulhWWzNwPMQ2I0yAWdzA/pksVmwTJTFenB3MHCiWc5rEwJ3yofe6NZZnZQrYyL9r1TNnVwfTwRUiykPiLSk4x9Mi6DX7RamDAxc8u3gDVfjPsTOTagBOEGUWlGAL54KE/E6sgCQ5DEAt12chk8AxbjBFLPgV+/idrzS0lZHOL+IVBI9D0i3Bq1yZcSIqcjZB0M3IbxbPm4gLAYOWEiTUN2ecsEHHg9nt6rhgffVoqSbCCFPbpC0xf7WOC3+BQORIZECOCC7cUAciXq3xn+GuxpFE40RWRJeKAK7bBQ21X89ABIXlQFkFddZ9kRvlZ2Pnl0oeF+2pjnZu0Yc2czNfZEQF2P7BKIdLrgMgxG89snxAY8qAYTCKyQw6xTG87wkjDcpy1wzsZLP3WsOuO7cAm7b27xU0jRKq8Cw4d1hDoyRG+RdS53F8RFJzVMaNNYgxU2tfRwUvXpTRXiOheeRVvh25+YGVnjakUXjx/dSDnOw4ETHGHD+7styDkeSfc3BdSZxswzc6OehgMI+xsCxeeRym15QUm9hxvg8X7Bfz/0WulgFwgzrm11TVynZYOmvyHpiZKoqQyQyKahIrfhwuchCr7lMsZ4a+umIkNkKxCLZnI+T7jd+eGFMgKItjz3kTTxRl3IhaJG3LbPmwRUJynMxQKdMi4Uf0qy0U7+i8hIJ9m50QXc+3tw2bwDSbx22XYJ9Wf14gxx5G5SPTb1JVCbhe4fxNt91xIxCow2zk62tzbYfRe6dfmDmgYHkv2PIEtMJZK8iKLDjFfu2ZUxsKT2A5g1q17og6o9MeXeuFS3mzJXJYFQZd+3UzlFR9qwkFkby9mg5y4XSeMvRLOHPt/H/r5SpEqBE6a9MadZYt61FBV152CUEzd43ihXtrAa0XH9HdsiySBcWI1SpM3mv9rRP0DiLjMUzHw/K1D8TE2f07zW4t/9kvE11tFj/NpICixQAAAAA=" sdmf_old_shares[1] = b"VGFob2UgbXV0YWJsZSBjb250YWluZXIgdjEKdQlEA47ESLbTdKdpLJXCpBxd5OH239tl5hvAiz1dvGdE5rIOpf8cbfxbPcwNF+Y5dM92uBVbmV6KAAAAAAAAB/wAAAAAAAAJ0AAAAAFOWSw7jSx7WXzaMpdleJYXwYsRCV82jNA5oex9m2YhXSnb2POh+vvC1LE1NAfRc9GOb2zQG84Xdsx1Jub2brEeKkyt0sRIttN0p2kslcKkHF3k4fbf22XmAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABamJprL6ecrsOoFKdrXUmWveLq8nzEGDOjFnyK9detI3noX3uyK2MwSnFdAfyN0tuAwoAAAAAAAAAFQAAAAAAAAAVAAABjwAAAo8AAAMXAAADNwAAAAAAAAM+AAAAAAAAB/wwggEgMA0GCSqGSIb3DQEBAQUAA4IBDQAwggEIAoIBAQC1IkainlJF12IBXBQdpRK1zXB7a26vuEYqRmQM09YjC6sQjCs0F2ICk8n9m/2Kw4l16eIEboB2Au9pODCE+u/dEAakEFh4qidTMn61rbGUbsLK8xzuWNW22ezzz9/nPia0HDrulXt51/FYtfnnAuD1RJGXJv/8tDllE9FL/18TzlH4WuB6Fp8FTgv7QdbZAfWJHDGFIpVCJr1XxOCsSZNFJIqGwZnD2lsChiWw5OJDbKd8otqN1hIbfHyMyfMOJ/BzRzvZXaUt4Dv5nf93EmQDWClxShRwpuX/NkZ5B2K9OFonFTbOCexm/MjMAdCBqebKKaiHFkiknUCn9eJQpZ5bAgERgV50VKj+AVTDfgTpqfO2vfo4wrufi6ZBb8QV7hllhUFBjYogQ9C96dnS7skv0s+cqFuUjwMILr5/rsbEmEMGvl0T0ytyAbtlXuowEFVj/YORNknM4yjY72YUtEPTlMpk0Cis7aIgTvu5qWMPER26PMApZuRqiwRsGIkaJIvOVOTHHjFYe3/YzdMkc7OZtqRMfQLtwVl2/zKQQV8b/a9vaT6q3mRLRd4P3esaAFe/+7sR/t+9tmB+a8kxtKM6kmaVQJMbXJZ4aoHGfeLX0m35Rcvu2Bmph7QfSDjk/eaE3q55zYSoGWShmlhlw4Kwg84sMuhmcVhLvo0LovR8bKmbdgACtTh7+7gs/l5w1lOkgbF6w7rkXLNslK7L2KYF4SPFLUcABOOLy8EETxh7h7/z9d62EiPu9CNpRrCOLxUhn+JUS+DuAAhgcAb/adrQFrhlrRNoRpvjDuxmFebA4F0qCyqWssm61AAP7FHJWQoU87gQFNsy015vnBvCBYTudJcuhMvwweODbTD8Rfh4L/X6EY5D+HgSIpSSrF2x/N1w+UO93XVIZZnRUeePDXEwhqYDE0Wua7Lx6Bnad5n91qmHAnwSEJE5YIhQM634omd6cq9Wk4seJCUIn+ucoknrpxp0IR9QMxpKSMRHRUg2K8ZegnY3YqFunRZKCfsq9ufQEKgjZN12AFqi551KPBdn4/3V5HK6xTv0P4robSsE/BvuIfByvRf/W7ZrDx+CFC4EEcsBOACOZCrkhhqd5TkYKbe9RA+vs56+9N5qZGurkxcoKviiyEncxvTuShD65DK/6x6kMDMgQv/EdZDI3x9GtHTnRBYXwDGnPJ19w+q2zC3e2XarbxTGYQIPEC5mYx0gAA0sbjf018NGfwBhl6SB54iGsa8uLvR3jHv6OSRJgwxL6j7P0Ts4Hv2EtO12P0Lv21pwi3JC1O/WviSrKCvrQD5lMHL9Uym3hwFi2zu0mqwZvxOAbGy7kfOPXkLYKOHTZLthzKj3PsdjeceWBfYIvPGKYcd6wDr36d1aXSYS4IWeApTS2AQ2lu0DUcgSefAvsA8NkgOklvJY1cjTMSg6j6cxQo48Bvl8RAWGLbr4h2S/8KwDGxwLsSv0Gop/gnFc3GzCsmL0EkEyHHWkCA8YRXCghfW80KLDV495ff7yF5oiwK56GniqowZ3RG9Jxp5MXoJQgsLV1VMQFMAmsY69yz8eoxRH3wl9L0dMyndLulhWWzNwPMQ2I0yAWdzA/pksVmwTJTFenB3MHCiWc5rEwJ3yofe6NZZnZQrYyL9r1TNnVwfTwRUiykPiLSk4x9Mi6DX7RamDAxc8u3gDVfjPsTOTagBOEGUWlGAL54KE/E6sgCQ5DEAt12chk8AxbjBFLPgV+/idrzS0lZHOL+IVBI9D0i3Bq1yZcSIqcjZB0M3IbxbPm4gLAYOWEiTUN2ecsEHHg9nt6rhgffVoqSbCCFPbpC0xf7WOC3+BQORIZECOCC7cUAciXq3xn+GuxpFE40RWRJeKAK7bBQ21X89ABIXlQFkFddZ9kRvlZ2Pnl0oeF+2pjnZu0Yc2czNfZEQF2P7BKIdLrgMgxG89snxAY8qAYTCKyQw6xTG87wkjDcpy1wzsZLP3WsOuO7cAm7b27xU0jRKq8Cw4d1hDoyRG+RdS53F8RFJzVMaNNYgxU2tfRwUvXpTRXiOheeRVvh25+YGVnjakUXjx/dSDnOw4ETHGHD+7styDkeSfc3BdSZxswzc6OehgMI+xsCxeeRym15QUm9hxvg8X7Bfz/0WulgFwgzrm11TVynZYOmvyHpiZKoqQyQyKahIrfhwuchCr7lMsZ4a+umIkNkKxCLZnI+T7jd+eGFMgKItjz3kTTxRl3IhaJG3LbPmwRUJynMxQKdMi4Uf0qy0U7+i8hIJ9m50QXc+3tw2bwDSbx22XYJ9Wf14gxx5G5SPTb1JVCbhe4fxNt91xIxCow2zk62tzbYfRe6dfmDmgYHkv2PIEtMJZK8iKLDjFfu2ZUxsKT2A5g1q17og6o9MeXeuFS3mzJXJYFQZd+3UzlFR9qwkFkby9mg5y4XSeMvRLOHPt/H/r5SpEqBE6a9MadZYt61FBV152CUEzd43ihXtrAa0XH9HdsiySBcWI1SpM3mv9rRP0DiLjMUzHw/K1D8TE2f07zW4t/9kvE11tFj/NpICixQAAAAA=" @@ -53,7 +54,7 @@ class Interoperability(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixi sharedata) # ...and verify that the shares are there. shares = self.find_uri_shares(self.sdmf_old_cap) - assert len(shares) == 10 + self.assertThat(shares, HasLength(10)) def test_new_downloader_can_read_old_shares(self): self.basedir = "mutable/Interoperability/new_downloader_can_read_old_shares" @@ -62,5 +63,5 @@ class Interoperability(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixi nm = self.g.clients[0].nodemaker n = nm.create_from_cap(self.sdmf_old_cap) d = n.download_best_version() - d.addCallback(self.failUnlessEqual, self.sdmf_old_contents) + d.addCallback(self.assertEqual, self.sdmf_old_contents) return d diff --git a/src/allmydata/test/mutable/test_multiple_encodings.py b/src/allmydata/test/mutable/test_multiple_encodings.py index 12c5be051..2291b60d8 100644 --- a/src/allmydata/test/mutable/test_multiple_encodings.py +++ b/src/allmydata/test/mutable/test_multiple_encodings.py @@ -10,7 +10,8 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import Equals from allmydata.interfaces import SDMF_VERSION from allmydata.monitor import Monitor from foolscap.logging import log @@ -20,8 +21,9 @@ from allmydata.mutable.servermap import ServerMap, ServermapUpdater from ..common_util import DevNullDictionary from .util import FakeStorage, make_nodemaker -class MultipleEncodings(unittest.TestCase): +class MultipleEncodings(AsyncTestCase): def setUp(self): + super(MultipleEncodings, self).setUp() self.CONTENTS = b"New contents go here" self.uploadable = MutableData(self.CONTENTS) self._storage = FakeStorage() @@ -159,6 +161,6 @@ class MultipleEncodings(unittest.TestCase): d.addCallback(lambda res: fn3.download_best_version()) def _retrieved(new_contents): # the current specified behavior is "first version recoverable" - self.failUnlessEqual(new_contents, contents1) + self.assertThat(new_contents, Equals(contents1)) d.addCallback(_retrieved) return d diff --git a/src/allmydata/test/mutable/test_multiple_versions.py b/src/allmydata/test/mutable/test_multiple_versions.py index 460cde4b3..c9b7e71df 100644 --- a/src/allmydata/test/mutable/test_multiple_versions.py +++ b/src/allmydata/test/mutable/test_multiple_versions.py @@ -10,15 +10,17 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import Equals, HasLength from allmydata.monitor import Monitor from allmydata.mutable.common import MODE_CHECK, MODE_READ from .util import PublishMixin, CheckerMixin -class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin): +class MultipleVersions(AsyncTestCase, PublishMixin, CheckerMixin): def setUp(self): + super(MultipleVersions, self).setUp() return self.publish_multiple() def test_multiple_versions(self): @@ -26,7 +28,7 @@ class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin): # should get the latest one self._set_versions(dict([(i,2) for i in (0,2,4,6,8)])) d = self._fn.download_best_version() - d.addCallback(lambda res: self.failUnlessEqual(res, self.CONTENTS[4])) + d.addCallback(lambda res: self.assertThat(res, Equals(self.CONTENTS[4]))) # and the checker should report problems d.addCallback(lambda res: self._fn.check(Monitor())) d.addCallback(self.check_bad, "test_multiple_versions") @@ -35,23 +37,23 @@ class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin): d.addCallback(lambda res: self._set_versions(dict([(i,2) for i in range(10)]))) d.addCallback(lambda res: self._fn.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, self.CONTENTS[2])) + d.addCallback(lambda res: self.assertThat(res, Equals(self.CONTENTS[2]))) # if exactly one share is at version 3, we should still get v2 d.addCallback(lambda res: self._set_versions({0:3})) d.addCallback(lambda res: self._fn.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, self.CONTENTS[2])) + d.addCallback(lambda res: self.assertThat(res, Equals(self.CONTENTS[2]))) # but the servermap should see the unrecoverable version. This # depends upon the single newer share being queried early. d.addCallback(lambda res: self._fn.get_servermap(MODE_READ)) def _check_smap(smap): - self.failUnlessEqual(len(smap.unrecoverable_versions()), 1) + self.assertThat(smap.unrecoverable_versions(), HasLength(1)) newer = smap.unrecoverable_newer_versions() - self.failUnlessEqual(len(newer), 1) + self.assertThat(newer, HasLength(1)) verinfo, health = list(newer.items())[0] - self.failUnlessEqual(verinfo[0], 4) - self.failUnlessEqual(health, (1,3)) - self.failIf(smap.needs_merge()) + self.assertThat(verinfo[0], Equals(4)) + self.assertThat(health, Equals((1,3))) + self.assertThat(smap.needs_merge(), Equals(False)) d.addCallback(_check_smap) # if we have a mix of two parallel versions (s4a and s4b), we could # recover either @@ -60,13 +62,13 @@ class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin): 1:4,3:4,5:4,7:4,9:4})) d.addCallback(lambda res: self._fn.get_servermap(MODE_READ)) def _check_smap_mixed(smap): - self.failUnlessEqual(len(smap.unrecoverable_versions()), 0) + self.assertThat(smap.unrecoverable_versions(), HasLength(0)) newer = smap.unrecoverable_newer_versions() - self.failUnlessEqual(len(newer), 0) - self.failUnless(smap.needs_merge()) + self.assertThat(newer, HasLength(0)) + self.assertTrue(smap.needs_merge()) d.addCallback(_check_smap_mixed) d.addCallback(lambda res: self._fn.download_best_version()) - d.addCallback(lambda res: self.failUnless(res == self.CONTENTS[3] or + d.addCallback(lambda res: self.assertTrue(res == self.CONTENTS[3] or res == self.CONTENTS[4])) return d @@ -86,12 +88,12 @@ class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin): d = self._fn.modify(_modify) d.addCallback(lambda res: self._fn.download_best_version()) expected = self.CONTENTS[2] + b" modified" - d.addCallback(lambda res: self.failUnlessEqual(res, expected)) + d.addCallback(lambda res: self.assertThat(res, Equals(expected))) # and the servermap should indicate that the outlier was replaced too d.addCallback(lambda res: self._fn.get_servermap(MODE_CHECK)) def _check_smap(smap): - self.failUnlessEqual(smap.highest_seqnum(), 5) - self.failUnlessEqual(len(smap.unrecoverable_versions()), 0) - self.failUnlessEqual(len(smap.recoverable_versions()), 1) + self.assertThat(smap.highest_seqnum(), Equals(5)) + self.assertThat(smap.unrecoverable_versions(), HasLength(0)) + self.assertThat(smap.recoverable_versions(), HasLength(1)) d.addCallback(_check_smap) return d diff --git a/src/allmydata/test/mutable/test_problems.py b/src/allmydata/test/mutable/test_problems.py index 86a367596..9abee560d 100644 --- a/src/allmydata/test/mutable/test_problems.py +++ b/src/allmydata/test/mutable/test_problems.py @@ -11,7 +11,8 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os, base64 -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import HasLength from twisted.internet import defer from foolscap.logging import log from allmydata import uri @@ -61,7 +62,7 @@ class FirstServerGetsDeleted(object): return (True, {}) return retval -class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): +class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): def do_publish_surprise(self, version): self.basedir = "mutable/Problems/test_publish_surprise_%s" % version self.set_up_grid() @@ -198,8 +199,8 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): def _overwritten_again(smap): # Make sure that all shares were updated by making sure that # there aren't any other versions in the sharemap. - self.failUnlessEqual(len(smap.recoverable_versions()), 1) - self.failUnlessEqual(len(smap.unrecoverable_versions()), 0) + self.assertThat(smap.recoverable_versions(), HasLength(1)) + self.assertThat(smap.unrecoverable_versions(), HasLength(0)) d.addCallback(_overwritten_again) return d @@ -240,7 +241,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): # that ought to work def _got_node(n): d = n.download_best_version() - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1")) + d.addCallback(lambda res: self.assertTrue(res, b"contents 1")) # now break the second peer def _break_peer1(res): self.g.break_server(self.server1.get_serverid()) @@ -248,7 +249,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) # that ought to work too d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2")) + d.addCallback(lambda res: self.assertTrue(res, b"contents 2")) def _explain_error(f): print(f) if f.check(NotEnoughServersError): @@ -280,7 +281,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): d = nm.create_mutable_file(MutableData(b"contents 1")) def _created(n): d = n.download_best_version() - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1")) + d.addCallback(lambda res: self.assertTrue(res, b"contents 1")) # now break one of the remaining servers def _break_second_server(res): self.g.break_server(peerids[1]) @@ -288,7 +289,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) # that ought to work too d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2")) + d.addCallback(lambda res: self.assertTrue(res, b"contents 2")) return d d.addCallback(_created) return d @@ -419,7 +420,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): return self._node.download_version(servermap, ver) d.addCallback(_then) d.addCallback(lambda data: - self.failUnlessEqual(data, CONTENTS)) + self.assertTrue(data, CONTENTS)) return d def test_1654(self): diff --git a/src/allmydata/test/mutable/test_repair.py b/src/allmydata/test/mutable/test_repair.py index fb1caa974..987b21cc3 100644 --- a/src/allmydata/test/mutable/test_repair.py +++ b/src/allmydata/test/mutable/test_repair.py @@ -10,7 +10,8 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import Equals, HasLength from allmydata.interfaces import IRepairResults, ICheckAndRepairResults from allmydata.monitor import Monitor from allmydata.mutable.common import MODE_CHECK @@ -19,7 +20,7 @@ from allmydata.mutable.repairer import MustForceRepairError from ..common import ShouldFailMixin from .util import PublishMixin -class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): +class Repair(AsyncTestCase, PublishMixin, ShouldFailMixin): def get_shares(self, s): all_shares = {} # maps (peerid, shnum) to share data @@ -40,8 +41,8 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): d.addCallback(lambda res: self._fn.check(Monitor())) d.addCallback(lambda check_results: self._fn.repair(check_results)) def _check_results(rres): - self.failUnless(IRepairResults.providedBy(rres)) - self.failUnless(rres.get_successful()) + self.assertThat(IRepairResults.providedBy(rres), Equals(True)) + self.assertThat(rres.get_successful(), Equals(True)) # TODO: examine results self.copy_shares() @@ -50,11 +51,11 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): new_shares = self.old_shares[1] # TODO: this really shouldn't change anything. When we implement # a "minimal-bandwidth" repairer", change this test to assert: - #self.failUnlessEqual(new_shares, initial_shares) + #self.assertThat(new_shares, Equals(initial_shares)) # all shares should be in the same place as before - self.failUnlessEqual(set(initial_shares.keys()), - set(new_shares.keys())) + self.assertThat(set(initial_shares.keys()), + Equals(set(new_shares.keys()))) # but they should all be at a newer seqnum. The IV will be # different, so the roothash will be too. for key in initial_shares: @@ -70,19 +71,19 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): IV1, k1, N1, segsize1, datalen1, o1) = unpack_header(new_shares[key]) - self.failUnlessEqual(version0, version1) - self.failUnlessEqual(seqnum0+1, seqnum1) - self.failUnlessEqual(k0, k1) - self.failUnlessEqual(N0, N1) - self.failUnlessEqual(segsize0, segsize1) - self.failUnlessEqual(datalen0, datalen1) + self.assertThat(version0, Equals(version1)) + self.assertThat(seqnum0+1, Equals(seqnum1)) + self.assertThat(k0, Equals(k1)) + self.assertThat(N0, Equals(N1)) + self.assertThat(segsize0, Equals(segsize1)) + self.assertThat(datalen0, Equals(datalen1)) d.addCallback(_check_results) return d def failIfSharesChanged(self, ignored=None): old_shares = self.old_shares[-2] current_shares = self.old_shares[-1] - self.failUnlessEqual(old_shares, current_shares) + self.assertThat(old_shares, Equals(current_shares)) def _test_whether_repairable(self, publisher, nshares, expected_result): @@ -96,12 +97,12 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): d.addCallback(_delete_some_shares) d.addCallback(lambda ign: self._fn.check(Monitor())) def _check(cr): - self.failIf(cr.is_healthy()) - self.failUnlessEqual(cr.is_recoverable(), expected_result) + self.assertThat(cr.is_healthy(), Equals(False)) + self.assertThat(cr.is_recoverable(), Equals(expected_result)) return cr d.addCallback(_check) d.addCallback(lambda check_results: self._fn.repair(check_results)) - d.addCallback(lambda crr: self.failUnlessEqual(crr.get_successful(), expected_result)) + d.addCallback(lambda crr: self.assertThat(crr.get_successful(), Equals(expected_result))) return d def test_unrepairable_0shares(self): @@ -136,7 +137,7 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): del shares[peerid][shnum] d.addCallback(_delete_some_shares) d.addCallback(lambda ign: self._fn.check_and_repair(Monitor())) - d.addCallback(lambda crr: self.failUnlessEqual(crr.get_repair_successful(), expected_result)) + d.addCallback(lambda crr: self.assertThat(crr.get_repair_successful(), Equals(expected_result))) return d def test_unrepairable_0shares_checkandrepair(self): @@ -181,13 +182,13 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): self._fn.repair(check_results, force=True)) # this should give us 10 shares of the highest roothash def _check_repair_results(rres): - self.failUnless(rres.get_successful()) + self.assertThat(rres.get_successful(), Equals(True)) pass # TODO d.addCallback(_check_repair_results) d.addCallback(lambda res: self._fn.get_servermap(MODE_CHECK)) def _check_smap(smap): - self.failUnlessEqual(len(smap.recoverable_versions()), 1) - self.failIf(smap.unrecoverable_versions()) + self.assertThat(smap.recoverable_versions(), HasLength(1)) + self.assertThat(smap.unrecoverable_versions(), HasLength(0)) # now, which should have won? roothash_s4a = self.get_roothash_for(3) roothash_s4b = self.get_roothash_for(4) @@ -196,9 +197,9 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): else: expected_contents = self.CONTENTS[3] new_versionid = smap.best_recoverable_version() - self.failUnlessEqual(new_versionid[0], 5) # seqnum 5 + self.assertThat(new_versionid[0], Equals(5)) # seqnum 5 d2 = self._fn.download_version(smap, new_versionid) - d2.addCallback(self.failUnlessEqual, expected_contents) + d2.addCallback(self.assertEqual, expected_contents) return d2 d.addCallback(_check_smap) return d @@ -216,19 +217,19 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): d.addCallback(lambda check_results: self._fn.repair(check_results)) # this should give us 10 shares of v3 def _check_repair_results(rres): - self.failUnless(rres.get_successful()) + self.assertThat(rres.get_successful(), Equals(True)) pass # TODO d.addCallback(_check_repair_results) d.addCallback(lambda res: self._fn.get_servermap(MODE_CHECK)) def _check_smap(smap): - self.failUnlessEqual(len(smap.recoverable_versions()), 1) - self.failIf(smap.unrecoverable_versions()) + self.assertThat(smap.recoverable_versions(), HasLength(1)) + self.assertThat(smap.unrecoverable_versions(), HasLength(0)) # now, which should have won? expected_contents = self.CONTENTS[3] new_versionid = smap.best_recoverable_version() - self.failUnlessEqual(new_versionid[0], 5) # seqnum 5 + self.assertThat(new_versionid[0], Equals(5)) # seqnum 5 d2 = self._fn.download_version(smap, new_versionid) - d2.addCallback(self.failUnlessEqual, expected_contents) + d2.addCallback(self.assertTrue, expected_contents) return d2 d.addCallback(_check_smap) return d @@ -256,12 +257,12 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): d.addCallback(_get_readcap) d.addCallback(lambda res: self._fn3.check_and_repair(Monitor())) def _check_results(crr): - self.failUnless(ICheckAndRepairResults.providedBy(crr)) + self.assertThat(ICheckAndRepairResults.providedBy(crr), Equals(True)) # we should detect the unhealthy, but skip over mutable-readcap # repairs until #625 is fixed - self.failIf(crr.get_pre_repair_results().is_healthy()) - self.failIf(crr.get_repair_attempted()) - self.failIf(crr.get_post_repair_results().is_healthy()) + self.assertThat(crr.get_pre_repair_results().is_healthy(), Equals(False)) + self.assertThat(crr.get_repair_attempted(), Equals(False)) + self.assertThat(crr.get_post_repair_results().is_healthy(), Equals(False)) d.addCallback(_check_results) return d @@ -281,6 +282,6 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): d.addCallback(lambda ign: self._fn2.check(Monitor())) d.addCallback(lambda check_results: self._fn2.repair(check_results)) def _check(crr): - self.failUnlessEqual(crr.get_successful(), True) + self.assertThat(crr.get_successful(), Equals(True)) d.addCallback(_check) return d From 61b9f15fd1f27f428c1492a14ff3e998e07ac79b Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 10 Sep 2021 00:59:55 +0100 Subject: [PATCH 03/87] test.mutable : refactored roundtrip and servermap tests Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_roundtrip.py | 44 +++++++-------- src/allmydata/test/mutable/test_servermap.py | 57 ++++++++++---------- 2 files changed, 52 insertions(+), 49 deletions(-) diff --git a/src/allmydata/test/mutable/test_roundtrip.py b/src/allmydata/test/mutable/test_roundtrip.py index 79292b000..96ecdf640 100644 --- a/src/allmydata/test/mutable/test_roundtrip.py +++ b/src/allmydata/test/mutable/test_roundtrip.py @@ -11,7 +11,8 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six.moves import cStringIO as StringIO -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import Equals, HasLength, Contains from twisted.internet import defer from allmydata.util import base32, consumer @@ -23,8 +24,9 @@ from allmydata.mutable.retrieve import Retrieve from .util import PublishMixin, make_storagebroker, corrupt from .. import common_util as testutil -class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): +class Roundtrip(AsyncTestCase, testutil.ShouldFailMixin, PublishMixin): def setUp(self): + super(Roundtrip, self).setUp() return self.publish_one() def make_servermap(self, mode=MODE_READ, oldmap=None, sb=None): @@ -73,11 +75,11 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): def _do_retrieve(servermap): self._smap = servermap #self.dump_servermap(servermap) - self.failUnlessEqual(len(servermap.recoverable_versions()), 1) + self.assertThat(servermap.recoverable_versions(), HasLength(1)) return self.do_download(servermap) d.addCallback(_do_retrieve) def _retrieved(new_contents): - self.failUnlessEqual(new_contents, self.CONTENTS) + self.assertThat(new_contents, Equals(self.CONTENTS)) d.addCallback(_retrieved) # we should be able to re-use the same servermap, both with and # without updating it. @@ -132,10 +134,10 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): # back empty d = self.make_servermap(sb=sb2) def _check_servermap(servermap): - self.failUnlessEqual(servermap.best_recoverable_version(), None) - self.failIf(servermap.recoverable_versions()) - self.failIf(servermap.unrecoverable_versions()) - self.failIf(servermap.all_servers()) + self.assertThat(servermap.best_recoverable_version(), Equals(None)) + self.assertFalse(servermap.recoverable_versions()) + self.assertFalse(servermap.unrecoverable_versions()) + self.assertFalse(servermap.all_servers()) d.addCallback(_check_servermap) return d @@ -154,7 +156,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): self._fn._storage_broker = self._storage_broker return self._fn.download_best_version() def _retrieved(new_contents): - self.failUnlessEqual(new_contents, self.CONTENTS) + self.assertThat(new_contents, Equals(self.CONTENTS)) d.addCallback(_restore) d.addCallback(_retrieved) return d @@ -178,13 +180,13 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): # should be noted in the servermap's list of problems. if substring: allproblems = [str(f) for f in servermap.get_problems()] - self.failUnlessIn(substring, "".join(allproblems)) + self.assertThat("".join(allproblems), Contains(substring)) return servermap if should_succeed: d1 = self._fn.download_version(servermap, ver, fetch_privkey) d1.addCallback(lambda new_contents: - self.failUnlessEqual(new_contents, self.CONTENTS)) + self.assertThat(new_contents, Equals(self.CONTENTS))) else: d1 = self.shouldFail(NotEnoughSharesError, "_corrupt_all(offset=%s)" % (offset,), @@ -207,7 +209,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): # and the dump should mention the problems s = StringIO() dump = servermap.dump(s).getvalue() - self.failUnless("30 PROBLEMS" in dump, dump) + self.assertTrue("30 PROBLEMS" in dump, msg=dump) d.addCallback(_check_servermap) return d @@ -299,8 +301,8 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): # in NotEnoughSharesError, since each share will look invalid def _check(res): f = res[0] - self.failUnless(f.check(NotEnoughSharesError)) - self.failUnless("uncoordinated write" in str(f)) + self.assertThat(f.check(NotEnoughSharesError), HasLength(1)) + self.assertThat("uncoordinated write" in str(f), Equals(True)) return self._test_corrupt_all(1, "ran out of servers", corrupt_early=False, failure_checker=_check) @@ -309,7 +311,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): def test_corrupt_all_block_late(self): def _check(res): f = res[0] - self.failUnless(f.check(NotEnoughSharesError)) + self.assertTrue(f.check(NotEnoughSharesError)) return self._test_corrupt_all("share_data", "block hash tree failure", corrupt_early=False, failure_checker=_check) @@ -330,9 +332,9 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): shnums_to_corrupt=list(range(0, N-k))) d.addCallback(lambda res: self.make_servermap()) def _do_retrieve(servermap): - self.failUnless(servermap.get_problems()) - self.failUnless("pubkey doesn't match fingerprint" - in str(servermap.get_problems()[0])) + self.assertTrue(servermap.get_problems()) + self.assertThat("pubkey doesn't match fingerprint" + in str(servermap.get_problems()[0]), Equals(True)) ver = servermap.best_recoverable_version() r = Retrieve(self._fn, self._storage_broker, servermap, ver) c = consumer.MemoryConsumer() @@ -340,7 +342,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): d.addCallback(_do_retrieve) d.addCallback(lambda mc: b"".join(mc.chunks)) d.addCallback(lambda new_contents: - self.failUnlessEqual(new_contents, self.CONTENTS)) + self.assertThat(new_contents, Equals(self.CONTENTS))) return d @@ -355,11 +357,11 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): self.make_servermap()) def _do_retrieve(servermap): ver = servermap.best_recoverable_version() - self.failUnless(ver) + self.assertTrue(ver) return self._fn.download_best_version() d.addCallback(_do_retrieve) d.addCallback(lambda new_contents: - self.failUnlessEqual(new_contents, self.CONTENTS)) + self.assertThat(new_contents, Equals(self.CONTENTS))) return d diff --git a/src/allmydata/test/mutable/test_servermap.py b/src/allmydata/test/mutable/test_servermap.py index e8f933977..505d31e73 100644 --- a/src/allmydata/test/mutable/test_servermap.py +++ b/src/allmydata/test/mutable/test_servermap.py @@ -11,7 +11,8 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import Equals, NotEquals, HasLength from twisted.internet import defer from allmydata.monitor import Monitor from allmydata.mutable.common import \ @@ -20,8 +21,9 @@ from allmydata.mutable.publish import MutableData from allmydata.mutable.servermap import ServerMap, ServermapUpdater from .util import PublishMixin -class Servermap(unittest.TestCase, PublishMixin): +class Servermap(AsyncTestCase, PublishMixin): def setUp(self): + super(Servermap, self).setUp() return self.publish_one() def make_servermap(self, mode=MODE_CHECK, fn=None, sb=None, @@ -42,17 +44,17 @@ class Servermap(unittest.TestCase, PublishMixin): return d def failUnlessOneRecoverable(self, sm, num_shares): - self.failUnlessEqual(len(sm.recoverable_versions()), 1) - self.failUnlessEqual(len(sm.unrecoverable_versions()), 0) + self.assertThat(sm.recoverable_versions(), HasLength(1)) + self.assertThat(sm.unrecoverable_versions(), HasLength(0)) best = sm.best_recoverable_version() - self.failIfEqual(best, None) - self.failUnlessEqual(sm.recoverable_versions(), set([best])) - self.failUnlessEqual(len(sm.shares_available()), 1) - self.failUnlessEqual(sm.shares_available()[best], (num_shares, 3, 10)) + self.assertThat(best, NotEquals(None)) + self.assertThat(sm.recoverable_versions(), Equals(set([best]))) + self.assertThat(sm.shares_available(), HasLength(1)) + self.assertThat(sm.shares_available()[best], Equals((num_shares, 3, 10))) shnum, servers = list(sm.make_sharemap().items())[0] server = list(servers)[0] - self.failUnlessEqual(sm.version_on_server(server, shnum), best) - self.failUnlessEqual(sm.version_on_server(server, 666), None) + self.assertThat(sm.version_on_server(server, shnum), Equals(best)) + self.assertThat(sm.version_on_server(server, 666), Equals(None)) return sm def test_basic(self): @@ -117,7 +119,7 @@ class Servermap(unittest.TestCase, PublishMixin): v = sm.best_recoverable_version() vm = sm.make_versionmap() shares = list(vm[v]) - self.failUnlessEqual(len(shares), 6) + self.assertThat(shares, HasLength(6)) self._corrupted = set() # mark the first 5 shares as corrupt, then update the servermap. # The map should not have the marked shares it in any more, and @@ -135,18 +137,17 @@ class Servermap(unittest.TestCase, PublishMixin): shares = list(vm[v]) for (server, shnum) in self._corrupted: server_shares = sm.debug_shares_on_server(server) - self.failIf(shnum in server_shares, - "%d was in %s" % (shnum, server_shares)) - self.failUnlessEqual(len(shares), 5) + self.assertFalse(shnum in server_shares, "%d was in %s" % (shnum, server_shares)) + self.assertThat(shares, HasLength(5)) d.addCallback(_check_map) return d def failUnlessNoneRecoverable(self, sm): - self.failUnlessEqual(len(sm.recoverable_versions()), 0) - self.failUnlessEqual(len(sm.unrecoverable_versions()), 0) + self.assertThat(sm.recoverable_versions(), HasLength(0)) + self.assertThat(sm.unrecoverable_versions(), HasLength(0)) best = sm.best_recoverable_version() - self.failUnlessEqual(best, None) - self.failUnlessEqual(len(sm.shares_available()), 0) + self.assertThat(best, Equals(None)) + self.assertThat(sm.shares_available(), HasLength(0)) def test_no_shares(self): self._storage._peers = {} # delete all shares @@ -168,12 +169,12 @@ class Servermap(unittest.TestCase, PublishMixin): return d def failUnlessNotQuiteEnough(self, sm): - self.failUnlessEqual(len(sm.recoverable_versions()), 0) - self.failUnlessEqual(len(sm.unrecoverable_versions()), 1) + self.assertThat(sm.recoverable_versions(), HasLength(0)) + self.assertThat(sm.unrecoverable_versions(), HasLength(1)) best = sm.best_recoverable_version() - self.failUnlessEqual(best, None) - self.failUnlessEqual(len(sm.shares_available()), 1) - self.failUnlessEqual(list(sm.shares_available().values())[0], (2,3,10) ) + self.assertThat(best, Equals(None)) + self.assertThat(sm.shares_available(), HasLength(1)) + self.assertThat(list(sm.shares_available().values())[0], Equals((2,3,10))) return sm def test_not_quite_enough_shares(self): @@ -193,7 +194,7 @@ class Servermap(unittest.TestCase, PublishMixin): d.addCallback(lambda res: ms(mode=MODE_CHECK)) d.addCallback(lambda sm: self.failUnlessNotQuiteEnough(sm)) d.addCallback(lambda sm: - self.failUnlessEqual(len(sm.make_sharemap()), 2)) + self.assertThat(sm.make_sharemap(), HasLength(2))) d.addCallback(lambda res: ms(mode=MODE_ANYTHING)) d.addCallback(lambda sm: self.failUnlessNotQuiteEnough(sm)) d.addCallback(lambda res: ms(mode=MODE_WRITE)) @@ -216,7 +217,7 @@ class Servermap(unittest.TestCase, PublishMixin): # Calling make_servermap also updates the servermap in the mode # that we specify, so we just need to see what it says. def _check_servermap(sm): - self.failUnlessEqual(len(sm.recoverable_versions()), 1) + self.assertThat(sm.recoverable_versions(), HasLength(1)) d.addCallback(_check_servermap) return d @@ -229,10 +230,10 @@ class Servermap(unittest.TestCase, PublishMixin): self.make_servermap(mode=MODE_WRITE, update_range=(1, 2))) def _check_servermap(sm): # 10 shares - self.failUnlessEqual(len(sm.update_data), 10) + self.assertThat(sm.update_data, HasLength(10)) # one version for data in sm.update_data.values(): - self.failUnlessEqual(len(data), 1) + self.assertThat(data, HasLength(1)) d.addCallback(_check_servermap) return d @@ -244,5 +245,5 @@ class Servermap(unittest.TestCase, PublishMixin): d.addCallback(lambda ignored: self.make_servermap(mode=MODE_CHECK)) d.addCallback(lambda servermap: - self.failUnlessEqual(len(servermap.recoverable_versions()), 1)) + self.assertThat(servermap.recoverable_versions(), HasLength(1))) return d From 3b80b8cbe97a14e9e14e46a0a329c9e6bdcc8c12 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 10 Sep 2021 14:24:20 +0100 Subject: [PATCH 04/87] test.mutable : refactored test_version.py Signed-off-by: fenn-cs --- newsfragments/3788.minor | 0 src/allmydata/test/mutable/test_version.py | 116 +++++++++++---------- 2 files changed, 61 insertions(+), 55 deletions(-) create mode 100644 newsfragments/3788.minor diff --git a/newsfragments/3788.minor b/newsfragments/3788.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/mutable/test_version.py b/src/allmydata/test/mutable/test_version.py index 042305c24..d5c44f204 100644 --- a/src/allmydata/test/mutable/test_version.py +++ b/src/allmydata/test/mutable/test_version.py @@ -14,7 +14,13 @@ import os from six.moves import cStringIO as StringIO from twisted.internet import defer -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import ( + Equals, + IsInstance, + HasLength, + Contains, +) from allmydata import uri from allmydata.interfaces import SDMF_VERSION, MDMF_VERSION @@ -29,7 +35,7 @@ from ..no_network import GridTestMixin from .util import PublishMixin from .. import common_util as testutil -class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ +class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ PublishMixin): def setUp(self): GridTestMixin.setUp(self) @@ -47,8 +53,8 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d = self.nm.create_mutable_file(MutableData(data), version=MDMF_VERSION) def _then(n): - assert isinstance(n, MutableFileNode) - assert n._protocol_version == MDMF_VERSION + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n._protocol_version, Equals(MDMF_VERSION)) self.mdmf_node = n return n d.addCallback(_then) @@ -59,8 +65,8 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ data = self.small_data d = self.nm.create_mutable_file(MutableData(data)) def _then(n): - assert isinstance(n, MutableFileNode) - assert n._protocol_version == SDMF_VERSION + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n._protocol_version, Equals(SDMF_VERSION)) self.sdmf_node = n return n d.addCallback(_then) @@ -69,9 +75,9 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ def do_upload_empty_sdmf(self): d = self.nm.create_mutable_file(MutableData(b"")) def _then(n): - assert isinstance(n, MutableFileNode) + self.assertThat(n, IsInstance(MutableFileNode)) self.sdmf_zero_length_node = n - assert n._protocol_version == SDMF_VERSION + self.assertThat(n._protocol_version, Equals(SDMF_VERSION)) return n d.addCallback(_then) return d @@ -95,7 +101,7 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ debug.find_shares(fso) sharefiles = fso.stdout.getvalue().splitlines() expected = self.nm.default_encoding_parameters["n"] - self.failUnlessEqual(len(sharefiles), expected) + self.assertThat(sharefiles, HasLength(expected)) do = debug.DumpOptions() do["filename"] = sharefiles[0] @@ -103,17 +109,17 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ debug.dump_share(do) output = do.stdout.getvalue() lines = set(output.splitlines()) - self.failUnless("Mutable slot found:" in lines, output) - self.failUnless(" share_type: MDMF" in lines, output) - self.failUnless(" num_extra_leases: 0" in lines, output) - self.failUnless(" MDMF contents:" in lines, output) - self.failUnless(" seqnum: 1" in lines, output) - self.failUnless(" required_shares: 3" in lines, output) - self.failUnless(" total_shares: 10" in lines, output) - self.failUnless(" segsize: 131073" in lines, output) - self.failUnless(" datalen: %d" % len(self.data) in lines, output) + self.assertTrue("Mutable slot found:" in lines, output) + self.assertTrue(" share_type: MDMF" in lines, output) + self.assertTrue(" num_extra_leases: 0" in lines, output) + self.assertTrue(" MDMF contents:" in lines, output) + self.assertTrue(" seqnum: 1" in lines, output) + self.assertTrue(" required_shares: 3" in lines, output) + self.assertTrue(" total_shares: 10" in lines, output) + self.assertTrue(" segsize: 131073" in lines, output) + self.assertTrue(" datalen: %d" % len(self.data) in lines, output) vcap = str(n.get_verify_cap().to_string(), "utf-8") - self.failUnless(" verify-cap: %s" % vcap in lines, output) + self.assertTrue(" verify-cap: %s" % vcap in lines, output) cso = debug.CatalogSharesOptions() cso.nodedirs = fso.nodedirs cso.stdout = StringIO() @@ -122,13 +128,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ shares = cso.stdout.getvalue().splitlines() oneshare = shares[0] # all shares should be MDMF self.failIf(oneshare.startswith("UNKNOWN"), oneshare) - self.failUnless(oneshare.startswith("MDMF"), oneshare) + self.assertTrue(oneshare.startswith("MDMF"), oneshare) fields = oneshare.split() - self.failUnlessEqual(fields[0], "MDMF") - self.failUnlessEqual(fields[1].encode("ascii"), storage_index) - self.failUnlessEqual(fields[2], "3/10") - self.failUnlessEqual(fields[3], "%d" % len(self.data)) - self.failUnless(fields[4].startswith("#1:"), fields[3]) + self.assertThat(fields[0], Equals("MDMF")) + self.assertThat(fields[1].encode("ascii"), Equals(storage_index)) + self.assertThat(fields[2], Equals("3/10")) + self.assertThat(fields[3], Equals("%d" % len(self.data))) + self.assertTrue(fields[4].startswith("#1:"), fields[3]) # the rest of fields[4] is the roothash, which depends upon # encryption salts and is not constant. fields[5] is the # remaining time on the longest lease, which is timing dependent. @@ -140,11 +146,11 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d = self.do_upload() d.addCallback(lambda ign: self.mdmf_node.get_best_readable_version()) d.addCallback(lambda bv: - self.failUnlessEqual(bv.get_sequence_number(), 1)) + self.assertThat(bv.get_sequence_number(), Equals(1))) d.addCallback(lambda ignored: self.sdmf_node.get_best_readable_version()) d.addCallback(lambda bv: - self.failUnlessEqual(bv.get_sequence_number(), 1)) + self.assertThat(bv.get_sequence_number(), Equals(1))) # Now update. The sequence number in both cases should be 1 in # both cases. def _do_update(ignored): @@ -158,11 +164,11 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d.addCallback(lambda ignored: self.mdmf_node.get_best_readable_version()) d.addCallback(lambda bv: - self.failUnlessEqual(bv.get_sequence_number(), 2)) + self.assertThat(bv.get_sequence_number(), Equals(2))) d.addCallback(lambda ignored: self.sdmf_node.get_best_readable_version()) d.addCallback(lambda bv: - self.failUnlessEqual(bv.get_sequence_number(), 2)) + self.assertThat(bv.get_sequence_number(), Equals(2))) return d @@ -175,10 +181,10 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ def _then(ign): mdmf_uri = self.mdmf_node.get_uri() cap = uri.from_string(mdmf_uri) - self.failUnless(isinstance(cap, uri.WriteableMDMFFileURI)) + self.assertTrue(isinstance(cap, uri.WriteableMDMFFileURI)) readonly_mdmf_uri = self.mdmf_node.get_readonly_uri() cap = uri.from_string(readonly_mdmf_uri) - self.failUnless(isinstance(cap, uri.ReadonlyMDMFFileURI)) + self.assertTrue(isinstance(cap, uri.ReadonlyMDMFFileURI)) d.addCallback(_then) return d @@ -189,16 +195,16 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d.addCallback(lambda ign: self.mdmf_node.get_best_mutable_version()) def _check_mdmf(bv): n = self.mdmf_node - self.failUnlessEqual(bv.get_writekey(), n.get_writekey()) - self.failUnlessEqual(bv.get_storage_index(), n.get_storage_index()) - self.failIf(bv.is_readonly()) + self.assertThat(bv.get_writekey(), Equals(n.get_writekey())) + self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index())) + self.assertFalse(bv.is_readonly()) d.addCallback(_check_mdmf) d.addCallback(lambda ign: self.sdmf_node.get_best_mutable_version()) def _check_sdmf(bv): n = self.sdmf_node - self.failUnlessEqual(bv.get_writekey(), n.get_writekey()) - self.failUnlessEqual(bv.get_storage_index(), n.get_storage_index()) - self.failIf(bv.is_readonly()) + self.assertThat(bv.get_writekey(), Equals(n.get_writekey())) + self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index())) + self.assertFalse(bv.is_readonly()) d.addCallback(_check_sdmf) return d @@ -206,21 +212,21 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ def test_get_readonly_version(self): d = self.do_upload() d.addCallback(lambda ign: self.mdmf_node.get_best_readable_version()) - d.addCallback(lambda bv: self.failUnless(bv.is_readonly())) + d.addCallback(lambda bv: self.assertTrue(bv.is_readonly())) # Attempting to get a mutable version of a mutable file from a # filenode initialized with a readcap should return a readonly # version of that same node. d.addCallback(lambda ign: self.mdmf_node.get_readonly()) d.addCallback(lambda ro: ro.get_best_mutable_version()) - d.addCallback(lambda v: self.failUnless(v.is_readonly())) + d.addCallback(lambda v: self.assertTrue(v.is_readonly())) d.addCallback(lambda ign: self.sdmf_node.get_best_readable_version()) - d.addCallback(lambda bv: self.failUnless(bv.is_readonly())) + d.addCallback(lambda bv: self.assertTrue(bv.is_readonly())) d.addCallback(lambda ign: self.sdmf_node.get_readonly()) d.addCallback(lambda ro: ro.get_best_mutable_version()) - d.addCallback(lambda v: self.failUnless(v.is_readonly())) + d.addCallback(lambda v: self.assertTrue(v.is_readonly())) return d @@ -232,13 +238,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d.addCallback(lambda ignored: self.mdmf_node.download_best_version()) d.addCallback(lambda data: - self.failUnlessEqual(data, b"foo bar baz" * 100000)) + self.assertThat(data, Equals(b"foo bar baz" * 100000))) d.addCallback(lambda ignored: self.sdmf_node.overwrite(new_small_data)) d.addCallback(lambda ignored: self.sdmf_node.download_best_version()) d.addCallback(lambda data: - self.failUnlessEqual(data, b"foo bar baz" * 10)) + self.assertThat(data, Equals(b"foo bar baz" * 10))) return d @@ -250,13 +256,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d.addCallback(lambda ignored: self.mdmf_node.download_best_version()) d.addCallback(lambda data: - self.failUnlessIn(b"modified", data)) + self.assertThat(data, Contains(b"modified"))) d.addCallback(lambda ignored: self.sdmf_node.modify(modifier)) d.addCallback(lambda ignored: self.sdmf_node.download_best_version()) d.addCallback(lambda data: - self.failUnlessIn(b"modified", data)) + self.assertThat(data, Contains(b"modified"))) return d @@ -271,13 +277,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d.addCallback(lambda ignored: self.mdmf_node.download_best_version()) d.addCallback(lambda data: - self.failUnlessIn(b"modified", data)) + self.assertThat(data, Contains(b"modified"))) d.addCallback(lambda ignored: self.sdmf_node.modify(modifier)) d.addCallback(lambda ignored: self.sdmf_node.download_best_version()) d.addCallback(lambda data: - self.failUnlessIn(b"modified", data)) + self.assertThat(data, Contains(b"modified"))) return d @@ -308,13 +314,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d.addCallback(lambda ignored: self._fn.download_version(self.servermap, self.version1)) d.addCallback(lambda results: - self.failUnlessEqual(self.CONTENTS[self.version1_index], - results)) + self.assertThat(self.CONTENTS[self.version1_index], + Equals(results))) d.addCallback(lambda ignored: self._fn.download_version(self.servermap, self.version2)) d.addCallback(lambda results: - self.failUnlessEqual(self.CONTENTS[self.version2_index], - results)) + self.assertThat(self.CONTENTS[self.version2_index], + Equals(results))) return d @@ -344,7 +350,7 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ for i in range(0, len(expected), step): d2.addCallback(lambda ignored, i=i: version.read(c, i, step)) d2.addCallback(lambda ignored: - self.failUnlessEqual(expected, b"".join(c.chunks))) + self.assertThat(expected, Equals(b"".join(c.chunks)))) return d2 d.addCallback(_read_data) return d @@ -447,16 +453,16 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d2 = defer.succeed(None) d2.addCallback(lambda ignored: version.read(c)) d2.addCallback(lambda ignored: - self.failUnlessEqual(expected, b"".join(c.chunks))) + self.assertThat(expected, Equals(b"".join(c.chunks)))) d2.addCallback(lambda ignored: version.read(c2, offset=0, size=len(expected))) d2.addCallback(lambda ignored: - self.failUnlessEqual(expected, b"".join(c2.chunks))) + self.assertThat(expected, Equals(b"".join(c2.chunks)))) return d2 d.addCallback(_read_data) d.addCallback(lambda ignored: node.download_best_version()) - d.addCallback(lambda data: self.failUnlessEqual(expected, data)) + d.addCallback(lambda data: self.assertThat(expected, Equals(data))) return d def test_read_and_download_mdmf(self): From a3168b384450dbd4a9a576425442d0a92267e950 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Mon, 13 Sep 2021 23:45:16 +0100 Subject: [PATCH 05/87] test.mutable : refactored test_update.py Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_update.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/allmydata/test/mutable/test_update.py b/src/allmydata/test/mutable/test_update.py index da5d53e4c..c3ba1e9f7 100644 --- a/src/allmydata/test/mutable/test_update.py +++ b/src/allmydata/test/mutable/test_update.py @@ -11,7 +11,12 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import ( + Equals, + IsInstance, + GreaterThan, +) from twisted.internet import defer from allmydata.interfaces import MDMF_VERSION from allmydata.mutable.filenode import MutableFileNode @@ -25,7 +30,7 @@ from .. import common_util as testutil # this up. SEGSIZE = 128*1024 -class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): +class Update(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): def setUp(self): GridTestMixin.setUp(self) self.basedir = self.mktemp() @@ -35,14 +40,14 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): # self.data should be at least three segments long. td = b"testdata " self.data = td*(int(3*SEGSIZE//len(td))+10) # currently about 400kB - assert len(self.data) > 3*SEGSIZE + self.assertThat(len(self.data), GreaterThan(3*SEGSIZE)) self.small_data = b"test data" * 10 # 90 B; SDMF def do_upload_sdmf(self): d = self.nm.create_mutable_file(MutableData(self.small_data)) def _then(n): - assert isinstance(n, MutableFileNode) + self.assertThat(n, IsInstance(MutableFileNode)) self.sdmf_node = n d.addCallback(_then) return d @@ -51,7 +56,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): d = self.nm.create_mutable_file(MutableData(self.data), version=MDMF_VERSION) def _then(n): - assert isinstance(n, MutableFileNode) + self.assertThat(n, IsInstance(MutableFileNode)) self.mdmf_node = n d.addCallback(_then) return d @@ -185,7 +190,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): len(self.data))) d.addCallback(lambda ign: self.mdmf_node.download_best_version()) d.addCallback(lambda results: - self.failUnlessEqual(results, new_data)) + self.assertThat(results, Equals(new_data))) return d d0.addCallback(_run) return d0 @@ -201,7 +206,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): len(self.small_data))) d.addCallback(lambda ign: self.sdmf_node.download_best_version()) d.addCallback(lambda results: - self.failUnlessEqual(results, new_data)) + self.assertThat(results, Equals(new_data))) return d d0.addCallback(_run) return d0 @@ -221,7 +226,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): replace_offset)) d.addCallback(lambda ign: self.mdmf_node.download_best_version()) d.addCallback(lambda results: - self.failUnlessEqual(results, new_data)) + self.assertThat(results, Equals(new_data))) return d d0.addCallback(_run) return d0 @@ -242,7 +247,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): replace_offset)) d.addCallback(lambda ignored: self.mdmf_node.download_best_version()) d.addCallback(lambda results: - self.failUnlessEqual(results, new_data)) + self.assertThat(results, Equals(new_data))) return d d0.addCallback(_run) return d0 From 88cbb7b109946709fb93e982a73b4b4e84fda595 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 24 Sep 2021 23:04:01 +0100 Subject: [PATCH 06/87] remove methods that break test_filenode with AsyncBrokenTest Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_filenode.py | 95 +++++++++++---------- 1 file changed, 52 insertions(+), 43 deletions(-) diff --git a/src/allmydata/test/mutable/test_filenode.py b/src/allmydata/test/mutable/test_filenode.py index de03afc5a..748df1fde 100644 --- a/src/allmydata/test/mutable/test_filenode.py +++ b/src/allmydata/test/mutable/test_filenode.py @@ -13,6 +13,14 @@ if PY2: from six.moves import cStringIO as StringIO from twisted.internet import defer, reactor from twisted.trial import unittest +from ..common import AsyncTestCase, AsyncBrokenTestCase +from testtools.matchers import ( + Equals, + Contains, + HasLength, + Is, + IsInstance, +) from allmydata import uri, client from allmydata.util.consumer import MemoryConsumer from allmydata.interfaces import SDMF_VERSION, MDMF_VERSION, DownloadStopped @@ -29,12 +37,13 @@ from .util import ( make_peer, ) -class Filenode(unittest.TestCase, testutil.ShouldFailMixin): +class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): # this used to be in Publish, but we removed the limit. Some of # these tests test whether the new code correctly allows files # larger than the limit. OLD_MAX_SEGMENT_SIZE = 3500000 def setUp(self): + super(Filenode, self).setUp() self._storage = FakeStorage() self._peers = list( make_peer(self._storage, n) @@ -48,12 +57,12 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): def test_create(self): d = self.nodemaker.create_mutable_file() def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) - self.failUnlessEqual(n.get_storage_index(), n._storage_index) + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n.get_storage_index(), Equals(n._storage_index)) sb = self.nodemaker.storage_broker peer0 = sorted(sb.get_all_serverids())[0] shnums = self._storage._peers[peer0].keys() - self.failUnlessEqual(len(shnums), 1) + self.assertThat(shnums, HasLength(1)) d.addCallback(_created) return d @@ -61,12 +70,12 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): def test_create_mdmf(self): d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) - self.failUnlessEqual(n.get_storage_index(), n._storage_index) + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n.get_storage_index(), Equals(n._storage_index)) sb = self.nodemaker.storage_broker peer0 = sorted(sb.get_all_serverids())[0] shnums = self._storage._peers[peer0].keys() - self.failUnlessEqual(len(shnums), 1) + self.assertThat(shnums, HasLength(1)) d.addCallback(_created) return d @@ -80,7 +89,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda ignored, v=v: self.nodemaker.create_mutable_file(version=v)) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) + self.assertThat(n, IsInstance(MutableFileNode)) self._node = n return n d.addCallback(_created) @@ -89,19 +98,19 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda ignored: self._node.download_best_version()) d.addCallback(lambda contents: - self.failUnlessEqual(contents, b"Contents" * 50000)) + self.assertThat(contents, Equals(b"Contents" * 50000))) return d def test_max_shares(self): self.nodemaker.default_encoding_parameters['n'] = 255 d = self.nodemaker.create_mutable_file(version=SDMF_VERSION) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) - self.failUnlessEqual(n.get_storage_index(), n._storage_index) + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n.get_storage_index(), Equals(n._storage_index)) sb = self.nodemaker.storage_broker num_shares = sum([len(self._storage._peers[x].keys()) for x \ in sb.get_all_serverids()]) - self.failUnlessEqual(num_shares, 255) + self.assertThat(num_shares, Equals(255)) self._node = n return n d.addCallback(_created) @@ -121,12 +130,12 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): self.nodemaker.default_encoding_parameters['n'] = 255 d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) - self.failUnlessEqual(n.get_storage_index(), n._storage_index) + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n.get_storage_index(), Equals(n._storage_index)) sb = self.nodemaker.storage_broker num_shares = sum([len(self._storage._peers[x].keys()) for x \ in sb.get_all_serverids()]) - self.failUnlessEqual(num_shares, 255) + self.assertThat(num_shares, Equals(255)) self._node = n return n d.addCallback(_created) @@ -135,20 +144,20 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda ignored: self._node.download_best_version()) d.addCallback(lambda contents: - self.failUnlessEqual(contents, b"contents" * 50000)) + self.assertThat(contents, Equals(b"contents" * 50000))) return d def test_mdmf_filenode_cap(self): # Test that an MDMF filenode, once created, returns an MDMF URI. d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) + self.assertThat(n, IsInstance(MutableFileNode)) cap = n.get_cap() - self.failUnless(isinstance(cap, uri.WriteableMDMFFileURI)) + self.assertThat(cap, IsInstance(uri.WriteableMDMFFileURI)) rcap = n.get_readcap() - self.failUnless(isinstance(rcap, uri.ReadonlyMDMFFileURI)) + self.assertThat(rcap, IsInstance(uri.ReadonlyMDMFFileURI)) vcap = n.get_verify_cap() - self.failUnless(isinstance(vcap, uri.MDMFVerifierURI)) + self.assertThat(vcap, IsInstance(uri.MDMFVerifierURI)) d.addCallback(_created) return d @@ -158,13 +167,13 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): # filenode given an MDMF cap. d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) + self.assertThat(n, IsInstance(MutableFileNode)) s = n.get_uri() self.failUnless(s.startswith(b"URI:MDMF")) n2 = self.nodemaker.create_from_cap(s) - self.failUnless(isinstance(n2, MutableFileNode)) - self.failUnlessEqual(n.get_storage_index(), n2.get_storage_index()) - self.failUnlessEqual(n.get_uri(), n2.get_uri()) + self.assertThat(n2, IsInstance(MutableFileNode)) + self.assertThat(n.get_storage_index(), Equals(n2.get_storage_index())) + self.assertThat(n.get_uri(), Equals(n2.get_uri())) d.addCallback(_created) return d @@ -172,10 +181,10 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): def test_create_from_mdmf_readcap(self): d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) + self.assertThat(n, IsInstance(MutableFileNode)) s = n.get_readonly_uri() n2 = self.nodemaker.create_from_cap(s) - self.failUnless(isinstance(n2, MutableFileNode)) + self.assertThat(n2, IsInstance(MutableFileNode)) # Check that it's a readonly node self.failUnless(n2.is_readonly()) @@ -191,10 +200,10 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) def _created(n): self.uri = n.get_uri() - self.failUnlessEqual(n._protocol_version, MDMF_VERSION) + self.assertThat(n._protocol_version, Equals(MDMF_VERSION)) n2 = self.nodemaker.create_from_cap(self.uri) - self.failUnlessEqual(n2._protocol_version, MDMF_VERSION) + self.assertThat(n2._protocol_version, Equals(MDMF_VERSION)) d.addCallback(_created) return d @@ -203,14 +212,14 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): n = MutableFileNode(None, None, {"k": 3, "n": 10}, None) calls = [] def _callback(*args, **kwargs): - self.failUnlessEqual(args, (4,) ) - self.failUnlessEqual(kwargs, {"foo": 5}) + self.assertThat(args, Equals((4,))) + self.assertThat(kwargs, Equals({"foo": 5})) calls.append(1) return 6 d = n._do_serialized(_callback, 4, foo=5) def _check_callback(res): - self.failUnlessEqual(res, 6) - self.failUnlessEqual(calls, [1]) + self.assertThat(res, Equals(6)) + self.assertThat(calls, Equals([1])) d.addCallback(_check_callback) def _errback(): @@ -229,24 +238,24 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda sio: self.failUnless("3-of-10" in sio.getvalue())) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 1"))) - d.addCallback(lambda res: self.failUnlessIdentical(res, None)) + d.addCallback(lambda res: self.assertThat(res, Is(None))) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 1"))) d.addCallback(lambda res: n.get_size_of_best_version()) d.addCallback(lambda size: - self.failUnlessEqual(size, len(b"contents 1"))) + self.assertThat(size, Equals(len(b"contents 1")))) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 2"))) d.addCallback(lambda res: n.get_servermap(MODE_WRITE)) d.addCallback(lambda smap: n.upload(MutableData(b"contents 3"), smap)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 3")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 3"))) d.addCallback(lambda res: n.get_servermap(MODE_ANYTHING)) d.addCallback(lambda smap: n.download_version(smap, smap.best_recoverable_version())) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 3")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 3"))) # test a file that is large enough to overcome the # mapupdate-to-retrieve data caching (i.e. make the shares larger # than the default readsize, which is 2000 bytes). A 15kB file @@ -254,7 +263,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.overwrite(MutableData(b"large size file" * 1000))) d.addCallback(lambda res: n.download_best_version()) d.addCallback(lambda res: - self.failUnlessEqual(res, b"large size file" * 1000)) + self.assertThat(res, Equals(b"large size file" * 1000))) return d d.addCallback(_created) return d @@ -268,7 +277,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): n.get_servermap(MODE_READ)) def _then(servermap): dumped = servermap.dump(StringIO()) - self.failUnlessIn("3-of-10", dumped.getvalue()) + self.assertThat(dumped.getvalue(), Contains("3-of-10")) d.addCallback(_then) # Now overwrite the contents with some new contents. We want # to make them big enough to force the file to be uploaded @@ -431,7 +440,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): def test_create_with_initial_contents_function(self): data = b"initial contents" def _make_contents(n): - self.failUnless(isinstance(n, MutableFileNode)) + self.assertThat(n, IsInstance(MutableFileNode)) key = n.get_writekey() self.failUnless(isinstance(key, bytes), key) self.failUnlessEqual(len(key), 16) # AES key size @@ -447,7 +456,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): def test_create_mdmf_with_initial_contents_function(self): data = b"initial contents" * 100000 def _make_contents(n): - self.failUnless(isinstance(n, MutableFileNode)) + self.assertThat(n, IsInstance(MutableFileNode)) key = n.get_writekey() self.failUnless(isinstance(key, bytes), key) self.failUnlessEqual(len(key), 16) @@ -643,7 +652,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda sio: self.failUnless("3-of-10" in sio.getvalue())) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 1"))) - d.addCallback(lambda res: self.failUnlessIdentical(res, None)) + d.addCallback(lambda res: self.assertThat(res, Is(None))) d.addCallback(lambda res: n.download_best_version()) d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1")) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) From 49b6080097e12a9150db45534b81cfee48f39b9d Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sat, 25 Sep 2021 21:03:01 +0100 Subject: [PATCH 07/87] remove depracated assert methods Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_filenode.py | 95 ++++++++++----------- 1 file changed, 47 insertions(+), 48 deletions(-) diff --git a/src/allmydata/test/mutable/test_filenode.py b/src/allmydata/test/mutable/test_filenode.py index 748df1fde..579734433 100644 --- a/src/allmydata/test/mutable/test_filenode.py +++ b/src/allmydata/test/mutable/test_filenode.py @@ -12,8 +12,7 @@ if PY2: from six.moves import cStringIO as StringIO from twisted.internet import defer, reactor -from twisted.trial import unittest -from ..common import AsyncTestCase, AsyncBrokenTestCase +from ..common import AsyncBrokenTestCase from testtools.matchers import ( Equals, Contains, @@ -122,7 +121,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): self._node.download_best_version()) # ...and check to make sure everything went okay. d.addCallback(lambda contents: - self.failUnlessEqual(b"contents" * 50000, contents)) + self.assertThat(b"contents" * 50000, Equals(contents))) return d def test_max_shares_mdmf(self): @@ -169,7 +168,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): def _created(n): self.assertThat(n, IsInstance(MutableFileNode)) s = n.get_uri() - self.failUnless(s.startswith(b"URI:MDMF")) + self.assertTrue(s.startswith(b"URI:MDMF")) n2 = self.nodemaker.create_from_cap(s) self.assertThat(n2, IsInstance(MutableFileNode)) self.assertThat(n.get_storage_index(), Equals(n2.get_storage_index())) @@ -187,7 +186,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): self.assertThat(n2, IsInstance(MutableFileNode)) # Check that it's a readonly node - self.failUnless(n2.is_readonly()) + self.assertTrue(n2.is_readonly()) d.addCallback(_created) return d @@ -236,7 +235,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.get_servermap(MODE_READ)) d.addCallback(lambda smap: smap.dump(StringIO())) d.addCallback(lambda sio: - self.failUnless("3-of-10" in sio.getvalue())) + self.assertTrue("3-of-10" in sio.getvalue())) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 1"))) d.addCallback(lambda res: self.assertThat(res, Is(None))) d.addCallback(lambda res: n.download_best_version()) @@ -289,7 +288,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d.addCallback(lambda ignored: n.download_best_version()) d.addCallback(lambda data: - self.failUnlessEqual(data, big_contents)) + self.assertThat(data, Equals(big_contents))) # Overwrite the contents again with some new contents. As # before, they need to be big enough to force multiple # segments, so that we make the downloader deal with @@ -301,7 +300,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d.addCallback(lambda ignored: n.download_best_version()) d.addCallback(lambda data: - self.failUnlessEqual(data, bigger_contents)) + self.assertThat(data, Equals(bigger_contents))) return d d.addCallback(_created) return d @@ -332,7 +331,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): # Now we'll retrieve it into a pausing consumer. c = PausingConsumer() d = version.read(c) - d.addCallback(lambda ign: self.failUnlessEqual(c.size, len(data))) + d.addCallback(lambda ign: self.assertThat(c.size, Equals(len(data)))) c2 = PausingAndStoppingConsumer() d.addCallback(lambda ign: @@ -369,14 +368,14 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): self.uri = node.get_uri() # also confirm that the cap has no extension fields pieces = self.uri.split(b":") - self.failUnlessEqual(len(pieces), 4) + self.assertThat(pieces, HasLength(4)) return node.overwrite(MutableData(b"contents1" * 100000)) def _then(ignored): node = self.nodemaker.create_from_cap(self.uri) return node.download_best_version() def _downloaded(data): - self.failUnlessEqual(data, b"contents1" * 100000) + self.assertThat(data, Equals(b"contents1" * 100000)) d.addCallback(_created) d.addCallback(_then) d.addCallback(_downloaded) @@ -406,11 +405,11 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d = self.nodemaker.create_mutable_file(upload1) def _created(n): d = n.download_best_version() - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 1"))) upload2 = MutableData(b"contents 2") d.addCallback(lambda res: n.overwrite(upload2)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 2"))) return d d.addCallback(_created) return d @@ -424,15 +423,15 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): def _created(n): d = n.download_best_version() d.addCallback(lambda data: - self.failUnlessEqual(data, initial_contents)) + self.assertThat(data, Equals(initial_contents))) uploadable2 = MutableData(initial_contents + b"foobarbaz") d.addCallback(lambda ignored: n.overwrite(uploadable2)) d.addCallback(lambda ignored: n.download_best_version()) d.addCallback(lambda data: - self.failUnlessEqual(data, initial_contents + - b"foobarbaz")) + self.assertThat(data, Equals(initial_contents + + b"foobarbaz"))) return d d.addCallback(_created) return d @@ -442,14 +441,14 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): def _make_contents(n): self.assertThat(n, IsInstance(MutableFileNode)) key = n.get_writekey() - self.failUnless(isinstance(key, bytes), key) - self.failUnlessEqual(len(key), 16) # AES key size + self.assertTrue(isinstance(key, bytes), key) + self.assertThat(key, HasLength(16)) # AES key size return MutableData(data) d = self.nodemaker.create_mutable_file(_make_contents) def _created(n): return n.download_best_version() d.addCallback(_created) - d.addCallback(lambda data2: self.failUnlessEqual(data2, data)) + d.addCallback(lambda data2: self.assertThat(data2, Equals(data))) return d @@ -458,15 +457,15 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): def _make_contents(n): self.assertThat(n, IsInstance(MutableFileNode)) key = n.get_writekey() - self.failUnless(isinstance(key, bytes), key) - self.failUnlessEqual(len(key), 16) + self.assertTrue(isinstance(key, bytes), key) + self.assertThat(key, HasLength(16)) return MutableData(data) d = self.nodemaker.create_mutable_file(_make_contents, version=MDMF_VERSION) d.addCallback(lambda n: n.download_best_version()) d.addCallback(lambda data2: - self.failUnlessEqual(data2, data)) + self.assertThat(data2, Equals(data))) return d @@ -485,7 +484,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d = n.get_servermap(MODE_READ) d.addCallback(lambda servermap: servermap.best_recoverable_version()) d.addCallback(lambda verinfo: - self.failUnlessEqual(verinfo[0], expected_seqnum, which)) + self.assertThat(verinfo[0], Equals(expected_seqnum), which)) return d def test_modify(self): @@ -522,36 +521,36 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): def _created(n): d = n.modify(_modifier) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "m")) d.addCallback(lambda res: n.modify(_non_modifier)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "non")) d.addCallback(lambda res: n.modify(_none_modifier)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "none")) d.addCallback(lambda res: self.shouldFail(ValueError, "error_modifier", None, n.modify, _error_modifier)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "err")) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "big")) d.addCallback(lambda res: n.modify(_ucw_error_modifier)) - d.addCallback(lambda res: self.failUnlessEqual(len(calls), 2)) + d.addCallback(lambda res: self.assertThat(calls, HasLength(2))) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, - b"line1line2line3")) + d.addCallback(lambda res: self.assertThat(res, + Equals(b"line1line2line3"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 3, "ucw")) def _reset_ucw_error_modifier(res): @@ -566,10 +565,10 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): # will only be one larger than the previous test, not two (i.e. 4 # instead of 5). d.addCallback(lambda res: n.modify(_ucw_error_non_modifier)) - d.addCallback(lambda res: self.failUnlessEqual(len(calls), 2)) + d.addCallback(lambda res: self.assertThat(calls, HasLength(2))) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, - b"line1line2line3")) + d.addCallback(lambda res: self.assertThat(res, + Equals(b"line1line2line3"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 4, "ucw")) d.addCallback(lambda res: n.modify(_toobig_modifier)) return d @@ -605,7 +604,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): def _created(n): d = n.modify(_modifier) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "m")) d.addCallback(lambda res: @@ -614,7 +613,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): n.modify, _ucw_error_modifier, _backoff_stopper)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "stop")) def _reset_ucw_error_modifier(res): @@ -624,8 +623,8 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.modify(_ucw_error_modifier, _backoff_pauser)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, - b"line1line2line3")) + d.addCallback(lambda res: self.assertThat(res, + Equals(b"line1line2line3"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 3, "pause")) d.addCallback(lambda res: @@ -634,8 +633,8 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): n.modify, _always_ucw_error_modifier, giveuper.delay)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, - b"line1line2line3")) + d.addCallback(lambda res: self.assertThat(res, + Equals(b"line1line2line3"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 3, "giveup")) return d @@ -650,23 +649,23 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.get_servermap(MODE_READ)) d.addCallback(lambda smap: smap.dump(StringIO())) d.addCallback(lambda sio: - self.failUnless("3-of-10" in sio.getvalue())) + self.assertTrue("3-of-10" in sio.getvalue())) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 1"))) d.addCallback(lambda res: self.assertThat(res, Is(None))) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 1"))) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 2"))) d.addCallback(lambda res: n.get_servermap(MODE_WRITE)) d.addCallback(lambda smap: n.upload(MutableData(b"contents 3"), smap)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 3")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 3"))) d.addCallback(lambda res: n.get_servermap(MODE_ANYTHING)) d.addCallback(lambda smap: n.download_version(smap, smap.best_recoverable_version())) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 3")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 3"))) return d d.addCallback(_created) return d @@ -682,14 +681,14 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): return n.get_servermap(MODE_READ) d.addCallback(_created) d.addCallback(lambda ignored: - self.failUnlessEqual(self.n.get_size(), 0)) + self.assertThat(self.n.get_size(), Equals(0))) d.addCallback(lambda ignored: self.n.overwrite(MutableData(b"foobarbaz"))) d.addCallback(lambda ignored: - self.failUnlessEqual(self.n.get_size(), 9)) + self.assertThat(self.n.get_size(), Equals(9))) d.addCallback(lambda ignored: self.nodemaker.create_mutable_file(MutableData(b"foobarbaz"))) d.addCallback(_created) d.addCallback(lambda ignored: - self.failUnlessEqual(self.n.get_size(), 9)) + self.assertThat(self.n.get_size(), Equals(9))) return d From 54c032d0d7f467ff3861e5dcefa4c0024b415b34 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sat, 27 Nov 2021 00:59:13 +0100 Subject: [PATCH 08/87] change assertTrue -> assertEquals for non bools Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_problems.py | 8 ++++---- src/allmydata/test/mutable/test_repair.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/mutable/test_problems.py b/src/allmydata/test/mutable/test_problems.py index 9abee560d..40105142a 100644 --- a/src/allmydata/test/mutable/test_problems.py +++ b/src/allmydata/test/mutable/test_problems.py @@ -241,7 +241,7 @@ class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): # that ought to work def _got_node(n): d = n.download_best_version() - d.addCallback(lambda res: self.assertTrue(res, b"contents 1")) + d.addCallback(lambda res: self.assertEquals(res, b"contents 1")) # now break the second peer def _break_peer1(res): self.g.break_server(self.server1.get_serverid()) @@ -249,7 +249,7 @@ class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) # that ought to work too d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.assertTrue(res, b"contents 2")) + d.addCallback(lambda res: self.assertEquals(res, b"contents 2")) def _explain_error(f): print(f) if f.check(NotEnoughServersError): @@ -281,7 +281,7 @@ class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): d = nm.create_mutable_file(MutableData(b"contents 1")) def _created(n): d = n.download_best_version() - d.addCallback(lambda res: self.assertTrue(res, b"contents 1")) + d.addCallback(lambda res: self.assertEquals(res, b"contents 1")) # now break one of the remaining servers def _break_second_server(res): self.g.break_server(peerids[1]) @@ -289,7 +289,7 @@ class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) # that ought to work too d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.assertTrue(res, b"contents 2")) + d.addCallback(lambda res: self.assertEquals(res, b"contents 2")) return d d.addCallback(_created) return d diff --git a/src/allmydata/test/mutable/test_repair.py b/src/allmydata/test/mutable/test_repair.py index 987b21cc3..deddb8d92 100644 --- a/src/allmydata/test/mutable/test_repair.py +++ b/src/allmydata/test/mutable/test_repair.py @@ -229,7 +229,7 @@ class Repair(AsyncTestCase, PublishMixin, ShouldFailMixin): new_versionid = smap.best_recoverable_version() self.assertThat(new_versionid[0], Equals(5)) # seqnum 5 d2 = self._fn.download_version(smap, new_versionid) - d2.addCallback(self.assertTrue, expected_contents) + d2.addCallback(self.assertEquals, expected_contents) return d2 d.addCallback(_check_smap) return d From c02d8cab3ab7ebee5cc57a07dde5d4b52393eb03 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 29 Nov 2021 08:56:05 -0500 Subject: [PATCH 09/87] change one more assertTrue to assertEquals --- src/allmydata/test/mutable/test_problems.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/mutable/test_problems.py b/src/allmydata/test/mutable/test_problems.py index 40105142a..4bcb8161b 100644 --- a/src/allmydata/test/mutable/test_problems.py +++ b/src/allmydata/test/mutable/test_problems.py @@ -420,7 +420,7 @@ class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): return self._node.download_version(servermap, ver) d.addCallback(_then) d.addCallback(lambda data: - self.assertTrue(data, CONTENTS)) + self.assertEquals(data, CONTENTS)) return d def test_1654(self): From 4c92f9c8cfd1d77d5549de3de50c552dbb442461 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Jan 2022 13:10:23 -0500 Subject: [PATCH 10/87] Document additional semantics. --- docs/proposed/http-storage-node-protocol.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 0d8cee466..a8555cd26 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -483,6 +483,13 @@ For example:: The upload secret is an opaque _byte_ string. +Handling repeat calls: + +* If the same API call is repeated with the same upload secret, the response is the same and no change is made to server state. + This is necessary to ensure retries work in the face of lost responses from the server. +* If the API calls is with a different upload secret, this implies a new client, perhaps because the old client died. + In this case, all relevant in-progress uploads are canceled, and then the command is handled as usual. + Discussion `````````` From cac291eb91129e39336ebc508727912224fda606 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Jan 2022 13:10:38 -0500 Subject: [PATCH 11/87] News file. --- newsfragments/3855.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3855.minor diff --git a/newsfragments/3855.minor b/newsfragments/3855.minor new file mode 100644 index 000000000..e69de29bb From 5f4db487f787b7c88c01974005f19c1f854e8fa4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Jan 2022 13:43:19 -0500 Subject: [PATCH 12/87] Sketch of required business logic. --- src/allmydata/storage/http_server.py | 49 +++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 83bbbe49d..2dfb49b65 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -22,12 +22,14 @@ from base64 import b64decode from klein import Klein from twisted.web import http +import attr # TODO Make sure to use pure Python versions? -from cbor2 import dumps +from cbor2 import dumps, loads from .server import StorageServer from .http_client import swissnum_auth_header +from .immutable import BucketWriter from ..util.hashutil import timing_safe_compare @@ -125,6 +127,19 @@ def _authorized_route(app, required_secrets, *route_args, **route_kwargs): return decorator +@attr.s +class StorageIndexUploads(object): + """ + In-progress upload to storage index. + """ + + # Map share number to BucketWriter + shares = attr.ib() # type: Dict[int,BucketWriter] + + # The upload key. + upload_key = attr.ib() # type: bytes + + class HTTPServer(object): """ A HTTP interface to the storage server. @@ -137,6 +152,8 @@ class HTTPServer(object): ): # type: (StorageServer, bytes) -> None self._storage_server = storage_server self._swissnum = swissnum + # Maps storage index to StorageIndexUploads: + self._uploads = {} # type: Dict[bytes,StorageIndexUploads] def get_resource(self): """Return twisted.web ``Resource`` for this object.""" @@ -154,3 +171,33 @@ class HTTPServer(object): def version(self, request, authorization): """Return version information.""" return self._cbor(request, self._storage_server.get_version()) + + ##### Immutable APIs ##### + + @_authorized_route( + _app, + {Secrets.LEASE_RENEW, Secrets.LEASE_CANCEL, Secrets.UPLOAD}, + "/v1/immutable/", + methods=["POST"], + ) + def allocate_buckets(self, request, authorization, storage_index): + """Allocate buckets.""" + info = loads(request.content.read()) + upload_key = authorization[Secrets.UPLOAD] + + if storage_index in self._uploads: + # Pre-existing upload. + in_progress = self._uploads[storage_index] + if in_progress.upload_key == upload_key: + # Same session. + # TODO add BucketWriters only for new shares + pass + else: + # New session. + # TODO cancel all existing BucketWriters, then do + # self._storage_server.allocate_buckets() with given inputs. + pass + else: + # New upload. + # TODO self._storage_server.allocate_buckets() with given inputs. + # TODO add results to self._uploads. From 9c20ac8e7b1aaae6705b7980620410b792a4fee5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 Jan 2022 16:06:29 -0500 Subject: [PATCH 13/87] Client API sketch for basic immutable interactions. --- src/allmydata/storage/http_client.py | 82 +++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 72e1af080..a13ab1ce6 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -16,17 +16,19 @@ if PY2: else: # typing module not available in Python 2, and we only do type checking in # Python 3 anyway. - from typing import Union + from typing import Union, Set, List from treq.testing import StubTreq from base64 import b64encode +import attr + # TODO Make sure to import Python version? from cbor2 import loads from twisted.web.http_headers import Headers -from twisted.internet.defer import inlineCallbacks, returnValue, fail +from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred from hyperlink import DecodedURL import treq @@ -47,6 +49,80 @@ def swissnum_auth_header(swissnum): # type: (bytes) -> bytes return b"Tahoe-LAFS " + b64encode(swissnum).strip() +@attr.s +class ImmutableCreateResult(object): + """Result of creating a storage index for an immutable.""" + + already_have = attr.ib(type=Set[int]) + allocated = attr.ib(type=Set[int]) + + +class StorageClientImmutables(object): + """ + APIs for interacting with immutables. + """ + + def __init__(self, client): # type: (StorageClient) -> None + self._client = client + + @inlineCallbacks + def create( + self, + storage_index, + share_numbers, + allocated_size, + upload_secret, + lease_renew_secret, + lease_cancel_secret, + ): # type: (bytes, List[int], int, bytes, bytes, bytes) -> Deferred[ImmutableCreateResult] + """ + Create a new storage index for an immutable. + + TODO retry internally on failure, to ensure the operation fully + succeeded. If sufficient number of failures occurred, the result may + fire with an error, but there's no expectation that user code needs to + have a recovery codepath; it will most likely just report an error to + the user. + + Result fires when creating the storage index succeeded, if creating the + storage index failed the result will fire with an exception. + """ + + @inlineCallbacks + def write_share_chunk( + self, storage_index, share_number, upload_secret, offset, data + ): # type: (bytes, int, bytes, int, bytes) -> Deferred[bool] + """ + Upload a chunk of data for a specific share. + + TODO The implementation should retry failed uploads transparently a number + of times, so that if a failure percolates up, the caller can assume the + failure isn't a short-term blip. + + Result fires when the upload succeeded, with a boolean indicating + whether the _complete_ share (i.e. all chunks, not just this one) has + been uploaded. + """ + + @inlineCallbacks + def read_share_chunk( + self, storage_index, share_number, offset, length + ): # type: (bytes, int, int, int) -> Deferred[bytes] + """ + Download a chunk of data from a share. + + TODO Failed downloads should be transparently retried and redownloaded + by the implementation a few times so that if a failure percolates up, + the caller can assume the failure isn't a short-term blip. + + NOTE: the underlying HTTP protocol is much more flexible than this API, + so a future refactor may expand this in order to simplify the calling + code and perhaps download data more efficiently. But then again maybe + the HTTP protocol will be simplified, see + https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 + """ + + class StorageClient(object): """ HTTP client that talks to the HTTP storage server. @@ -77,7 +153,7 @@ class StorageClient(object): for key, value in secrets.items(): headers.addRawHeader( "X-Tahoe-Authorization", - b"%s %s" % (key.value.encode("ascii"), b64encode(value).strip()) + b"%s %s" % (key.value.encode("ascii"), b64encode(value).strip()), ) return self._treq.request(method, url, headers=headers, **kwargs) From 90a25d010953b1ddf702d27d21d354c13a703d2c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Jan 2022 12:36:46 -0500 Subject: [PATCH 14/87] Reorganize into shared file. --- src/allmydata/storage/http_client.py | 124 ++++++++++++++++----------- src/allmydata/storage/http_common.py | 25 ++++++ src/allmydata/storage/http_server.py | 12 +-- 3 files changed, 99 insertions(+), 62 deletions(-) create mode 100644 src/allmydata/storage/http_common.py diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index a13ab1ce6..cdcb94a94 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -24,7 +24,7 @@ from base64 import b64encode import attr # TODO Make sure to import Python version? -from cbor2 import loads +from cbor2 import loads, dumps from twisted.web.http_headers import Headers @@ -32,6 +32,8 @@ from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred from hyperlink import DecodedURL import treq +from .http_common import swissnum_auth_header, Secrets + class ClientException(Exception): """An unexpected error.""" @@ -44,11 +46,6 @@ def _decode_cbor(response): return fail(ClientException(response.code, response.phrase)) -def swissnum_auth_header(swissnum): # type: (bytes) -> bytes - """Return value for ``Authentication`` header.""" - return b"Tahoe-LAFS " + b64encode(swissnum).strip() - - @attr.s class ImmutableCreateResult(object): """Result of creating a storage index for an immutable.""" @@ -57,12 +54,75 @@ class ImmutableCreateResult(object): allocated = attr.ib(type=Set[int]) +class StorageClient(object): + """ + HTTP client that talks to the HTTP storage server. + """ + + def __init__( + self, url, swissnum, treq=treq + ): # type: (DecodedURL, bytes, Union[treq,StubTreq]) -> None + self._base_url = url + self._swissnum = swissnum + self._treq = treq + + def _url(self, path): + """Get a URL relative to the base URL.""" + return self._base_url.click(path) + + def _get_headers(self): # type: () -> Headers + """Return the basic headers to be used by default.""" + headers = Headers() + headers.addRawHeader( + "Authorization", + swissnum_auth_header(self._swissnum), + ) + return headers + + def _request( + self, + method, + url, + lease_renewal_secret=None, + lease_cancel_secret=None, + upload_secret=None, + **kwargs + ): + """ + Like ``treq.request()``, but with optional secrets that get translated + into corresponding HTTP headers. + """ + headers = self._get_headers() + for secret, value in [ + (Secrets.LEASE_RENEW, lease_renewal_secret), + (Secrets.LEASE_CANCEL, lease_cancel_secret), + (Secrets.UPLOAD, upload_secret), + ]: + if value is None: + continue + headers.addRawHeader( + "X-Tahoe-Authorization", + b"%s %s" % (secret.value.encode("ascii"), b64encode(value).strip()), + ) + return self._treq.request(method, url, headers=headers, **kwargs) + + @inlineCallbacks + def get_version(self): + """ + Return the version metadata for the server. + """ + url = self._url("/v1/version") + response = yield self._request("GET", url, {}) + decoded_response = yield _decode_cbor(response) + returnValue(decoded_response) + + class StorageClientImmutables(object): """ APIs for interacting with immutables. """ - def __init__(self, client): # type: (StorageClient) -> None + def __init__(self, client: StorageClient):# # type: (StorageClient) -> None self._client = client @inlineCallbacks @@ -87,6 +147,11 @@ class StorageClientImmutables(object): Result fires when creating the storage index succeeded, if creating the storage index failed the result will fire with an exception. """ + url = self._client._url("/v1/immutable/" + str(storage_index, "ascii")) + message = dumps( + {"share-numbers": share_numbers, "allocated-size": allocated_size} + ) + self._client._request("POST", ) @inlineCallbacks def write_share_chunk( @@ -121,48 +186,3 @@ class StorageClientImmutables(object): the HTTP protocol will be simplified, see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 """ - - -class StorageClient(object): - """ - HTTP client that talks to the HTTP storage server. - """ - - def __init__( - self, url, swissnum, treq=treq - ): # type: (DecodedURL, bytes, Union[treq,StubTreq]) -> None - self._base_url = url - self._swissnum = swissnum - self._treq = treq - - def _get_headers(self): # type: () -> Headers - """Return the basic headers to be used by default.""" - headers = Headers() - headers.addRawHeader( - "Authorization", - swissnum_auth_header(self._swissnum), - ) - return headers - - def _request(self, method, url, secrets, **kwargs): - """ - Like ``treq.request()``, but additional argument of secrets mapping - ``http_server.Secret`` to the bytes value of the secret. - """ - headers = self._get_headers() - for key, value in secrets.items(): - headers.addRawHeader( - "X-Tahoe-Authorization", - b"%s %s" % (key.value.encode("ascii"), b64encode(value).strip()), - ) - return self._treq.request(method, url, headers=headers, **kwargs) - - @inlineCallbacks - def get_version(self): - """ - Return the version metadata for the server. - """ - url = self._base_url.click("/v1/version") - response = yield self._request("GET", url, {}) - decoded_response = yield _decode_cbor(response) - returnValue(decoded_response) diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py new file mode 100644 index 000000000..af4224bd0 --- /dev/null +++ b/src/allmydata/storage/http_common.py @@ -0,0 +1,25 @@ +""" +Common HTTP infrastructure for the storge server. +""" +from future.utils import PY2 + +if PY2: + # fmt: off + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + # fmt: on + +from enum import Enum +from base64 import b64encode + + +def swissnum_auth_header(swissnum): # type: (bytes) -> bytes + """Return value for ``Authentication`` header.""" + return b"Tahoe-LAFS " + b64encode(swissnum).strip() + + +class Secrets(Enum): + """Different kinds of secrets the client may send.""" + + LEASE_RENEW = "lease-renew-secret" + LEASE_CANCEL = "lease-cancel-secret" + UPLOAD = "upload-secret" diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 2dfb49b65..78752e9c5 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -17,7 +17,6 @@ else: from typing import Dict, List, Set from functools import wraps -from enum import Enum from base64 import b64decode from klein import Klein @@ -28,19 +27,11 @@ import attr from cbor2 import dumps, loads from .server import StorageServer -from .http_client import swissnum_auth_header +from .http_common import swissnum_auth_header, Secrets from .immutable import BucketWriter from ..util.hashutil import timing_safe_compare -class Secrets(Enum): - """Different kinds of secrets the client may send.""" - - LEASE_RENEW = "lease-renew-secret" - LEASE_CANCEL = "lease-cancel-secret" - UPLOAD = "upload-secret" - - class ClientSecretsException(Exception): """The client did not send the appropriate secrets.""" @@ -201,3 +192,4 @@ class HTTPServer(object): # New upload. # TODO self._storage_server.allocate_buckets() with given inputs. # TODO add results to self._uploads. + pass From 57405ea722d9ea97b74d71c2dca9a7eacd1b4ad5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Jan 2022 14:15:16 -0500 Subject: [PATCH 15/87] Finish sketch of minimal immutable HTTP client code. --- src/allmydata/storage/http_client.py | 102 ++++++++++++++++++++++----- 1 file changed, 86 insertions(+), 16 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index cdcb94a94..3accb3c62 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -16,7 +16,7 @@ if PY2: else: # typing module not available in Python 2, and we only do type checking in # Python 3 anyway. - from typing import Union, Set, List + from typing import Union, Set, List, Optional from treq.testing import StubTreq from base64 import b64encode @@ -28,6 +28,7 @@ from cbor2 import loads, dumps from twisted.web.http_headers import Headers +from twisted.web import http from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred from hyperlink import DecodedURL import treq @@ -70,9 +71,10 @@ class StorageClient(object): """Get a URL relative to the base URL.""" return self._base_url.click(path) - def _get_headers(self): # type: () -> Headers + def _get_headers(self, headers): # type: (Optional[Headers]) -> Headers """Return the basic headers to be used by default.""" - headers = Headers() + if headers is None: + headers = Headers() headers.addRawHeader( "Authorization", swissnum_auth_header(self._swissnum), @@ -86,13 +88,14 @@ class StorageClient(object): lease_renewal_secret=None, lease_cancel_secret=None, upload_secret=None, + headers=None, **kwargs ): """ Like ``treq.request()``, but with optional secrets that get translated into corresponding HTTP headers. """ - headers = self._get_headers() + headers = self._get_headers(headers) for secret, value in [ (Secrets.LEASE_RENEW, lease_renewal_secret), (Secrets.LEASE_CANCEL, lease_cancel_secret), @@ -122,7 +125,7 @@ class StorageClientImmutables(object): APIs for interacting with immutables. """ - def __init__(self, client: StorageClient):# # type: (StorageClient) -> None + def __init__(self, client: StorageClient): # # type: (StorageClient) -> None self._client = client @inlineCallbacks @@ -138,11 +141,12 @@ class StorageClientImmutables(object): """ Create a new storage index for an immutable. - TODO retry internally on failure, to ensure the operation fully - succeeded. If sufficient number of failures occurred, the result may - fire with an error, but there's no expectation that user code needs to - have a recovery codepath; it will most likely just report an error to - the user. + TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 retry + internally on failure, to ensure the operation fully succeeded. If + sufficient number of failures occurred, the result may fire with an + error, but there's no expectation that user code needs to have a + recovery codepath; it will most likely just report an error to the + user. Result fires when creating the storage index succeeded, if creating the storage index failed the result will fire with an exception. @@ -151,7 +155,22 @@ class StorageClientImmutables(object): message = dumps( {"share-numbers": share_numbers, "allocated-size": allocated_size} ) - self._client._request("POST", ) + response = yield self._client._request( + "POST", + url, + lease_renew_secret=lease_renew_secret, + lease_cancel_secret=lease_cancel_secret, + upload_secret=upload_secret, + data=message, + headers=Headers({"content-type": "application/cbor"}), + ) + decoded_response = yield _decode_cbor(response) + returnValue( + ImmutableCreateResult( + already_have=decoded_response["already-have"], + allocated=decoded_response["allocated"], + ) + ) @inlineCallbacks def write_share_chunk( @@ -160,14 +179,45 @@ class StorageClientImmutables(object): """ Upload a chunk of data for a specific share. - TODO The implementation should retry failed uploads transparently a number - of times, so that if a failure percolates up, the caller can assume the + TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 The + implementation should retry failed uploads transparently a number of + times, so that if a failure percolates up, the caller can assume the failure isn't a short-term blip. Result fires when the upload succeeded, with a boolean indicating whether the _complete_ share (i.e. all chunks, not just this one) has been uploaded. """ + url = self._client._url( + "/v1/immutable/{}/{}".format(str(storage_index, "ascii"), share_number) + ) + response = yield self._client._request( + "POST", + url, + upload_secret=upload_secret, + data=data, + headers=Headers( + { + # The range is inclusive, thus the '- 1'. '*' means "length + # unknown", which isn't technically true but adding it just + # makes things slightly harder for calling API. + "content-range": "bytes {}-{}/*".format( + offset, offset + len(data) - 1 + ) + } + ), + ) + + if response.code == http.OK: + # Upload is still unfinished. + returnValue(False) + elif response.code == http.CREATED: + # Upload is done! + returnValue(True) + else: + raise ClientException( + response.code, + ) @inlineCallbacks def read_share_chunk( @@ -176,9 +226,10 @@ class StorageClientImmutables(object): """ Download a chunk of data from a share. - TODO Failed downloads should be transparently retried and redownloaded - by the implementation a few times so that if a failure percolates up, - the caller can assume the failure isn't a short-term blip. + TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 Failed + downloads should be transparently retried and redownloaded by the + implementation a few times so that if a failure percolates up, the + caller can assume the failure isn't a short-term blip. NOTE: the underlying HTTP protocol is much more flexible than this API, so a future refactor may expand this in order to simplify the calling @@ -186,3 +237,22 @@ class StorageClientImmutables(object): the HTTP protocol will be simplified, see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 """ + url = self._client._url( + "/v1/immutable/{}/{}".format(str(storage_index, "ascii"), share_number) + ) + response = yield self._client._request( + "GET", + url, + headers=Headers( + { + # The range is inclusive, thus the -1. + "range": "bytes={}-{}".format(offset, offset + length - 1) + } + ), + ) + if response.code == 200: + returnValue(response.content.read()) + else: + raise ClientException( + response.code, + ) From db68defe8897c48fb4a0ad31b9959b89664882b8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Jan 2022 14:50:29 -0500 Subject: [PATCH 16/87] Sketch of basic immutable server-side logic. --- src/allmydata/storage/http_server.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 78752e9c5..d2c9f6b7a 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -193,3 +193,31 @@ class HTTPServer(object): # TODO self._storage_server.allocate_buckets() with given inputs. # TODO add results to self._uploads. pass + + @_authorized_route( + _app, + {Secrets.UPLOAD}, "/v1/immutable//", methods=["PATCH"] + ) + def write_share_data(self, request, authorization, storage_index, share_number): + """Write data to an in-progress immutable upload.""" + # TODO parse the content-range header to get offset for writing + # TODO basic checks on validity of offset + # TODO basic check that body isn't infinite. require content-length? if so, needs t be in protocol spec. + data = request.content.read() + # TODO write to bucket at that offset. + + # TODO check if it conflicts with existing data (probably underlying code already handles that) if so, CONFLICT. + + # TODO if it finished writing altogether, 201 CREATED. Otherwise 200 OK. + + @_authorized_route( + _app, set(), "/v1/immutable//", methods=["GET"] + ) + def read_share_chunk(self, request, authorization, storage_index, share_number): + """Read a chunk for an already uploaded immutable.""" + # TODO read offset and length from Range header + # TODO basic checks on validity + # TODO lookup the share + # TODO if not found, 404 + # TODO otherwise, return data from that offset + From 040569b47acac5a00eddcbd03ef880c2b4f6727a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Jan 2022 15:11:16 -0500 Subject: [PATCH 17/87] Sketch of tests to write for basic HTTP immutable APIs. --- src/allmydata/test/test_storage_http.py | 69 +++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 181b6d347..6cf7a521e 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -263,3 +263,72 @@ class GenericHTTPAPITests(AsyncTestCase): b"maximum-immutable-share-size" ) self.assertEqual(version, expected_version) + + +class ImmutableHTTPAPITests(AsyncTestCase): + """ + Tests for immutable upload/download APIs. + """ + + def test_upload_can_be_downloaded(self): + """ + A single share can be uploaded in (possibly overlapping) chunks, and + then a random chunk can be downloaded, and it will match the original + file. + """ + + def test_multiple_shares_uploaded_to_different_place(self): + """ + If a storage index has multiple shares, uploads to different shares are + stored separately and can be downloaded separately. + """ + + def test_bucket_allocated_with_new_shares(self): + """ + If some shares already exist, allocating shares indicates only the new + ones were created. + """ + + def test_bucket_allocation_new_upload_key(self): + """ + If a bucket was allocated with one upload key, and a different upload + key is used to allocate the bucket again, the previous download is + cancelled. + """ + + def test_upload_with_wrong_upload_key_fails(self): + """ + Uploading with a key that doesn't match the one used to allocate the + bucket will fail. + """ + + def test_upload_offset_cannot_be_negative(self): + """ + A negative upload offset will be rejected. + """ + + def test_mismatching_upload_fails(self): + """ + If an uploaded chunk conflicts with an already uploaded chunk, a + CONFLICT error is returned. + """ + + def test_read_of_wrong_storage_index_fails(self): + """ + Reading from unknown storage index results in 404. + """ + + def test_read_of_wrong_share_number_fails(self): + """ + Reading from unknown storage index results in 404. + """ + + def test_read_with_negative_offset_fails(self): + """ + The offset for reads cannot be negative. + """ + + def test_read_with_negative_length_fails(self): + """ + The length for reads cannot be negative. + """ From 2369de6873f20791f36c884760085010cca64941 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Jan 2022 15:45:15 -0500 Subject: [PATCH 18/87] Simple upload/download test for immutables. --- src/allmydata/test/test_storage_http.py | 69 ++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 6cf7a521e..95708d211 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -15,6 +15,7 @@ if PY2: # fmt: on from base64 import b64encode +from os import urandom from twisted.internet.defer import inlineCallbacks @@ -33,7 +34,12 @@ from ..storage.http_server import ( ClientSecretsException, _authorized_route, ) -from ..storage.http_client import StorageClient, ClientException +from ..storage.http_client import ( + StorageClient, + ClientException, + StorageClientImmutables, + ImmutableCreateResult, +) def _post_process(params): @@ -270,12 +276,73 @@ class ImmutableHTTPAPITests(AsyncTestCase): Tests for immutable upload/download APIs. """ + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(ImmutableHTTPAPITests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + + @inlineCallbacks def test_upload_can_be_downloaded(self): """ A single share can be uploaded in (possibly overlapping) chunks, and then a random chunk can be downloaded, and it will match the original file. + + We don't exercise the full variation of overlapping chunks because + that's already done in test_storage.py. """ + length = 100 + expected_data = b"".join(bytes([i]) for i in range(100)) + + im_client = StorageClientImmutables(self.http.client) + + # Create a upload: + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = b"".join(bytes([i]) for i in range(16)) + created = yield im_client.create( + storage_index, [1], 100, upload_secret, lease_secret, lease_secret + ) + self.assertEqual( + created, ImmutableCreateResult(already_have=set(), allocated={1}) + ) + + # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. + def write(offset, length): + return im_client.write_share_chunk( + storage_index, + 1, + upload_secret, + offset, + expected_data[offset : offset + length], + ) + + finished = yield write(10, 10) + self.assertFalse(finished) + finished = yield write(30, 10) + self.assertFalse(finished) + finished = yield write(50, 10) + self.assertFalse(finished) + + # Then, an overlapping write with matching data (15-35): + finished = yield write(15, 20) + self.assertFalse(finished) + + # Now fill in the holes: + finished = yield write(0, 10) + self.assertFalse(finished) + finished = yield write(40, 10) + self.assertFalse(finished) + finished = yield write(60, 40) + self.assertTrue(finished) + + # We can now read: + for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: + downloaded = yield im_client.read_share_chunk( + storage_index, 1, upload_secret, offset, length + ) + self.assertEqual(downloaded, expected_data[offset : offset + length]) def test_multiple_shares_uploaded_to_different_place(self): """ From 004e5fbc9d2266585508dc376900e33ba1557a49 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Jan 2022 15:47:32 -0500 Subject: [PATCH 19/87] Get to point where we get failing HTTP response. --- src/allmydata/storage/http_client.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 3accb3c62..002ffc928 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -85,7 +85,7 @@ class StorageClient(object): self, method, url, - lease_renewal_secret=None, + lease_renew_secret=None, lease_cancel_secret=None, upload_secret=None, headers=None, @@ -97,7 +97,7 @@ class StorageClient(object): """ headers = self._get_headers(headers) for secret, value in [ - (Secrets.LEASE_RENEW, lease_renewal_secret), + (Secrets.LEASE_RENEW, lease_renew_secret), (Secrets.LEASE_CANCEL, lease_cancel_secret), (Secrets.UPLOAD, upload_secret), ]: @@ -162,7 +162,7 @@ class StorageClientImmutables(object): lease_cancel_secret=lease_cancel_secret, upload_secret=upload_secret, data=message, - headers=Headers({"content-type": "application/cbor"}), + headers=Headers({"content-type": ["application/cbor"]}), ) decoded_response = yield _decode_cbor(response) returnValue( @@ -201,9 +201,9 @@ class StorageClientImmutables(object): # The range is inclusive, thus the '- 1'. '*' means "length # unknown", which isn't technically true but adding it just # makes things slightly harder for calling API. - "content-range": "bytes {}-{}/*".format( - offset, offset + len(data) - 1 - ) + "content-range": [ + "bytes {}-{}/*".format(offset, offset + len(data) - 1) + ] } ), ) @@ -245,8 +245,8 @@ class StorageClientImmutables(object): url, headers=Headers( { - # The range is inclusive, thus the -1. - "range": "bytes={}-{}".format(offset, offset + length - 1) + # The range is inclusive. + "range": ["bytes={}-{}".format(offset, offset + length)] } ), ) From 6e2aaa8391e46bf02ee37a794dcc51c8ced84a25 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Jan 2022 09:14:58 -0500 Subject: [PATCH 20/87] Refactor more integration-y tests out. --- integration/test_storage_http.py | 283 ++++++++++++++++++++++++ src/allmydata/test/test_storage_http.py | 274 +---------------------- 2 files changed, 284 insertions(+), 273 deletions(-) create mode 100644 integration/test_storage_http.py diff --git a/integration/test_storage_http.py b/integration/test_storage_http.py new file mode 100644 index 000000000..714562cc4 --- /dev/null +++ b/integration/test_storage_http.py @@ -0,0 +1,283 @@ +""" +Connect the HTTP storage client to the HTTP storage server and make sure they +can talk to each other. +""" + +from future.utils import PY2 + +from os import urandom + +from twisted.internet.defer import inlineCallbacks +from fixtures import Fixture, TempDir +from treq.testing import StubTreq +from hyperlink import DecodedURL +from klein import Klein + +from allmydata.storage.server import StorageServer +from allmydata.storage.http_server import ( + HTTPServer, + _authorized_route, +) +from allmydata.storage.http_client import ( + StorageClient, + ClientException, + StorageClientImmutables, + ImmutableCreateResult, +) +from allmydata.storage.http_common import Secrets +from allmydata.test.common import AsyncTestCase + + +# TODO should be actual swissnum +SWISSNUM_FOR_TEST = b"abcd" + + +class TestApp(object): + """HTTP API for testing purposes.""" + + _app = Klein() + _swissnum = SWISSNUM_FOR_TEST # Match what the test client is using + + @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"]) + def validate_upload_secret(self, request, authorization): + if authorization == {Secrets.UPLOAD: b"MAGIC"}: + return "GOOD SECRET" + else: + return "BAD: {}".format(authorization) + + +class RoutingTests(AsyncTestCase): + """ + Tests for the HTTP routing infrastructure. + """ + + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(RoutingTests, self).setUp() + # Could be a fixture, but will only be used in this test class so not + # going to bother: + self._http_server = TestApp() + self.client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self._http_server._app.resource()), + ) + + @inlineCallbacks + def test_authorization_enforcement(self): + """ + The requirement for secrets is enforced; if they are not given, a 400 + response code is returned. + """ + # Without secret, get a 400 error. + response = yield self.client._request( + "GET", "http://127.0.0.1/upload_secret", {} + ) + self.assertEqual(response.code, 400) + + # With secret, we're good. + response = yield self.client._request( + "GET", "http://127.0.0.1/upload_secret", {Secrets.UPLOAD: b"MAGIC"} + ) + self.assertEqual(response.code, 200) + self.assertEqual((yield response.content()), b"GOOD SECRET") + + +class HttpTestFixture(Fixture): + """ + Setup HTTP tests' infrastructure, the storage server and corresponding + client. + """ + + def _setUp(self): + self.tempdir = self.useFixture(TempDir()) + self.storage_server = StorageServer(self.tempdir.path, b"\x00" * 20) + self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) + self.client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self.http_server.get_resource()), + ) + + +class GenericHTTPAPITests(AsyncTestCase): + """ + Tests of HTTP client talking to the HTTP server, for generic HTTP API + endpoints and concerns. + """ + + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(GenericHTTPAPITests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + + @inlineCallbacks + def test_bad_authentication(self): + """ + If the wrong swissnum is used, an ``Unauthorized`` response code is + returned. + """ + client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + b"something wrong", + treq=StubTreq(self.http.http_server.get_resource()), + ) + with self.assertRaises(ClientException) as e: + yield client.get_version() + self.assertEqual(e.exception.args[0], 401) + + @inlineCallbacks + def test_version(self): + """ + The client can return the version. + + We ignore available disk space and max immutable share size, since that + might change across calls. + """ + version = yield self.http.client.get_version() + version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"available-space" + ) + version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"maximum-immutable-share-size" + ) + expected_version = self.http.storage_server.get_version() + expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"available-space" + ) + expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"maximum-immutable-share-size" + ) + self.assertEqual(version, expected_version) + + +class ImmutableHTTPAPITests(AsyncTestCase): + """ + Tests for immutable upload/download APIs. + """ + + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(ImmutableHTTPAPITests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + + @inlineCallbacks + def test_upload_can_be_downloaded(self): + """ + A single share can be uploaded in (possibly overlapping) chunks, and + then a random chunk can be downloaded, and it will match the original + file. + + We don't exercise the full variation of overlapping chunks because + that's already done in test_storage.py. + """ + length = 100 + expected_data = b"".join(bytes([i]) for i in range(100)) + + im_client = StorageClientImmutables(self.http.client) + + # Create a upload: + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = b"".join(bytes([i]) for i in range(16)) + created = yield im_client.create( + storage_index, [1], 100, upload_secret, lease_secret, lease_secret + ) + self.assertEqual( + created, ImmutableCreateResult(already_have=set(), allocated={1}) + ) + + # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. + def write(offset, length): + return im_client.write_share_chunk( + storage_index, + 1, + upload_secret, + offset, + expected_data[offset : offset + length], + ) + + finished = yield write(10, 10) + self.assertFalse(finished) + finished = yield write(30, 10) + self.assertFalse(finished) + finished = yield write(50, 10) + self.assertFalse(finished) + + # Then, an overlapping write with matching data (15-35): + finished = yield write(15, 20) + self.assertFalse(finished) + + # Now fill in the holes: + finished = yield write(0, 10) + self.assertFalse(finished) + finished = yield write(40, 10) + self.assertFalse(finished) + finished = yield write(60, 40) + self.assertTrue(finished) + + # We can now read: + for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: + downloaded = yield im_client.read_share_chunk( + storage_index, 1, upload_secret, offset, length + ) + self.assertEqual(downloaded, expected_data[offset : offset + length]) + + def test_multiple_shares_uploaded_to_different_place(self): + """ + If a storage index has multiple shares, uploads to different shares are + stored separately and can be downloaded separately. + """ + + def test_bucket_allocated_with_new_shares(self): + """ + If some shares already exist, allocating shares indicates only the new + ones were created. + """ + + def test_bucket_allocation_new_upload_key(self): + """ + If a bucket was allocated with one upload key, and a different upload + key is used to allocate the bucket again, the previous download is + cancelled. + """ + + def test_upload_with_wrong_upload_key_fails(self): + """ + Uploading with a key that doesn't match the one used to allocate the + bucket will fail. + """ + + def test_upload_offset_cannot_be_negative(self): + """ + A negative upload offset will be rejected. + """ + + def test_mismatching_upload_fails(self): + """ + If an uploaded chunk conflicts with an already uploaded chunk, a + CONFLICT error is returned. + """ + + def test_read_of_wrong_storage_index_fails(self): + """ + Reading from unknown storage index results in 404. + """ + + def test_read_of_wrong_share_number_fails(self): + """ + Reading from unknown storage index results in 404. + """ + + def test_read_with_negative_offset_fails(self): + """ + The offset for reads cannot be negative. + """ + + def test_read_with_negative_length_fails(self): + """ + The length for reads cannot be negative. + """ diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 95708d211..aaa455a03 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -15,30 +15,13 @@ if PY2: # fmt: on from base64 import b64encode -from os import urandom - -from twisted.internet.defer import inlineCallbacks from hypothesis import assume, given, strategies as st -from fixtures import Fixture, TempDir -from treq.testing import StubTreq -from klein import Klein -from hyperlink import DecodedURL - -from .common import AsyncTestCase, SyncTestCase -from ..storage.server import StorageServer +from .common import SyncTestCase from ..storage.http_server import ( - HTTPServer, _extract_secrets, Secrets, ClientSecretsException, - _authorized_route, -) -from ..storage.http_client import ( - StorageClient, - ClientException, - StorageClientImmutables, - ImmutableCreateResult, ) @@ -144,258 +127,3 @@ class ExtractSecretsTests(SyncTestCase): """ with self.assertRaises(ClientSecretsException): _extract_secrets(["lease-cancel-secret eA=="], {Secrets.LEASE_RENEW}) - - -SWISSNUM_FOR_TEST = b"abcd" - - -class TestApp(object): - """HTTP API for testing purposes.""" - - _app = Klein() - _swissnum = SWISSNUM_FOR_TEST # Match what the test client is using - - @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"]) - def validate_upload_secret(self, request, authorization): - if authorization == {Secrets.UPLOAD: b"MAGIC"}: - return "GOOD SECRET" - else: - return "BAD: {}".format(authorization) - - -class RoutingTests(AsyncTestCase): - """ - Tests for the HTTP routing infrastructure. - """ - - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(RoutingTests, self).setUp() - # Could be a fixture, but will only be used in this test class so not - # going to bother: - self._http_server = TestApp() - self.client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - SWISSNUM_FOR_TEST, - treq=StubTreq(self._http_server._app.resource()), - ) - - @inlineCallbacks - def test_authorization_enforcement(self): - """ - The requirement for secrets is enforced; if they are not given, a 400 - response code is returned. - """ - # Without secret, get a 400 error. - response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", {} - ) - self.assertEqual(response.code, 400) - - # With secret, we're good. - response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", {Secrets.UPLOAD: b"MAGIC"} - ) - self.assertEqual(response.code, 200) - self.assertEqual((yield response.content()), b"GOOD SECRET") - - -class HttpTestFixture(Fixture): - """ - Setup HTTP tests' infrastructure, the storage server and corresponding - client. - """ - - def _setUp(self): - self.tempdir = self.useFixture(TempDir()) - self.storage_server = StorageServer(self.tempdir.path, b"\x00" * 20) - # TODO what should the swissnum _actually_ be? - self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) - self.client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - SWISSNUM_FOR_TEST, - treq=StubTreq(self.http_server.get_resource()), - ) - - -class GenericHTTPAPITests(AsyncTestCase): - """ - Tests of HTTP client talking to the HTTP server, for generic HTTP API - endpoints and concerns. - """ - - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(GenericHTTPAPITests, self).setUp() - self.http = self.useFixture(HttpTestFixture()) - - @inlineCallbacks - def test_bad_authentication(self): - """ - If the wrong swissnum is used, an ``Unauthorized`` response code is - returned. - """ - client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - b"something wrong", - treq=StubTreq(self.http.http_server.get_resource()), - ) - with self.assertRaises(ClientException) as e: - yield client.get_version() - self.assertEqual(e.exception.args[0], 401) - - @inlineCallbacks - def test_version(self): - """ - The client can return the version. - - We ignore available disk space and max immutable share size, since that - might change across calls. - """ - version = yield self.http.client.get_version() - version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"available-space" - ) - version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"maximum-immutable-share-size" - ) - expected_version = self.http.storage_server.get_version() - expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"available-space" - ) - expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"maximum-immutable-share-size" - ) - self.assertEqual(version, expected_version) - - -class ImmutableHTTPAPITests(AsyncTestCase): - """ - Tests for immutable upload/download APIs. - """ - - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(ImmutableHTTPAPITests, self).setUp() - self.http = self.useFixture(HttpTestFixture()) - - @inlineCallbacks - def test_upload_can_be_downloaded(self): - """ - A single share can be uploaded in (possibly overlapping) chunks, and - then a random chunk can be downloaded, and it will match the original - file. - - We don't exercise the full variation of overlapping chunks because - that's already done in test_storage.py. - """ - length = 100 - expected_data = b"".join(bytes([i]) for i in range(100)) - - im_client = StorageClientImmutables(self.http.client) - - # Create a upload: - upload_secret = urandom(32) - lease_secret = urandom(32) - storage_index = b"".join(bytes([i]) for i in range(16)) - created = yield im_client.create( - storage_index, [1], 100, upload_secret, lease_secret, lease_secret - ) - self.assertEqual( - created, ImmutableCreateResult(already_have=set(), allocated={1}) - ) - - # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. - def write(offset, length): - return im_client.write_share_chunk( - storage_index, - 1, - upload_secret, - offset, - expected_data[offset : offset + length], - ) - - finished = yield write(10, 10) - self.assertFalse(finished) - finished = yield write(30, 10) - self.assertFalse(finished) - finished = yield write(50, 10) - self.assertFalse(finished) - - # Then, an overlapping write with matching data (15-35): - finished = yield write(15, 20) - self.assertFalse(finished) - - # Now fill in the holes: - finished = yield write(0, 10) - self.assertFalse(finished) - finished = yield write(40, 10) - self.assertFalse(finished) - finished = yield write(60, 40) - self.assertTrue(finished) - - # We can now read: - for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: - downloaded = yield im_client.read_share_chunk( - storage_index, 1, upload_secret, offset, length - ) - self.assertEqual(downloaded, expected_data[offset : offset + length]) - - def test_multiple_shares_uploaded_to_different_place(self): - """ - If a storage index has multiple shares, uploads to different shares are - stored separately and can be downloaded separately. - """ - - def test_bucket_allocated_with_new_shares(self): - """ - If some shares already exist, allocating shares indicates only the new - ones were created. - """ - - def test_bucket_allocation_new_upload_key(self): - """ - If a bucket was allocated with one upload key, and a different upload - key is used to allocate the bucket again, the previous download is - cancelled. - """ - - def test_upload_with_wrong_upload_key_fails(self): - """ - Uploading with a key that doesn't match the one used to allocate the - bucket will fail. - """ - - def test_upload_offset_cannot_be_negative(self): - """ - A negative upload offset will be rejected. - """ - - def test_mismatching_upload_fails(self): - """ - If an uploaded chunk conflicts with an already uploaded chunk, a - CONFLICT error is returned. - """ - - def test_read_of_wrong_storage_index_fails(self): - """ - Reading from unknown storage index results in 404. - """ - - def test_read_of_wrong_share_number_fails(self): - """ - Reading from unknown storage index results in 404. - """ - - def test_read_with_negative_offset_fails(self): - """ - The offset for reads cannot be negative. - """ - - def test_read_with_negative_length_fails(self): - """ - The length for reads cannot be negative. - """ From 2bccb01be4bbd33b0b25642049f6ab2ae2697e17 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Jan 2022 11:16:21 -0500 Subject: [PATCH 21/87] Fix bug wrapping endpoints. --- src/allmydata/storage/http_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index d2c9f6b7a..b371fc395 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -110,6 +110,7 @@ def _authorized_route(app, required_secrets, *route_args, **route_kwargs): def decorator(f): @app.route(*route_args, **route_kwargs) @_authorization_decorator(required_secrets) + @wraps(f) def handle_route(*args, **kwargs): return f(*args, **kwargs) From 018f53105e9ad6c29dd82e324f2405ef9c75eb54 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Jan 2022 11:16:39 -0500 Subject: [PATCH 22/87] Pass correct arguments. --- src/allmydata/storage/http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 002ffc928..e38525583 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -115,7 +115,7 @@ class StorageClient(object): Return the version metadata for the server. """ url = self._url("/v1/version") - response = yield self._request("GET", url, {}) + response = yield self._request("GET", url) decoded_response = yield _decode_cbor(response) returnValue(decoded_response) From c4bb3c21d13a757007c058c467ee1dbc602be521 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Jan 2022 11:18:34 -0500 Subject: [PATCH 23/87] Update test to match current API. --- integration/test_storage_http.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integration/test_storage_http.py b/integration/test_storage_http.py index 714562cc4..66a8b2af1 100644 --- a/integration/test_storage_http.py +++ b/integration/test_storage_http.py @@ -72,13 +72,14 @@ class RoutingTests(AsyncTestCase): """ # Without secret, get a 400 error. response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", {} + "GET", + "http://127.0.0.1/upload_secret", ) self.assertEqual(response.code, 400) # With secret, we're good. response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", {Secrets.UPLOAD: b"MAGIC"} + "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC" ) self.assertEqual(response.code, 200) self.assertEqual((yield response.content()), b"GOOD SECRET") From f5437d9be73b42b891f892336551c95074d84b4d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Jan 2022 11:51:56 -0500 Subject: [PATCH 24/87] Some progress towards bucket allocation endpoint, and defining the protocol better. --- docs/proposed/http-storage-node-protocol.rst | 5 +++ src/allmydata/storage/http_client.py | 12 ++++++-- src/allmydata/storage/http_server.py | 32 ++++++++++++++++---- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index a8555cd26..26f1a2bb7 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -382,6 +382,11 @@ the server will respond with ``400 BAD REQUEST``. If authorization using the secret fails, then a ``401 UNAUTHORIZED`` response should be sent. +Encoding +~~~~~~~~ + +* ``storage_index`` should be base32 encoded (RFC3548) in URLs. + General ~~~~~~~ diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index e38525583..5e964bfbe 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -34,6 +34,12 @@ from hyperlink import DecodedURL import treq from .http_common import swissnum_auth_header, Secrets +from .common import si_b2a + + +def _encode_si(si): # type: (bytes) -> str + """Encode the storage index into Unicode string.""" + return str(si_b2a(si), "ascii") class ClientException(Exception): @@ -151,7 +157,7 @@ class StorageClientImmutables(object): Result fires when creating the storage index succeeded, if creating the storage index failed the result will fire with an exception. """ - url = self._client._url("/v1/immutable/" + str(storage_index, "ascii")) + url = self._client._url("/v1/immutable/" + _encode_si(storage_index)) message = dumps( {"share-numbers": share_numbers, "allocated-size": allocated_size} ) @@ -189,7 +195,7 @@ class StorageClientImmutables(object): been uploaded. """ url = self._client._url( - "/v1/immutable/{}/{}".format(str(storage_index, "ascii"), share_number) + "/v1/immutable/{}/{}".format(_encode_si(storage_index), share_number) ) response = yield self._client._request( "POST", @@ -238,7 +244,7 @@ class StorageClientImmutables(object): https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 """ url = self._client._url( - "/v1/immutable/{}/{}".format(str(storage_index, "ascii"), share_number) + "/v1/immutable/{}/{}".format(_encode_si(storage_index), share_number) ) response = yield self._client._request( "GET", diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index b371fc395..23f0d2f1c 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -28,6 +28,7 @@ from cbor2 import dumps, loads from .server import StorageServer from .http_common import swissnum_auth_header, Secrets +from .common import si_a2b from .immutable import BucketWriter from ..util.hashutil import timing_safe_compare @@ -174,6 +175,7 @@ class HTTPServer(object): ) def allocate_buckets(self, request, authorization, storage_index): """Allocate buckets.""" + storage_index = si_a2b(storage_index.encode("ascii")) info = loads(request.content.read()) upload_key = authorization[Secrets.UPLOAD] @@ -191,13 +193,29 @@ class HTTPServer(object): pass else: # New upload. - # TODO self._storage_server.allocate_buckets() with given inputs. - # TODO add results to self._uploads. - pass + already_got, sharenum_to_bucket = self._storage_server.allocate_buckets( + storage_index, + renew_secret=authorization[Secrets.LEASE_RENEW], + cancel_secret=authorization[Secrets.LEASE_CANCEL], + sharenums=info["share-numbers"], + allocated_size=info["allocated-size"], + ) + self._uploads[storage_index] = StorageIndexUploads( + shares=sharenum_to_bucket, upload_key=authorization[Secrets.UPLOAD] + ) + return self._cbor( + request, + { + "already-have": set(already_got), + "allocated": set(sharenum_to_bucket), + }, + ) @_authorized_route( _app, - {Secrets.UPLOAD}, "/v1/immutable//", methods=["PATCH"] + {Secrets.UPLOAD}, + "/v1/immutable//", + methods=["PATCH"], ) def write_share_data(self, request, authorization, storage_index, share_number): """Write data to an in-progress immutable upload.""" @@ -212,7 +230,10 @@ class HTTPServer(object): # TODO if it finished writing altogether, 201 CREATED. Otherwise 200 OK. @_authorized_route( - _app, set(), "/v1/immutable//", methods=["GET"] + _app, + set(), + "/v1/immutable//", + methods=["GET"], ) def read_share_chunk(self, request, authorization, storage_index, share_number): """Read a chunk for an already uploaded immutable.""" @@ -221,4 +242,3 @@ class HTTPServer(object): # TODO lookup the share # TODO if not found, 404 # TODO otherwise, return data from that offset - From 3bed0678285deb5ac2064bbc4c045c0b75b4df2b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Jan 2022 08:34:17 -0500 Subject: [PATCH 25/87] Implement more of the writing logic. --- src/allmydata/storage/http_client.py | 2 +- src/allmydata/storage/http_server.py | 32 ++++++++++++++++++++++------ src/allmydata/storage/immutable.py | 11 ++++++++-- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 5e964bfbe..697af91b5 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -198,7 +198,7 @@ class StorageClientImmutables(object): "/v1/immutable/{}/{}".format(_encode_si(storage_index), share_number) ) response = yield self._client._request( - "POST", + "PATCH", url, upload_secret=upload_secret, data=data, diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 23f0d2f1c..28367752d 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -219,15 +219,35 @@ class HTTPServer(object): ) def write_share_data(self, request, authorization, storage_index, share_number): """Write data to an in-progress immutable upload.""" - # TODO parse the content-range header to get offset for writing - # TODO basic checks on validity of offset - # TODO basic check that body isn't infinite. require content-length? if so, needs t be in protocol spec. + storage_index = si_a2b(storage_index.encode("ascii")) + content_range = request.getHeader("content-range") + if content_range is None: + offset = 0 + else: + offset = int(content_range.split()[1].split("-")[0]) + + # TODO basic checks on validity of start, offset, and content-range in general. also of share_number. + # TODO basic check that body isn't infinite. require content-length? or maybe we should require content-range (it's optional now)? if so, needs to be rflected in protocol spec. + data = request.content.read() - # TODO write to bucket at that offset. + try: + bucket = self._uploads[storage_index].shares[share_number] + except (KeyError, IndexError): + # TODO return 404 + raise - # TODO check if it conflicts with existing data (probably underlying code already handles that) if so, CONFLICT. + finished = bucket.write(offset, data) - # TODO if it finished writing altogether, 201 CREATED. Otherwise 200 OK. + # TODO if raises ConflictingWriteError, return HTTP CONFLICT code. + + if finished: + request.setResponseCode(http.CREATED) + else: + request.setResponseCode(http.OK) + + # TODO spec says we should return missing ranges. but client doesn't + # actually use them? So is it actually useful? + return b"" @_authorized_route( _app, diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index da9aa473f..5878e254a 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -13,7 +13,7 @@ if PY2: import os, stat, struct, time -from collections_extended import RangeMap +from collections_extended import RangeMap, MappedRange from foolscap.api import Referenceable @@ -375,7 +375,10 @@ class BucketWriter(object): def allocated_size(self): return self._max_size - def write(self, offset, data): + def write(self, offset, data): # type: (int, bytes) -> bool + """ + Write data at given offset, return whether the upload is complete. + """ # Delay the timeout, since we received data: self._timeout.reset(30 * 60) start = self._clock.seconds() @@ -399,6 +402,10 @@ class BucketWriter(object): self.ss.add_latency("write", self._clock.seconds() - start) self.ss.count("write") + # Return whether the whole thing has been written. + # TODO needs property test + return self._already_written.ranges() == [MappedRange(0, self._max_size, True)] + def close(self): precondition(not self.closed) self._timeout.cancel() From 4ea6bf2381f5a43f1265be988b2b9c50ad017384 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Sat, 15 Jan 2022 12:59:23 -0500 Subject: [PATCH 26/87] A test and some progress to making it pass. --- src/allmydata/storage/immutable.py | 12 +++++++---- src/allmydata/test/test_storage.py | 33 ++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 5878e254a..0bcef8246 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -13,7 +13,7 @@ if PY2: import os, stat, struct, time -from collections_extended import RangeMap, MappedRange +from collections_extended import RangeMap from foolscap.api import Referenceable @@ -402,9 +402,13 @@ class BucketWriter(object): self.ss.add_latency("write", self._clock.seconds() - start) self.ss.count("write") - # Return whether the whole thing has been written. - # TODO needs property test - return self._already_written.ranges() == [MappedRange(0, self._max_size, True)] + # Return whether the whole thing has been written. See + # https://github.com/mlenzen/collections-extended/issues/169 for why + # it's done this way. + print([tuple(mr) for mr in self._already_written.ranges()]) + return [tuple(mr) for mr in self._already_written.ranges()] == [ + (0, self._max_size, True) + ] def close(self): precondition(not self.closed) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index bd74a1052..e4f01f6f1 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -13,6 +13,7 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str +from array import array from io import ( BytesIO, ) @@ -34,7 +35,7 @@ from twisted.trial import unittest from twisted.internet import defer from twisted.internet.task import Clock -from hypothesis import given, strategies +from hypothesis import given, strategies, example import itertools from allmydata import interfaces @@ -230,7 +231,6 @@ class Bucket(unittest.TestCase): br = BucketReader(self, bw.finalhome) self.assertEqual(br.read(0, length), expected_data) - @given( maybe_overlapping_offset=strategies.integers(min_value=0, max_value=98), maybe_overlapping_length=strategies.integers(min_value=1, max_value=100), @@ -264,6 +264,35 @@ class Bucket(unittest.TestCase): bw.write(40, b"1" * 10) bw.write(60, b"1" * 40) + @given( + offsets=strategies.lists( + strategies.integers(min_value=0, max_value=99), + min_size=20, + max_size=20 + ), + ) + @example(offsets=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 40, 70]) + def test_writes_return_when_finished( + self, offsets + ): + """ + The ``BucketWriter.write()`` return true if and only if the maximum + size has been reached via potentially overlapping writes. + """ + length = 100 + incoming, final = self.make_workdir("overlapping_writes_{}".format(uuid4())) + bw = BucketWriter( + self, incoming, final, length, self.make_lease(), Clock() + ) + local_written = [0] * 100 + for offset in offsets: + length = min(30, 100 - offset) + data = b"1" * length + for i in range(offset, offset+length): + local_written[i] = 1 + finished = bw.write(offset, data) + self.assertEqual(finished, sum(local_written) == 100) + def test_read_past_end_of_share_data(self): # test vector for immutable files (hard-coded contents of an immutable share # file): From 25e2100219ddc067f5090072fb318930a1eb0660 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Jan 2022 14:06:21 -0500 Subject: [PATCH 27/87] Immutable writing now knows when it's finished. --- src/allmydata/storage/immutable.py | 8 +++----- src/allmydata/test/test_storage.py | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 0bcef8246..d17f69c07 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -403,12 +403,10 @@ class BucketWriter(object): self.ss.count("write") # Return whether the whole thing has been written. See - # https://github.com/mlenzen/collections-extended/issues/169 for why + # https://github.com/mlenzen/collections-extended/issues/169 and + # https://github.com/mlenzen/collections-extended/issues/172 for why # it's done this way. - print([tuple(mr) for mr in self._already_written.ranges()]) - return [tuple(mr) for mr in self._already_written.ranges()] == [ - (0, self._max_size, True) - ] + return sum([mr.stop - mr.start for mr in self._already_written.ranges()]) == self._max_size def close(self): precondition(not self.closed) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index e4f01f6f1..881bfb6fd 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -279,10 +279,9 @@ class Bucket(unittest.TestCase): The ``BucketWriter.write()`` return true if and only if the maximum size has been reached via potentially overlapping writes. """ - length = 100 incoming, final = self.make_workdir("overlapping_writes_{}".format(uuid4())) bw = BucketWriter( - self, incoming, final, length, self.make_lease(), Clock() + self, incoming, final, 100, self.make_lease(), Clock() ) local_written = [0] * 100 for offset in offsets: From d4ae7c89aa0ba41b45720bebfe90054bc9e53df7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Jan 2022 14:20:40 -0500 Subject: [PATCH 28/87] First end-to-end immutable upload then download test passes. --- integration/test_storage_http.py | 2 +- src/allmydata/storage/http_client.py | 5 +++-- src/allmydata/storage/http_server.py | 21 ++++++++++++++++++--- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/integration/test_storage_http.py b/integration/test_storage_http.py index 66a8b2af1..0e2cf89a6 100644 --- a/integration/test_storage_http.py +++ b/integration/test_storage_http.py @@ -223,7 +223,7 @@ class ImmutableHTTPAPITests(AsyncTestCase): # We can now read: for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: downloaded = yield im_client.read_share_chunk( - storage_index, 1, upload_secret, offset, length + storage_index, 1, offset, length ) self.assertEqual(downloaded, expected_data[offset : offset + length]) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 697af91b5..b091b3ca7 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -252,12 +252,13 @@ class StorageClientImmutables(object): headers=Headers( { # The range is inclusive. - "range": ["bytes={}-{}".format(offset, offset + length)] + "range": ["bytes={}-{}".format(offset, offset + length - 1)] } ), ) if response.code == 200: - returnValue(response.content.read()) + body = yield response.content() + returnValue(body) else: raise ClientException( response.code, diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 28367752d..50d955127 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -241,6 +241,7 @@ class HTTPServer(object): # TODO if raises ConflictingWriteError, return HTTP CONFLICT code. if finished: + bucket.close() request.setResponseCode(http.CREATED) else: request.setResponseCode(http.OK) @@ -257,8 +258,22 @@ class HTTPServer(object): ) def read_share_chunk(self, request, authorization, storage_index, share_number): """Read a chunk for an already uploaded immutable.""" - # TODO read offset and length from Range header # TODO basic checks on validity - # TODO lookup the share + storage_index = si_a2b(storage_index.encode("ascii")) + range_header = request.getHeader("range") + if range_header is None: + offset = 0 + inclusive_end = None + else: + parts = range_header.split("=")[1].split("-") + offset = int(parts[0]) # TODO make sure valid + if len(parts) > 0: + inclusive_end = int(parts[1]) # TODO make sure valid + else: + inclusive_end = None + + assert inclusive_end != None # TODO support this case + # TODO if not found, 404 - # TODO otherwise, return data from that offset + bucket = self._storage_server.get_buckets(storage_index)[share_number] + return bucket.read(offset, inclusive_end - offset + 1) From 79cd9a3d6d236749943228ed5fcc2287c6187e48 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Jan 2022 14:22:15 -0500 Subject: [PATCH 29/87] Fix lint. --- src/allmydata/test/test_storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 881bfb6fd..27309a82a 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -13,7 +13,6 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str -from array import array from io import ( BytesIO, ) From 7aed7dbd8a7219e257dd0dc638eecd57e26fb66a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Jan 2022 14:24:28 -0500 Subject: [PATCH 30/87] Make module import on Python 2 (so tests can pass). --- src/allmydata/storage/http_client.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index b091b3ca7..8fea86396 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -13,6 +13,11 @@ if PY2: # fmt: off from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # fmt: on + from collections import defaultdict + + Optional = Set = defaultdict( + lambda: None + ) # some garbage to just make this module import else: # typing module not available in Python 2, and we only do type checking in # Python 3 anyway. @@ -131,7 +136,7 @@ class StorageClientImmutables(object): APIs for interacting with immutables. """ - def __init__(self, client: StorageClient): # # type: (StorageClient) -> None + def __init__(self, client): # type: (StorageClient) -> None self._client = client @inlineCallbacks From 28dbdbe019f53dc16e0b6ea4af0822eba61b0842 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Jan 2022 14:31:29 -0500 Subject: [PATCH 31/87] Make sure return type is consistent. --- src/allmydata/storage/immutable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index d17f69c07..0949929a9 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -384,7 +384,7 @@ class BucketWriter(object): start = self._clock.seconds() precondition(not self.closed) if self.throw_out_all_data: - return + return False # Make sure we're not conflicting with existing data: end = offset + len(data) From 406a06a5080c691c9216849bba43100061b8ac3c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Jan 2022 14:38:06 -0500 Subject: [PATCH 32/87] Make sure we don't violate the Foolscap interface definition for this method. --- src/allmydata/storage/immutable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 0949929a9..e35ae9782 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -494,7 +494,7 @@ class FoolscapBucketWriter(Referenceable): # type: ignore # warner/foolscap#78 self._bucket_writer = bucket_writer def remote_write(self, offset, data): - return self._bucket_writer.write(offset, data) + self._bucket_writer.write(offset, data) def remote_close(self): return self._bucket_writer.close() From 23368fc9d95811c2088d430cc0dced995d212527 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 10:34:09 -0500 Subject: [PATCH 33/87] Move tests back into unittest module. --- integration/test_storage_http.py | 284 ------------------------ src/allmydata/test/test_storage_http.py | 275 ++++++++++++++++++++++- 2 files changed, 274 insertions(+), 285 deletions(-) delete mode 100644 integration/test_storage_http.py diff --git a/integration/test_storage_http.py b/integration/test_storage_http.py deleted file mode 100644 index 0e2cf89a6..000000000 --- a/integration/test_storage_http.py +++ /dev/null @@ -1,284 +0,0 @@ -""" -Connect the HTTP storage client to the HTTP storage server and make sure they -can talk to each other. -""" - -from future.utils import PY2 - -from os import urandom - -from twisted.internet.defer import inlineCallbacks -from fixtures import Fixture, TempDir -from treq.testing import StubTreq -from hyperlink import DecodedURL -from klein import Klein - -from allmydata.storage.server import StorageServer -from allmydata.storage.http_server import ( - HTTPServer, - _authorized_route, -) -from allmydata.storage.http_client import ( - StorageClient, - ClientException, - StorageClientImmutables, - ImmutableCreateResult, -) -from allmydata.storage.http_common import Secrets -from allmydata.test.common import AsyncTestCase - - -# TODO should be actual swissnum -SWISSNUM_FOR_TEST = b"abcd" - - -class TestApp(object): - """HTTP API for testing purposes.""" - - _app = Klein() - _swissnum = SWISSNUM_FOR_TEST # Match what the test client is using - - @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"]) - def validate_upload_secret(self, request, authorization): - if authorization == {Secrets.UPLOAD: b"MAGIC"}: - return "GOOD SECRET" - else: - return "BAD: {}".format(authorization) - - -class RoutingTests(AsyncTestCase): - """ - Tests for the HTTP routing infrastructure. - """ - - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(RoutingTests, self).setUp() - # Could be a fixture, but will only be used in this test class so not - # going to bother: - self._http_server = TestApp() - self.client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - SWISSNUM_FOR_TEST, - treq=StubTreq(self._http_server._app.resource()), - ) - - @inlineCallbacks - def test_authorization_enforcement(self): - """ - The requirement for secrets is enforced; if they are not given, a 400 - response code is returned. - """ - # Without secret, get a 400 error. - response = yield self.client._request( - "GET", - "http://127.0.0.1/upload_secret", - ) - self.assertEqual(response.code, 400) - - # With secret, we're good. - response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC" - ) - self.assertEqual(response.code, 200) - self.assertEqual((yield response.content()), b"GOOD SECRET") - - -class HttpTestFixture(Fixture): - """ - Setup HTTP tests' infrastructure, the storage server and corresponding - client. - """ - - def _setUp(self): - self.tempdir = self.useFixture(TempDir()) - self.storage_server = StorageServer(self.tempdir.path, b"\x00" * 20) - self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) - self.client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - SWISSNUM_FOR_TEST, - treq=StubTreq(self.http_server.get_resource()), - ) - - -class GenericHTTPAPITests(AsyncTestCase): - """ - Tests of HTTP client talking to the HTTP server, for generic HTTP API - endpoints and concerns. - """ - - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(GenericHTTPAPITests, self).setUp() - self.http = self.useFixture(HttpTestFixture()) - - @inlineCallbacks - def test_bad_authentication(self): - """ - If the wrong swissnum is used, an ``Unauthorized`` response code is - returned. - """ - client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - b"something wrong", - treq=StubTreq(self.http.http_server.get_resource()), - ) - with self.assertRaises(ClientException) as e: - yield client.get_version() - self.assertEqual(e.exception.args[0], 401) - - @inlineCallbacks - def test_version(self): - """ - The client can return the version. - - We ignore available disk space and max immutable share size, since that - might change across calls. - """ - version = yield self.http.client.get_version() - version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"available-space" - ) - version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"maximum-immutable-share-size" - ) - expected_version = self.http.storage_server.get_version() - expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"available-space" - ) - expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"maximum-immutable-share-size" - ) - self.assertEqual(version, expected_version) - - -class ImmutableHTTPAPITests(AsyncTestCase): - """ - Tests for immutable upload/download APIs. - """ - - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(ImmutableHTTPAPITests, self).setUp() - self.http = self.useFixture(HttpTestFixture()) - - @inlineCallbacks - def test_upload_can_be_downloaded(self): - """ - A single share can be uploaded in (possibly overlapping) chunks, and - then a random chunk can be downloaded, and it will match the original - file. - - We don't exercise the full variation of overlapping chunks because - that's already done in test_storage.py. - """ - length = 100 - expected_data = b"".join(bytes([i]) for i in range(100)) - - im_client = StorageClientImmutables(self.http.client) - - # Create a upload: - upload_secret = urandom(32) - lease_secret = urandom(32) - storage_index = b"".join(bytes([i]) for i in range(16)) - created = yield im_client.create( - storage_index, [1], 100, upload_secret, lease_secret, lease_secret - ) - self.assertEqual( - created, ImmutableCreateResult(already_have=set(), allocated={1}) - ) - - # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. - def write(offset, length): - return im_client.write_share_chunk( - storage_index, - 1, - upload_secret, - offset, - expected_data[offset : offset + length], - ) - - finished = yield write(10, 10) - self.assertFalse(finished) - finished = yield write(30, 10) - self.assertFalse(finished) - finished = yield write(50, 10) - self.assertFalse(finished) - - # Then, an overlapping write with matching data (15-35): - finished = yield write(15, 20) - self.assertFalse(finished) - - # Now fill in the holes: - finished = yield write(0, 10) - self.assertFalse(finished) - finished = yield write(40, 10) - self.assertFalse(finished) - finished = yield write(60, 40) - self.assertTrue(finished) - - # We can now read: - for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: - downloaded = yield im_client.read_share_chunk( - storage_index, 1, offset, length - ) - self.assertEqual(downloaded, expected_data[offset : offset + length]) - - def test_multiple_shares_uploaded_to_different_place(self): - """ - If a storage index has multiple shares, uploads to different shares are - stored separately and can be downloaded separately. - """ - - def test_bucket_allocated_with_new_shares(self): - """ - If some shares already exist, allocating shares indicates only the new - ones were created. - """ - - def test_bucket_allocation_new_upload_key(self): - """ - If a bucket was allocated with one upload key, and a different upload - key is used to allocate the bucket again, the previous download is - cancelled. - """ - - def test_upload_with_wrong_upload_key_fails(self): - """ - Uploading with a key that doesn't match the one used to allocate the - bucket will fail. - """ - - def test_upload_offset_cannot_be_negative(self): - """ - A negative upload offset will be rejected. - """ - - def test_mismatching_upload_fails(self): - """ - If an uploaded chunk conflicts with an already uploaded chunk, a - CONFLICT error is returned. - """ - - def test_read_of_wrong_storage_index_fails(self): - """ - Reading from unknown storage index results in 404. - """ - - def test_read_of_wrong_share_number_fails(self): - """ - Reading from unknown storage index results in 404. - """ - - def test_read_with_negative_offset_fails(self): - """ - The offset for reads cannot be negative. - """ - - def test_read_with_negative_length_fails(self): - """ - The length for reads cannot be negative. - """ diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index aaa455a03..af53efbde 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -15,13 +15,30 @@ if PY2: # fmt: on from base64 import b64encode +from os import urandom + +from twisted.internet.defer import inlineCallbacks from hypothesis import assume, given, strategies as st -from .common import SyncTestCase +from fixtures import Fixture, TempDir +from treq.testing import StubTreq +from klein import Klein +from hyperlink import DecodedURL + +from .common import AsyncTestCase, SyncTestCase +from ..storage.server import StorageServer from ..storage.http_server import ( + HTTPServer, _extract_secrets, Secrets, ClientSecretsException, + _authorized_route, +) +from ..storage.http_client import ( + StorageClient, + ClientException, + StorageClientImmutables, + ImmutableCreateResult, ) @@ -127,3 +144,259 @@ class ExtractSecretsTests(SyncTestCase): """ with self.assertRaises(ClientSecretsException): _extract_secrets(["lease-cancel-secret eA=="], {Secrets.LEASE_RENEW}) + + +# TODO should be actual swissnum +SWISSNUM_FOR_TEST = b"abcd" + + +class TestApp(object): + """HTTP API for testing purposes.""" + + _app = Klein() + _swissnum = SWISSNUM_FOR_TEST # Match what the test client is using + + @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"]) + def validate_upload_secret(self, request, authorization): + if authorization == {Secrets.UPLOAD: b"MAGIC"}: + return "GOOD SECRET" + else: + return "BAD: {}".format(authorization) + + +class RoutingTests(AsyncTestCase): + """ + Tests for the HTTP routing infrastructure. + """ + + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(RoutingTests, self).setUp() + # Could be a fixture, but will only be used in this test class so not + # going to bother: + self._http_server = TestApp() + self.client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self._http_server._app.resource()), + ) + + @inlineCallbacks + def test_authorization_enforcement(self): + """ + The requirement for secrets is enforced; if they are not given, a 400 + response code is returned. + """ + # Without secret, get a 400 error. + response = yield self.client._request( + "GET", + "http://127.0.0.1/upload_secret", + ) + self.assertEqual(response.code, 400) + + # With secret, we're good. + response = yield self.client._request( + "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC" + ) + self.assertEqual(response.code, 200) + self.assertEqual((yield response.content()), b"GOOD SECRET") + + +class HttpTestFixture(Fixture): + """ + Setup HTTP tests' infrastructure, the storage server and corresponding + client. + """ + + def _setUp(self): + self.tempdir = self.useFixture(TempDir()) + self.storage_server = StorageServer(self.tempdir.path, b"\x00" * 20) + self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) + self.client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self.http_server.get_resource()), + ) + + +class GenericHTTPAPITests(AsyncTestCase): + """ + Tests of HTTP client talking to the HTTP server, for generic HTTP API + endpoints and concerns. + """ + + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(GenericHTTPAPITests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + + @inlineCallbacks + def test_bad_authentication(self): + """ + If the wrong swissnum is used, an ``Unauthorized`` response code is + returned. + """ + client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + b"something wrong", + treq=StubTreq(self.http.http_server.get_resource()), + ) + with self.assertRaises(ClientException) as e: + yield client.get_version() + self.assertEqual(e.exception.args[0], 401) + + @inlineCallbacks + def test_version(self): + """ + The client can return the version. + + We ignore available disk space and max immutable share size, since that + might change across calls. + """ + version = yield self.http.client.get_version() + version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"available-space" + ) + version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"maximum-immutable-share-size" + ) + expected_version = self.http.storage_server.get_version() + expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"available-space" + ) + expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"maximum-immutable-share-size" + ) + self.assertEqual(version, expected_version) + + +class ImmutableHTTPAPITests(AsyncTestCase): + """ + Tests for immutable upload/download APIs. + """ + + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(ImmutableHTTPAPITests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + + @inlineCallbacks + def test_upload_can_be_downloaded(self): + """ + A single share can be uploaded in (possibly overlapping) chunks, and + then a random chunk can be downloaded, and it will match the original + file. + + We don't exercise the full variation of overlapping chunks because + that's already done in test_storage.py. + """ + length = 100 + expected_data = b"".join(bytes([i]) for i in range(100)) + + im_client = StorageClientImmutables(self.http.client) + + # Create a upload: + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = b"".join(bytes([i]) for i in range(16)) + created = yield im_client.create( + storage_index, [1], 100, upload_secret, lease_secret, lease_secret + ) + self.assertEqual( + created, ImmutableCreateResult(already_have=set(), allocated={1}) + ) + + # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. + def write(offset, length): + return im_client.write_share_chunk( + storage_index, + 1, + upload_secret, + offset, + expected_data[offset : offset + length], + ) + + finished = yield write(10, 10) + self.assertFalse(finished) + finished = yield write(30, 10) + self.assertFalse(finished) + finished = yield write(50, 10) + self.assertFalse(finished) + + # Then, an overlapping write with matching data (15-35): + finished = yield write(15, 20) + self.assertFalse(finished) + + # Now fill in the holes: + finished = yield write(0, 10) + self.assertFalse(finished) + finished = yield write(40, 10) + self.assertFalse(finished) + finished = yield write(60, 40) + self.assertTrue(finished) + + # We can now read: + for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: + downloaded = yield im_client.read_share_chunk( + storage_index, 1, offset, length + ) + self.assertEqual(downloaded, expected_data[offset : offset + length]) + + def test_multiple_shares_uploaded_to_different_place(self): + """ + If a storage index has multiple shares, uploads to different shares are + stored separately and can be downloaded separately. + """ + + def test_bucket_allocated_with_new_shares(self): + """ + If some shares already exist, allocating shares indicates only the new + ones were created. + """ + + def test_bucket_allocation_new_upload_key(self): + """ + If a bucket was allocated with one upload key, and a different upload + key is used to allocate the bucket again, the previous download is + cancelled. + """ + + def test_upload_with_wrong_upload_key_fails(self): + """ + Uploading with a key that doesn't match the one used to allocate the + bucket will fail. + """ + + def test_upload_offset_cannot_be_negative(self): + """ + A negative upload offset will be rejected. + """ + + def test_mismatching_upload_fails(self): + """ + If an uploaded chunk conflicts with an already uploaded chunk, a + CONFLICT error is returned. + """ + + def test_read_of_wrong_storage_index_fails(self): + """ + Reading from unknown storage index results in 404. + """ + + def test_read_of_wrong_share_number_fails(self): + """ + Reading from unknown storage index results in 404. + """ + + def test_read_with_negative_offset_fails(self): + """ + The offset for reads cannot be negative. + """ + + def test_read_with_negative_length_fails(self): + """ + The length for reads cannot be negative. + """ From 1bf2b2ee5f4bec2df84f2a8bfac8aaa40ca08c95 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 10:52:44 -0500 Subject: [PATCH 34/87] Note follow-up issue. --- src/allmydata/test/test_storage_http.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index af53efbde..2689b429f 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -349,12 +349,16 @@ class ImmutableHTTPAPITests(AsyncTestCase): """ If a storage index has multiple shares, uploads to different shares are stored separately and can be downloaded separately. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_bucket_allocated_with_new_shares(self): """ If some shares already exist, allocating shares indicates only the new ones were created. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_bucket_allocation_new_upload_key(self): @@ -362,41 +366,57 @@ class ImmutableHTTPAPITests(AsyncTestCase): If a bucket was allocated with one upload key, and a different upload key is used to allocate the bucket again, the previous download is cancelled. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_upload_with_wrong_upload_key_fails(self): """ Uploading with a key that doesn't match the one used to allocate the bucket will fail. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_upload_offset_cannot_be_negative(self): """ A negative upload offset will be rejected. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_mismatching_upload_fails(self): """ If an uploaded chunk conflicts with an already uploaded chunk, a CONFLICT error is returned. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_read_of_wrong_storage_index_fails(self): """ Reading from unknown storage index results in 404. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_read_of_wrong_share_number_fails(self): """ Reading from unknown storage index results in 404. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_read_with_negative_offset_fails(self): """ The offset for reads cannot be negative. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_read_with_negative_length_fails(self): """ The length for reads cannot be negative. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ From d5bac8e186859f16c7de5637b601b215a87782c0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 10:56:08 -0500 Subject: [PATCH 35/87] Make sure upload secret semantics are still supporting the security goals. --- docs/proposed/http-storage-node-protocol.rst | 3 ++- src/allmydata/storage/http_server.py | 4 +--- src/allmydata/test/test_storage_http.py | 9 ++++----- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 26f1a2bb7..bb1db750c 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -493,7 +493,8 @@ Handling repeat calls: * If the same API call is repeated with the same upload secret, the response is the same and no change is made to server state. This is necessary to ensure retries work in the face of lost responses from the server. * If the API calls is with a different upload secret, this implies a new client, perhaps because the old client died. - In this case, all relevant in-progress uploads are canceled, and then the command is handled as usual. + In order to prevent storage servers from being able to mess with each other, this API call will fail, because the secret doesn't match. + The use case of restarting upload from scratch if the client dies can be implemented by having the client persist the upload secret. Discussion `````````` diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 50d955127..71c34124a 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -187,9 +187,7 @@ class HTTPServer(object): # TODO add BucketWriters only for new shares pass else: - # New session. - # TODO cancel all existing BucketWriters, then do - # self._storage_server.allocate_buckets() with given inputs. + # TODO Fail, since the secret doesnt match. pass else: # New upload. diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 2689b429f..a7aad608e 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -361,16 +361,15 @@ class ImmutableHTTPAPITests(AsyncTestCase): TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ - def test_bucket_allocation_new_upload_key(self): + def test_bucket_allocation_new_upload_secret(self): """ - If a bucket was allocated with one upload key, and a different upload - key is used to allocate the bucket again, the previous download is - cancelled. + If a bucket was allocated with one upload secret, and a different upload + key is used to allocate the bucket again, the second allocation fails. TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ - def test_upload_with_wrong_upload_key_fails(self): + def test_upload_with_wrong_upload_secret_fails(self): """ Uploading with a key that doesn't match the one used to allocate the bucket will fail. From f09aa8c7969d8455a958c278d3fdb889aae15d71 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 11:16:06 -0500 Subject: [PATCH 36/87] Use pre-existing parser for Range and Content-Range headers. --- docs/proposed/http-storage-node-protocol.rst | 2 +- nix/tahoe-lafs.nix | 4 +- setup.py | 1 + src/allmydata/storage/http_client.py | 2 +- src/allmydata/storage/http_server.py | 41 ++++++++++---------- src/allmydata/test/test_storage_http.py | 2 +- 6 files changed, 27 insertions(+), 25 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index bb1db750c..560220d00 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -640,7 +640,7 @@ For example:: Read a contiguous sequence of bytes from one share in one bucket. The response body is the raw share data (i.e., ``application/octet-stream``). -The ``Range`` header may be used to request exactly one ``bytes`` range. +The ``Range`` header may be used to request exactly one ``bytes`` range, in which case the response code will be 206 (partial content). Interpretation and response behavior is as specified in RFC 7233 § 4.1. Multiple ranges in a single request are *not* supported. diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 04d6c4163..1885dd9ca 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -4,7 +4,7 @@ , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography, netifaces -, html5lib, pyutil, distro, configparser, klein, cbor2 +, html5lib, pyutil, distro, configparser, klein, werkzeug, cbor2 }: python.pkgs.buildPythonPackage rec { # Most of the time this is not exactly the release version (eg 1.17.0). @@ -98,7 +98,7 @@ EOF service-identity pyyaml magic-wormhole eliot autobahn cryptography netifaces setuptools future pyutil distro configparser collections-extended - klein cbor2 treq + klein werkzeug cbor2 treq ]; checkInputs = with python.pkgs; [ diff --git a/setup.py b/setup.py index 7e7a955c6..36e82a2b2 100644 --- a/setup.py +++ b/setup.py @@ -143,6 +143,7 @@ install_requires = [ # HTTP server and client "klein", + "werkzeug", "treq", "cbor2" ] diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 8fea86396..cf453fcfc 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -261,7 +261,7 @@ class StorageClientImmutables(object): } ), ) - if response.code == 200: + if response.code == http.PARTIAL_CONTENT: body = yield response.content() returnValue(body) else: diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 71c34124a..bbb42dbe1 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -22,6 +22,7 @@ from base64 import b64decode from klein import Klein from twisted.web import http import attr +from werkzeug.http import parse_range_header, parse_content_range_header # TODO Make sure to use pure Python versions? from cbor2 import dumps, loads @@ -218,11 +219,12 @@ class HTTPServer(object): def write_share_data(self, request, authorization, storage_index, share_number): """Write data to an in-progress immutable upload.""" storage_index = si_a2b(storage_index.encode("ascii")) - content_range = request.getHeader("content-range") - if content_range is None: - offset = 0 - else: - offset = int(content_range.split()[1].split("-")[0]) + content_range = parse_content_range_header(request.getHeader("content-range")) + # TODO in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 + # 1. Malformed header should result in error + # 2. Non-bytes unit should result in error + # 3. Missing header means full upload in one request + offset = content_range.start # TODO basic checks on validity of start, offset, and content-range in general. also of share_number. # TODO basic check that body isn't infinite. require content-length? or maybe we should require content-range (it's optional now)? if so, needs to be rflected in protocol spec. @@ -256,22 +258,21 @@ class HTTPServer(object): ) def read_share_chunk(self, request, authorization, storage_index, share_number): """Read a chunk for an already uploaded immutable.""" - # TODO basic checks on validity + # TODO in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 + # 1. basic checks on validity on storage index, share number + # 2. missing range header should have response code 200 and return whole thing + # 3. malformed range header should result in error? or return everything? + # 4. non-bytes range results in error + # 5. ranges make sense semantically (positive, etc.) + # 6. multiple ranges fails with error + # 7. missing end of range means "to the end of share" storage_index = si_a2b(storage_index.encode("ascii")) - range_header = request.getHeader("range") - if range_header is None: - offset = 0 - inclusive_end = None - else: - parts = range_header.split("=")[1].split("-") - offset = int(parts[0]) # TODO make sure valid - if len(parts) > 0: - inclusive_end = int(parts[1]) # TODO make sure valid - else: - inclusive_end = None - - assert inclusive_end != None # TODO support this case + range_header = parse_range_header(request.getHeader("range")) + offset, end = range_header.ranges[0] + assert end != None # TODO support this case # TODO if not found, 404 bucket = self._storage_server.get_buckets(storage_index)[share_number] - return bucket.read(offset, inclusive_end - offset + 1) + data = bucket.read(offset, end - offset) + request.setResponseCode(http.PARTIAL_CONTENT) + return data diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index a7aad608e..540e40c16 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -339,7 +339,7 @@ class ImmutableHTTPAPITests(AsyncTestCase): self.assertTrue(finished) # We can now read: - for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: + for offset, length in [(0, 100), (10, 19), (99, 1), (49, 200)]: downloaded = yield im_client.read_share_chunk( storage_index, 1, offset, length ) From 5fa8c78f97c14a378ea51f457093647c88c4b597 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:04:20 -0500 Subject: [PATCH 37/87] Don't use reactor, since it's not necessary. --- src/allmydata/test/test_storage_http.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 540e40c16..8b71666b0 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -25,7 +25,7 @@ from treq.testing import StubTreq from klein import Klein from hyperlink import DecodedURL -from .common import AsyncTestCase, SyncTestCase +from .common import SyncTestCase from ..storage.server import StorageServer from ..storage.http_server import ( HTTPServer, @@ -164,7 +164,7 @@ class TestApp(object): return "BAD: {}".format(authorization) -class RoutingTests(AsyncTestCase): +class RoutingTests(SyncTestCase): """ Tests for the HTTP routing infrastructure. """ @@ -220,7 +220,7 @@ class HttpTestFixture(Fixture): ) -class GenericHTTPAPITests(AsyncTestCase): +class GenericHTTPAPITests(SyncTestCase): """ Tests of HTTP client talking to the HTTP server, for generic HTTP API endpoints and concerns. @@ -272,7 +272,7 @@ class GenericHTTPAPITests(AsyncTestCase): self.assertEqual(version, expected_version) -class ImmutableHTTPAPITests(AsyncTestCase): +class ImmutableHTTPAPITests(SyncTestCase): """ Tests for immutable upload/download APIs. """ From 9a0a19c15a70287d82dbc49e3a102c5a7b62b1d4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:07:58 -0500 Subject: [PATCH 38/87] Reminder we might want to support JSON too. --- src/allmydata/storage/http_server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index bbb42dbe1..236204d66 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -155,6 +155,8 @@ class HTTPServer(object): def _cbor(self, request, data): """Return CBOR-encoded data.""" + # TODO Might want to optionally send JSON someday, based on Accept + # headers, see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861 request.setHeader("Content-Type", "application/cbor") # TODO if data is big, maybe want to use a temporary file eventually... return dumps(data) From 587a510b06a406d47e4d61417830d1f69455fcce Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:38:01 -0500 Subject: [PATCH 39/87] Note a better way to implement this. --- src/allmydata/storage/server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 2daf081e4..0add9806b 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -353,6 +353,9 @@ class StorageServer(service.MultiService): max_space_per_bucket, lease_info, clock=self._clock) if self.no_storage: + # Really this should be done by having a separate class for + # this situation; see + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3862 bw.throw_out_all_data = True bucketwriters[shnum] = bw self._bucket_writers[incominghome] = bw From 2a2ab1ead722f60cc7771819908dd1b2fb35f58c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:39:25 -0500 Subject: [PATCH 40/87] Use a set, not a list, for share numbers. --- src/allmydata/storage/http_client.py | 4 ++-- src/allmydata/test/test_storage_http.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index cf453fcfc..36e745395 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -21,7 +21,7 @@ if PY2: else: # typing module not available in Python 2, and we only do type checking in # Python 3 anyway. - from typing import Union, Set, List, Optional + from typing import Union, Set, Optional from treq.testing import StubTreq from base64 import b64encode @@ -148,7 +148,7 @@ class StorageClientImmutables(object): upload_secret, lease_renew_secret, lease_cancel_secret, - ): # type: (bytes, List[int], int, bytes, bytes, bytes) -> Deferred[ImmutableCreateResult] + ): # type: (bytes, Set[int], int, bytes, bytes, bytes) -> Deferred[ImmutableCreateResult] """ Create a new storage index for an immutable. diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 8b71666b0..a3e7d1640 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -303,7 +303,7 @@ class ImmutableHTTPAPITests(SyncTestCase): lease_secret = urandom(32) storage_index = b"".join(bytes([i]) for i in range(16)) created = yield im_client.create( - storage_index, [1], 100, upload_secret, lease_secret, lease_secret + storage_index, {1}, 100, upload_secret, lease_secret, lease_secret ) self.assertEqual( created, ImmutableCreateResult(already_have=set(), allocated={1}) From b952e738dd60e5b5b6f85cbe232f82a18c828846 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:42:26 -0500 Subject: [PATCH 41/87] Try to clarify. --- src/allmydata/storage/http_client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 36e745395..dca8b761c 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -210,8 +210,10 @@ class StorageClientImmutables(object): headers=Headers( { # The range is inclusive, thus the '- 1'. '*' means "length - # unknown", which isn't technically true but adding it just - # makes things slightly harder for calling API. + # unknown", which isn't technically true but it's not clear + # there's any value in passing it in. The server has to + # handle this case anyway, and requiring share length means + # a bit more work for the calling API with no benefit. "content-range": [ "bytes {}-{}/*".format(offset, offset + len(data) - 1) ] From 4b5c71ffbc21a0ff961c15fb27da521adf294bf8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:50:36 -0500 Subject: [PATCH 42/87] Bit more info. --- src/allmydata/storage/http_server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 236204d66..84c1a2e69 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -223,9 +223,10 @@ class HTTPServer(object): storage_index = si_a2b(storage_index.encode("ascii")) content_range = parse_content_range_header(request.getHeader("content-range")) # TODO in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 - # 1. Malformed header should result in error - # 2. Non-bytes unit should result in error + # 1. Malformed header should result in error 416 + # 2. Non-bytes unit should result in error 416 # 3. Missing header means full upload in one request + # 4. Impossible range should resul tin error 416 offset = content_range.start # TODO basic checks on validity of start, offset, and content-range in general. also of share_number. From 65787e5603215fb2b9df3893662e9cefa8112da3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:57:52 -0500 Subject: [PATCH 43/87] Get rid of inlineCallbacks. --- src/allmydata/test/test_storage_http.py | 67 ++++++++++++++++--------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index a3e7d1640..948b6c718 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -17,8 +17,6 @@ if PY2: from base64 import b64encode from os import urandom -from twisted.internet.defer import inlineCallbacks - from hypothesis import assume, given, strategies as st from fixtures import Fixture, TempDir from treq.testing import StubTreq @@ -164,6 +162,23 @@ class TestApp(object): return "BAD: {}".format(authorization) +def result_of(d): + """ + Synchronously extract the result of a Deferred. + """ + result = [] + error = [] + d.addCallbacks(result.append, error.append) + if result: + return result[0] + if error: + error[0].raiseException() + raise RuntimeError( + "We expected given Deferred to have result already, but it wasn't. " + + "This is probably a test design issue." + ) + + class RoutingTests(SyncTestCase): """ Tests for the HTTP routing infrastructure. @@ -182,25 +197,28 @@ class RoutingTests(SyncTestCase): treq=StubTreq(self._http_server._app.resource()), ) - @inlineCallbacks def test_authorization_enforcement(self): """ The requirement for secrets is enforced; if they are not given, a 400 response code is returned. """ # Without secret, get a 400 error. - response = yield self.client._request( - "GET", - "http://127.0.0.1/upload_secret", + response = result_of( + self.client._request( + "GET", + "http://127.0.0.1/upload_secret", + ) ) self.assertEqual(response.code, 400) # With secret, we're good. - response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC" + response = result_of( + self.client._request( + "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC" + ) ) self.assertEqual(response.code, 200) - self.assertEqual((yield response.content()), b"GOOD SECRET") + self.assertEqual(result_of(response.content()), b"GOOD SECRET") class HttpTestFixture(Fixture): @@ -232,7 +250,6 @@ class GenericHTTPAPITests(SyncTestCase): super(GenericHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) - @inlineCallbacks def test_bad_authentication(self): """ If the wrong swissnum is used, an ``Unauthorized`` response code is @@ -244,10 +261,9 @@ class GenericHTTPAPITests(SyncTestCase): treq=StubTreq(self.http.http_server.get_resource()), ) with self.assertRaises(ClientException) as e: - yield client.get_version() + result_of(client.get_version()) self.assertEqual(e.exception.args[0], 401) - @inlineCallbacks def test_version(self): """ The client can return the version. @@ -255,7 +271,7 @@ class GenericHTTPAPITests(SyncTestCase): We ignore available disk space and max immutable share size, since that might change across calls. """ - version = yield self.http.client.get_version() + version = result_of(self.http.client.get_version()) version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( b"available-space" ) @@ -283,7 +299,6 @@ class ImmutableHTTPAPITests(SyncTestCase): super(ImmutableHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) - @inlineCallbacks def test_upload_can_be_downloaded(self): """ A single share can be uploaded in (possibly overlapping) chunks, and @@ -302,8 +317,10 @@ class ImmutableHTTPAPITests(SyncTestCase): upload_secret = urandom(32) lease_secret = urandom(32) storage_index = b"".join(bytes([i]) for i in range(16)) - created = yield im_client.create( - storage_index, {1}, 100, upload_secret, lease_secret, lease_secret + created = result_of( + im_client.create( + storage_index, {1}, 100, upload_secret, lease_secret, lease_secret + ) ) self.assertEqual( created, ImmutableCreateResult(already_have=set(), allocated={1}) @@ -319,29 +336,29 @@ class ImmutableHTTPAPITests(SyncTestCase): expected_data[offset : offset + length], ) - finished = yield write(10, 10) + finished = result_of(write(10, 10)) self.assertFalse(finished) - finished = yield write(30, 10) + finished = result_of(write(30, 10)) self.assertFalse(finished) - finished = yield write(50, 10) + finished = result_of(write(50, 10)) self.assertFalse(finished) # Then, an overlapping write with matching data (15-35): - finished = yield write(15, 20) + finished = result_of(write(15, 20)) self.assertFalse(finished) # Now fill in the holes: - finished = yield write(0, 10) + finished = result_of(write(0, 10)) self.assertFalse(finished) - finished = yield write(40, 10) + finished = result_of(write(40, 10)) self.assertFalse(finished) - finished = yield write(60, 40) + finished = result_of(write(60, 40)) self.assertTrue(finished) # We can now read: for offset, length in [(0, 100), (10, 19), (99, 1), (49, 200)]: - downloaded = yield im_client.read_share_chunk( - storage_index, 1, offset, length + downloaded = result_of( + im_client.read_share_chunk(storage_index, 1, offset, length) ) self.assertEqual(downloaded, expected_data[offset : offset + length]) From c4d71a4636503df1afa5a7884c473c0674f427f0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 13:10:42 -0500 Subject: [PATCH 44/87] Use abstractions for generating headers on client, note another place we should generate headers. --- src/allmydata/storage/http_client.py | 14 +++++--------- src/allmydata/storage/http_server.py | 6 ++++++ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index dca8b761c..4436f2fd2 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -31,7 +31,7 @@ import attr # TODO Make sure to import Python version? from cbor2 import loads, dumps - +from werkzeug.datastructures import Range, ContentRange from twisted.web.http_headers import Headers from twisted.web import http from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred @@ -209,13 +209,8 @@ class StorageClientImmutables(object): data=data, headers=Headers( { - # The range is inclusive, thus the '- 1'. '*' means "length - # unknown", which isn't technically true but it's not clear - # there's any value in passing it in. The server has to - # handle this case anyway, and requiring share length means - # a bit more work for the calling API with no benefit. "content-range": [ - "bytes {}-{}/*".format(offset, offset + len(data) - 1) + ContentRange("bytes", offset, offset+len(data)).to_header() ] } ), @@ -258,8 +253,9 @@ class StorageClientImmutables(object): url, headers=Headers( { - # The range is inclusive. - "range": ["bytes={}-{}".format(offset, offset + length - 1)] + "range": [ + Range("bytes", [(offset, offset + length)]).to_header() + ] } ), ) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 84c1a2e69..6b792cf06 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -278,4 +278,10 @@ class HTTPServer(object): bucket = self._storage_server.get_buckets(storage_index)[share_number] data = bucket.read(offset, end - offset) request.setResponseCode(http.PARTIAL_CONTENT) + # TODO set content-range on response. We we need to expand the + # BucketReader interface to return share's length. + # + # request.setHeader( + # "content-range", range_header.make_content_range(share_length).to_header() + # ) return data From e8e3a3e663458b4df5cab8553890a75db02c6942 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 21 Jan 2022 11:37:46 -0500 Subject: [PATCH 45/87] Expand. --- src/allmydata/storage/http_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 6b792cf06..73ef8e09e 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -187,7 +187,8 @@ class HTTPServer(object): in_progress = self._uploads[storage_index] if in_progress.upload_key == upload_key: # Same session. - # TODO add BucketWriters only for new shares + # TODO add BucketWriters only for new shares that don't already have buckets; see the HTTP spec for details. + # The backend code may already implement this logic. pass else: # TODO Fail, since the secret doesnt match. From a4cb4837e6ea122a2273bd1560def2550b438664 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 21 Jan 2022 11:43:36 -0500 Subject: [PATCH 46/87] It's a secret, compare it securely. --- src/allmydata/storage/http_server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 73ef8e09e..a19faf1fa 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -131,7 +131,7 @@ class StorageIndexUploads(object): shares = attr.ib() # type: Dict[int,BucketWriter] # The upload key. - upload_key = attr.ib() # type: bytes + upload_secret = attr.ib() # type: bytes class HTTPServer(object): @@ -180,12 +180,12 @@ class HTTPServer(object): """Allocate buckets.""" storage_index = si_a2b(storage_index.encode("ascii")) info = loads(request.content.read()) - upload_key = authorization[Secrets.UPLOAD] + upload_secret = authorization[Secrets.UPLOAD] if storage_index in self._uploads: # Pre-existing upload. in_progress = self._uploads[storage_index] - if in_progress.upload_key == upload_key: + if timing_safe_compare(in_progress.upload_secret, upload_secret): # Same session. # TODO add BucketWriters only for new shares that don't already have buckets; see the HTTP spec for details. # The backend code may already implement this logic. @@ -203,7 +203,7 @@ class HTTPServer(object): allocated_size=info["allocated-size"], ) self._uploads[storage_index] = StorageIndexUploads( - shares=sharenum_to_bucket, upload_key=authorization[Secrets.UPLOAD] + shares=sharenum_to_bucket, upload_secret=authorization[Secrets.UPLOAD] ) return self._cbor( request, From d2e3b74098c2a94c104f395d8c293ee40f0862be Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 21 Jan 2022 12:36:58 -0500 Subject: [PATCH 47/87] Some progress towards upload progress result from the server. --- src/allmydata/storage/http_client.py | 24 ++++++++++--- src/allmydata/storage/http_server.py | 8 +++-- src/allmydata/storage/immutable.py | 10 ++++++ src/allmydata/test/test_storage.py | 6 +++- src/allmydata/test/test_storage_http.py | 48 +++++++++++++++++-------- 5 files changed, 74 insertions(+), 22 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 4436f2fd2..d4837d4ab 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -30,7 +30,7 @@ import attr # TODO Make sure to import Python version? from cbor2 import loads, dumps - +from collections_extended import RangeMap from werkzeug.datastructures import Range, ContentRange from twisted.web.http_headers import Headers from twisted.web import http @@ -131,6 +131,17 @@ class StorageClient(object): returnValue(decoded_response) +@attr.s +class UploadProgress(object): + """ + Progress of immutable upload, per the server. + """ + # True when upload has finished. + finished = attr.ib(type=bool) + # Remaining ranges to upload. + required = attr.ib(type=RangeMap) + + class StorageClientImmutables(object): """ APIs for interacting with immutables. @@ -186,7 +197,7 @@ class StorageClientImmutables(object): @inlineCallbacks def write_share_chunk( self, storage_index, share_number, upload_secret, offset, data - ): # type: (bytes, int, bytes, int, bytes) -> Deferred[bool] + ): # type: (bytes, int, bytes, int, bytes) -> Deferred[UploadProgress] """ Upload a chunk of data for a specific share. @@ -218,14 +229,19 @@ class StorageClientImmutables(object): if response.code == http.OK: # Upload is still unfinished. - returnValue(False) + finished = False elif response.code == http.CREATED: # Upload is done! - returnValue(True) + finished = True else: raise ClientException( response.code, ) + body = loads((yield response.content())) + remaining = RangeMap() + for chunk in body["required"]: + remaining.set(True, chunk["begin"], chunk["end"]) + returnValue(UploadProgress(finished=finished, required=remaining)) @inlineCallbacks def read_share_chunk( diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index a19faf1fa..1d1a9466c 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -23,6 +23,7 @@ from klein import Klein from twisted.web import http import attr from werkzeug.http import parse_range_header, parse_content_range_header +from collections_extended import RangeMap # TODO Make sure to use pure Python versions? from cbor2 import dumps, loads @@ -250,9 +251,10 @@ class HTTPServer(object): else: request.setResponseCode(http.OK) - # TODO spec says we should return missing ranges. but client doesn't - # actually use them? So is it actually useful? - return b"" + required = [] + for start, end, _ in bucket.required_ranges().ranges(): + required.append({"begin": start, "end": end}) + return self._cbor(request, {"required": required}) @_authorized_route( _app, diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index e35ae9782..920bd3c5e 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -372,6 +372,16 @@ class BucketWriter(object): self._clock = clock self._timeout = clock.callLater(30 * 60, self._abort_due_to_timeout) + def required_ranges(self): # type: () -> RangeMap + """ + Return which ranges still need to be written. + """ + result = RangeMap() + result.set(True, 0, self._max_size) + for start, end, _ in self._already_written.ranges(): + result.delete(start, end) + return result + def allocated_size(self): return self._max_size diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 27309a82a..b37f74c24 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -276,7 +276,8 @@ class Bucket(unittest.TestCase): ): """ The ``BucketWriter.write()`` return true if and only if the maximum - size has been reached via potentially overlapping writes. + size has been reached via potentially overlapping writes. The + remaining ranges can be checked via ``BucketWriter.required_ranges()``. """ incoming, final = self.make_workdir("overlapping_writes_{}".format(uuid4())) bw = BucketWriter( @@ -290,6 +291,9 @@ class Bucket(unittest.TestCase): local_written[i] = 1 finished = bw.write(offset, data) self.assertEqual(finished, sum(local_written) == 100) + required_ranges = bw.required_ranges() + for i in range(0, 100): + self.assertEqual(local_written[i] == 1, required_ranges.get(i) is None) def test_read_past_end_of_share_data(self): # test vector for immutable files (hard-coded contents of an immutable share diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 948b6c718..b1eeca4e7 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -22,6 +22,7 @@ from fixtures import Fixture, TempDir from treq.testing import StubTreq from klein import Klein from hyperlink import DecodedURL +from collections_extended import RangeMap from .common import SyncTestCase from ..storage.server import StorageServer @@ -37,6 +38,7 @@ from ..storage.http_client import ( ClientException, StorageClientImmutables, ImmutableCreateResult, + UploadProgress, ) @@ -326,8 +328,12 @@ class ImmutableHTTPAPITests(SyncTestCase): created, ImmutableCreateResult(already_have=set(), allocated={1}) ) + remaining = RangeMap() + remaining.set(True, 0, 100) + # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. def write(offset, length): + remaining.empty(offset, offset + length) return im_client.write_share_chunk( storage_index, 1, @@ -336,24 +342,38 @@ class ImmutableHTTPAPITests(SyncTestCase): expected_data[offset : offset + length], ) - finished = result_of(write(10, 10)) - self.assertFalse(finished) - finished = result_of(write(30, 10)) - self.assertFalse(finished) - finished = result_of(write(50, 10)) - self.assertFalse(finished) + upload_progress = result_of(write(10, 10)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=remaining) + ) + upload_progress = result_of(write(30, 10)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=remaining) + ) + upload_progress = result_of(write(50, 10)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=remaining) + ) # Then, an overlapping write with matching data (15-35): - finished = result_of(write(15, 20)) - self.assertFalse(finished) + upload_progress = result_of(write(15, 20)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=remaining) + ) # Now fill in the holes: - finished = result_of(write(0, 10)) - self.assertFalse(finished) - finished = result_of(write(40, 10)) - self.assertFalse(finished) - finished = result_of(write(60, 40)) - self.assertTrue(finished) + upload_progress = result_of(write(0, 10)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=remaining) + ) + upload_progress = result_of(write(40, 10)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=remaining) + ) + upload_progress = result_of(write(60, 40)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=RangeMap()) + ) # We can now read: for offset, length in [(0, 100), (10, 19), (99, 1), (49, 200)]: From b64e6552a44a5ae0e4149dbcb363fc51fc469fe4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 11:30:41 -0500 Subject: [PATCH 48/87] Fix assertion. --- src/allmydata/test/test_storage_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index b1eeca4e7..dcefc9950 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -372,7 +372,7 @@ class ImmutableHTTPAPITests(SyncTestCase): ) upload_progress = result_of(write(60, 40)) self.assertEqual( - upload_progress, UploadProgress(finished=False, required=RangeMap()) + upload_progress, UploadProgress(finished=True, required=RangeMap()) ) # We can now read: From e9d6eb8d0ec862616b839f3aba0a2b98e5931e3e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 11:30:49 -0500 Subject: [PATCH 49/87] Need some fixes in this version. --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 36e82a2b2..75a034ff5 100644 --- a/setup.py +++ b/setup.py @@ -138,8 +138,11 @@ install_requires = [ # Backported configparser for Python 2: "configparser ; python_version < '3.0'", - # For the RangeMap datastructure. - "collections-extended", + # For the RangeMap datastructure. Need 2.0.2 at least for bugfixes. Python + # 2 doesn't actually need this, since HTTP storage protocol isn't supported + # there, so we just pick whatever version so that code imports. + "collections-extended >= 2.0.2 ; python_version > '3.0'", + "collections-extended ; python_version < '3.0'", # HTTP server and client "klein", From 0346dfea60d92c9864d0e8cd9ac3ec5cb20e719b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jan 2022 09:56:54 -0500 Subject: [PATCH 50/87] Note we can do this now. --- src/allmydata/util/encodingutil.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index f32710688..5e28f59fe 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -320,6 +320,9 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): # Although the problem is that doesn't work in Python 3.6, only 3.7 or # later... For now not thinking about it, just returning unicode since # that is the right thing to do on Python 3. + # + # Now that Python 3.7 is the minimum, this can in theory be done: + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3866 result = result.decode(encoding) return result From 0ad31e33eca75467c3bb5120c7ba05b1b3795323 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jan 2022 09:57:03 -0500 Subject: [PATCH 51/87] Not used. --- misc/python3/Makefile | 53 ------------------------------------------- 1 file changed, 53 deletions(-) delete mode 100644 misc/python3/Makefile diff --git a/misc/python3/Makefile b/misc/python3/Makefile deleted file mode 100644 index f0ef8b12a..000000000 --- a/misc/python3/Makefile +++ /dev/null @@ -1,53 +0,0 @@ -# Python 3 porting targets -# -# NOTE: this Makefile requires GNU make - -### Defensive settings for make: -# https://tech.davis-hansson.com/p/make/ -SHELL := bash -.ONESHELL: -.SHELLFLAGS := -xeu -o pipefail -c -.SILENT: -.DELETE_ON_ERROR: -MAKEFLAGS += --warn-undefined-variables -MAKEFLAGS += --no-builtin-rules - - -# Top-level, phony targets - -.PHONY: default -default: - @echo "no default target" - -.PHONY: test-py3-all-before -## Log the output of running all tests under Python 3 before changes -test-py3-all-before: ../../.tox/make-test-py3-all-old.log -.PHONY: test-py3-all-diff -## Compare the output of running all tests under Python 3 after changes -test-py3-all-diff: ../../.tox/make-test-py3-all.diff - - -# Real targets - -# Gauge the impact of changes on Python 3 compatibility -# Compare the output from running all tests under Python 3 before and after changes. -# Before changes: -# `$ rm -f .tox/make-test-py3-all-*.log && make .tox/make-test-py3-all-old.log` -# After changes: -# `$ make .tox/make-test-py3-all.diff` -$(foreach side,old new,../../.tox/make-test-py3-all-$(side).log): - cd "../../" - tox --develop --notest -e py36-coverage - (make VIRTUAL_ENV=./.tox/py36-coverage TEST_SUITE=allmydata \ - test-venv-coverage || true) | \ - sed -E 's/\([0-9]+\.[0-9]{3} secs\)/(#.### secs)/' | \ - tee "./misc/python3/$(@)" -../../.tox/make-test-py3-all.diff: ../../.tox/make-test-py3-all-new.log - (diff -u "$(<:%-new.log=%-old.log)" "$(<)" || true) | tee "$(@)" - -# Locate modules that are candidates for naively converting `unicode` -> `str`. -# List all Python source files that reference `unicode` but don't reference `str` -../../.tox/py3-unicode-no-str.ls: - cd "../../" - find src -type f -iname '*.py' -exec grep -l -E '\Wunicode\W' '{}' ';' | \ - xargs grep -L '\Wstr\W' | xargs ls -ld | tee "./misc/python3/$(@)" From 2583236ad8ab70e3b44bfd1bccbaf4e02aba8400 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jan 2022 10:56:45 -0500 Subject: [PATCH 52/87] Fix unused import. --- src/allmydata/storage/http_server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 1d1a9466c..d79e9a38b 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -23,7 +23,6 @@ from klein import Klein from twisted.web import http import attr from werkzeug.http import parse_range_header, parse_content_range_header -from collections_extended import RangeMap # TODO Make sure to use pure Python versions? from cbor2 import dumps, loads From 08911a5bddf6f159a7614894ddd04a07425e9a63 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 12:18:23 -0500 Subject: [PATCH 53/87] news fragment --- newsfragments/3867.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3867.minor diff --git a/newsfragments/3867.minor b/newsfragments/3867.minor new file mode 100644 index 000000000..e69de29bb From e482745a0bd4b0cfeb24cd782c99a4e3cfa42f57 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 12:19:24 -0500 Subject: [PATCH 54/87] drop all of the hand-rolled nix packaging expressions --- nix/autobahn.nix | 34 ---------- nix/cbor2.nix | 20 ------ nix/collections-extended.nix | 19 ------ nix/default.nix | 7 -- nix/eliot.nix | 31 --------- nix/future.nix | 35 ---------- nix/overlays.nix | 36 ---------- nix/py3.nix | 7 -- nix/pyutil.nix | 48 ------------- nix/tahoe-lafs.nix | 126 ----------------------------------- nix/twisted.nix | 63 ------------------ 11 files changed, 426 deletions(-) delete mode 100644 nix/autobahn.nix delete mode 100644 nix/cbor2.nix delete mode 100644 nix/collections-extended.nix delete mode 100644 nix/default.nix delete mode 100644 nix/eliot.nix delete mode 100644 nix/future.nix delete mode 100644 nix/overlays.nix delete mode 100644 nix/py3.nix delete mode 100644 nix/pyutil.nix delete mode 100644 nix/tahoe-lafs.nix delete mode 100644 nix/twisted.nix diff --git a/nix/autobahn.nix b/nix/autobahn.nix deleted file mode 100644 index 83148c4f8..000000000 --- a/nix/autobahn.nix +++ /dev/null @@ -1,34 +0,0 @@ -{ lib, buildPythonPackage, fetchPypi, isPy3k, - six, txaio, twisted, zope_interface, cffi, futures, - mock, pytest, cryptography, pynacl -}: -buildPythonPackage rec { - pname = "autobahn"; - version = "19.8.1"; - - src = fetchPypi { - inherit pname version; - sha256 = "294e7381dd54e73834354832604ae85567caf391c39363fed0ea2bfa86aa4304"; - }; - - propagatedBuildInputs = [ six txaio twisted zope_interface cffi cryptography pynacl ] ++ - (lib.optionals (!isPy3k) [ futures ]); - - checkInputs = [ mock pytest ]; - checkPhase = '' - runHook preCheck - USE_TWISTED=true py.test $out - runHook postCheck - ''; - - # Tests do no seem to be compatible yet with pytest 5.1 - # https://github.com/crossbario/autobahn-python/issues/1235 - doCheck = false; - - meta = with lib; { - description = "WebSocket and WAMP in Python for Twisted and asyncio."; - homepage = "https://crossbar.io/autobahn"; - license = licenses.mit; - maintainers = with maintainers; [ nand0p ]; - }; -} diff --git a/nix/cbor2.nix b/nix/cbor2.nix deleted file mode 100644 index 16ca8ff63..000000000 --- a/nix/cbor2.nix +++ /dev/null @@ -1,20 +0,0 @@ -{ lib, buildPythonPackage, fetchPypi, setuptools_scm }: -buildPythonPackage rec { - pname = "cbor2"; - version = "5.2.0"; - - src = fetchPypi { - sha256 = "1gwlgjl70vlv35cgkcw3cg7b5qsmws36hs4mmh0l9msgagjs4fm3"; - inherit pname version; - }; - - doCheck = false; - - propagatedBuildInputs = [ setuptools_scm ]; - - meta = with lib; { - homepage = https://github.com/agronholm/cbor2; - description = "CBOR encoder/decoder"; - license = licenses.mit; - }; -} diff --git a/nix/collections-extended.nix b/nix/collections-extended.nix deleted file mode 100644 index 3f1ad165a..000000000 --- a/nix/collections-extended.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ lib, buildPythonPackage, fetchPypi }: -buildPythonPackage rec { - pname = "collections-extended"; - version = "1.0.3"; - - src = fetchPypi { - inherit pname version; - sha256 = "0lb69x23asd68n0dgw6lzxfclavrp2764xsnh45jm97njdplznkw"; - }; - - # Tests aren't in tarball, for 1.0.3 at least. - doCheck = false; - - meta = with lib; { - homepage = https://github.com/mlenzen/collections-extended; - description = "Extra Python Collections - bags (multisets), setlists (unique list / indexed set), RangeMap and IndexedDict"; - license = licenses.asl20; - }; -} diff --git a/nix/default.nix b/nix/default.nix deleted file mode 100644 index bd7460c2f..000000000 --- a/nix/default.nix +++ /dev/null @@ -1,7 +0,0 @@ -# This is the main entrypoint for the Tahoe-LAFS derivation. -{ pkgs ? import { } }: -# Add our Python packages to nixpkgs to simplify the expression for the -# Tahoe-LAFS derivation. -let pkgs' = pkgs.extend (import ./overlays.nix); -# Evaluate the expression for our Tahoe-LAFS derivation. -in pkgs'.python2.pkgs.callPackage ./tahoe-lafs.nix { } diff --git a/nix/eliot.nix b/nix/eliot.nix deleted file mode 100644 index c5975e990..000000000 --- a/nix/eliot.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ lib, buildPythonPackage, fetchPypi, zope_interface, pyrsistent, boltons -, hypothesis, testtools, pytest }: -buildPythonPackage rec { - pname = "eliot"; - version = "1.7.0"; - - src = fetchPypi { - inherit pname version; - sha256 = "0ylyycf717s5qsrx8b9n6m38vyj2k8328lfhn8y6r31824991wv8"; - }; - - postPatch = '' - substituteInPlace setup.py \ - --replace "boltons >= 19.0.1" boltons - ''; - - # A seemingly random subset of the test suite fails intermittently. After - # Tahoe-LAFS is ported to Python 3 we can update to a newer Eliot and, if - # the test suite continues to fail, maybe it will be more likely that we can - # have upstream fix it for us. - doCheck = false; - - checkInputs = [ testtools pytest hypothesis ]; - propagatedBuildInputs = [ zope_interface pyrsistent boltons ]; - - meta = with lib; { - homepage = https://github.com/itamarst/eliot/; - description = "Logging library that tells you why it happened"; - license = licenses.asl20; - }; -} diff --git a/nix/future.nix b/nix/future.nix deleted file mode 100644 index 814b7c1b5..000000000 --- a/nix/future.nix +++ /dev/null @@ -1,35 +0,0 @@ -{ lib -, buildPythonPackage -, fetchPypi -}: - -buildPythonPackage rec { - pname = "future"; - version = "0.18.2"; - - src = fetchPypi { - inherit pname version; - sha256 = "sha256:0zakvfj87gy6mn1nba06sdha63rn4njm7bhh0wzyrxhcny8avgmi"; - }; - - doCheck = false; - - meta = { - description = "Clean single-source support for Python 3 and 2"; - longDescription = '' - python-future is the missing compatibility layer between Python 2 and - Python 3. It allows you to use a single, clean Python 3.x-compatible - codebase to support both Python 2 and Python 3 with minimal overhead. - - It provides future and past packages with backports and forward ports - of features from Python 3 and 2. It also comes with futurize and - pasteurize, customized 2to3-based scripts that helps you to convert - either Py2 or Py3 code easily to support both Python 2 and 3 in a - single clean Py3-style codebase, module by module. - ''; - homepage = https://python-future.org; - downloadPage = https://github.com/PythonCharmers/python-future/releases; - license = with lib.licenses; [ mit ]; - maintainers = with lib.maintainers; [ prikhi ]; - }; -} diff --git a/nix/overlays.nix b/nix/overlays.nix deleted file mode 100644 index 92f36e93e..000000000 --- a/nix/overlays.nix +++ /dev/null @@ -1,36 +0,0 @@ -self: super: { - python27 = super.python27.override { - packageOverrides = python-self: python-super: { - # eliot is not part of nixpkgs at all at this time. - eliot = python-self.pythonPackages.callPackage ./eliot.nix { }; - - # NixOS autobahn package has trollius as a dependency, although - # it is optional. Trollius is unmaintained and fails on CI. - autobahn = python-super.pythonPackages.callPackage ./autobahn.nix { }; - - # Porting to Python 3 is greatly aided by the future package. A - # slightly newer version than appears in nixos 19.09 is helpful. - future = python-super.pythonPackages.callPackage ./future.nix { }; - - # Need version of pyutil that supports Python 3. The version in 19.09 - # is too old. - pyutil = python-super.pythonPackages.callPackage ./pyutil.nix { }; - - # Need a newer version of Twisted, too. - twisted = python-super.pythonPackages.callPackage ./twisted.nix { }; - - # collections-extended is not part of nixpkgs at this time. - collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { }; - - # cbor2 is not part of nixpkgs at this time. - cbor2 = python-super.pythonPackages.callPackage ./cbor2.nix { }; - }; - }; - - python39 = super.python39.override { - packageOverrides = python-self: python-super: { - # collections-extended is not part of nixpkgs at this time. - collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { }; - }; - }; -} diff --git a/nix/py3.nix b/nix/py3.nix deleted file mode 100644 index 34ede49dd..000000000 --- a/nix/py3.nix +++ /dev/null @@ -1,7 +0,0 @@ -# This is the main entrypoint for the Tahoe-LAFS derivation. -{ pkgs ? import { } }: -# Add our Python packages to nixpkgs to simplify the expression for the -# Tahoe-LAFS derivation. -let pkgs' = pkgs.extend (import ./overlays.nix); -# Evaluate the expression for our Tahoe-LAFS derivation. -in pkgs'.python39.pkgs.callPackage ./tahoe-lafs.nix { } diff --git a/nix/pyutil.nix b/nix/pyutil.nix deleted file mode 100644 index 6852c2acc..000000000 --- a/nix/pyutil.nix +++ /dev/null @@ -1,48 +0,0 @@ -{ stdenv -, buildPythonPackage -, fetchPypi -, setuptoolsDarcs -, setuptoolsTrial -, simplejson -, twisted -, isPyPy -}: - -buildPythonPackage rec { - pname = "pyutil"; - version = "3.3.0"; - - src = fetchPypi { - inherit pname version; - sha256 = "8c4d4bf668c559186389bb9bce99e4b1b871c09ba252a756ccaacd2b8f401848"; - }; - - buildInputs = [ setuptoolsDarcs setuptoolsTrial ] ++ (if doCheck then [ simplejson ] else []); - propagatedBuildInputs = [ twisted ]; - - # Tests fail because they try to write new code into the twisted - # package, apparently some kind of plugin. - doCheck = false; - - prePatch = stdenv.lib.optionalString isPyPy '' - grep -rl 'utf-8-with-signature-unix' ./ | xargs sed -i -e "s|utf-8-with-signature-unix|utf-8|g" - ''; - - meta = with stdenv.lib; { - description = "Pyutil, a collection of mature utilities for Python programmers"; - - longDescription = '' - These are a few data structures, classes and functions which - we've needed over many years of Python programming and which - seem to be of general use to other Python programmers. Many of - the modules that have existed in pyutil over the years have - subsequently been obsoleted by new features added to the - Python language or its standard library, thus showing that - we're not alone in wanting tools like these. - ''; - - homepage = "http://allmydata.org/trac/pyutil"; - license = licenses.gpl2Plus; - }; - -} \ No newline at end of file diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix deleted file mode 100644 index 2b41e676e..000000000 --- a/nix/tahoe-lafs.nix +++ /dev/null @@ -1,126 +0,0 @@ -{ fetchFromGitHub, lib -, git, python -, twisted, foolscap, zfec -, setuptools, setuptoolsTrial, pyasn1, zope_interface -, service-identity, pyyaml, magic-wormhole, treq, appdirs -, beautifulsoup4, eliot, autobahn, cryptography, netifaces -, html5lib, pyutil, distro, configparser, klein, cbor2 -}: -python.pkgs.buildPythonPackage rec { - # Most of the time this is not exactly the release version (eg 1.17.1). - # Give it a `post` component to make it look newer than the release version - # and we'll bump this up at the time of each release. - # - # It's difficult to read the version from Git the way the Python code does - # for two reasons. First, doing so involves populating the Nix expression - # with values from the source. Nix calls this "import from derivation" or - # "IFD" (). This is - # discouraged in most cases - including this one, I think. Second, the - # Python code reads the contents of `.git` to determine its version. `.git` - # is not a reproducable artifact (in the sense of "reproducable builds") so - # it is excluded from the source tree by default. When it is included, the - # package tends to be frequently spuriously rebuilt. - version = "1.17.1.post1"; - name = "tahoe-lafs-${version}"; - src = lib.cleanSourceWith { - src = ../.; - filter = name: type: - let - basename = baseNameOf name; - - split = lib.splitString "."; - join = builtins.concatStringsSep "."; - ext = join (builtins.tail (split basename)); - - # Build up a bunch of knowledge about what kind of file this is. - isTox = type == "directory" && basename == ".tox"; - isTrialTemp = type == "directory" && basename == "_trial_temp"; - isVersion = basename == "_version.py"; - isBytecode = ext == "pyc" || ext == "pyo"; - isBackup = lib.hasSuffix "~" basename; - isTemporary = lib.hasPrefix "#" basename && lib.hasSuffix "#" basename; - isSymlink = type == "symlink"; - isGit = type == "directory" && basename == ".git"; - in - # Exclude all these things - ! (isTox - || isTrialTemp - || isVersion - || isBytecode - || isBackup - || isTemporary - || isSymlink - || isGit - ); - }; - - postPatch = '' - # Chroots don't have /etc/hosts and /etc/resolv.conf, so work around - # that. - for i in $(find src/allmydata/test -type f) - do - sed -i "$i" -e"s/localhost/127.0.0.1/g" - done - - # Some tests are flaky or fail to skip when dependencies are missing. - # This list is over-zealous because it's more work to disable individual - # tests with in a module. - - # Many of these tests don't properly skip when i2p or tor dependencies are - # not supplied (and we are not supplying them). - rm src/allmydata/test/test_i2p_provider.py - rm src/allmydata/test/test_connections.py - rm src/allmydata/test/cli/test_create.py - - # Generate _version.py ourselves since we can't rely on the Python code - # extracting the information from the .git directory we excluded. - cat > src/allmydata/_version.py < /dev/null - ''; - - checkPhase = '' - ${python.interpreter} -m unittest discover -s twisted/test - ''; - # Tests require network - doCheck = false; - - meta = with stdenv.lib; { - homepage = https://twistedmatrix.com/; - description = "Twisted, an event-driven networking engine written in Python"; - longDescription = '' - Twisted is an event-driven networking engine written in Python - and licensed under the MIT license. - ''; - license = licenses.mit; - maintainers = [ ]; - }; -} From 8a1d4617c23b8addf2955931ef1a3e234de916b1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:00:36 -0500 Subject: [PATCH 55/87] Add new Nix packaging using mach-nix.buildPythonPackage --- default.nix | 52 ++++++++++++++ nix/sources.json | 62 +++++++++++++++++ nix/sources.nix | 174 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 288 insertions(+) create mode 100644 default.nix create mode 100644 nix/sources.json create mode 100644 nix/sources.nix diff --git a/default.nix b/default.nix new file mode 100644 index 000000000..4c9ed1cc9 --- /dev/null +++ b/default.nix @@ -0,0 +1,52 @@ +let + sources = import nix/sources.nix; +in +{ pkgsVersion ? "nixpkgs-21.11" +, pkgs ? import sources.${pkgsVersion} { } +, pypiData ? sources.pypi-deps-db +, pythonVersion ? "python37" +, mach-nix ? import sources.mach-nix { + inherit pkgs pypiData; + python = pythonVersion; + } +}: +# The project name, version, and most other metadata are automatically +# extracted from the source. Some requirements are not properly extracted +# and those cases are handled below. The version can only be extracted if +# `setup.py update_version` has been run (this is not at all ideal but it +# seems difficult to fix) - so for now just be sure to run that first. +mach-nix.buildPythonPackage { + # Define the location of the Tahoe-LAFS source to be packaged. Clean up all + # as many of the non-source files (eg the `.git` directory, `~` backup + # files, nix's own `result` symlink, etc) as possible to avoid needing to + # re-build when files that make no difference to the package have changed. + src = pkgs.lib.cleanSource ./.; + + # Define some extra requirements that mach-nix does not automatically detect + # from inspection of the source. We typically don't need to put version + # constraints on any of these requirements. The pypi-deps-db we're + # operating with makes dependency resolution deterministic so as long as it + # works once it will always work. It could be that in the future we update + # pypi-deps-db and an incompatibility arises - in which case it would make + # sense to apply some version constraints here. + requirementsExtra = '' + # mach-nix does not yet support pyproject.toml which means it misses any + # build-time requirements of our dependencies which are declared in such a + # file. Tell it about them here. + setuptools_rust + + # mach-nix does not yet parse environment markers correctly. It misses + # all of our requirements which have an environment marker. Duplicate them + # here. + foolscap + eliot + pyrsistent + ''; + + providers = { + # Through zfec 1.5.5 the wheel has an incorrect runtime dependency + # declared on argparse, not available for recent versions of Python 3. + # Force mach-nix to use the sdist instead, side-stepping this issue. + zfec = "sdist"; + }; +} diff --git a/nix/sources.json b/nix/sources.json new file mode 100644 index 000000000..1169911e2 --- /dev/null +++ b/nix/sources.json @@ -0,0 +1,62 @@ +{ + "mach-nix": { + "branch": "master", + "description": "Create highly reproducible python environments", + "homepage": "", + "owner": "davhau", + "repo": "mach-nix", + "rev": "bdc97ba6b2ecd045a467b008cff4ae337b6a7a6b", + "sha256": "12b3jc0g0ak6s93g3ifvdpwxbyqx276k1kl66bpwz8a67qjbcbwf", + "type": "tarball", + "url": "https://github.com/davhau/mach-nix/archive/bdc97ba6b2ecd045a467b008cff4ae337b6a7a6b.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "niv": { + "branch": "master", + "description": "Easy dependency management for Nix projects", + "homepage": "https://github.com/nmattia/niv", + "owner": "nmattia", + "repo": "niv", + "rev": "5830a4dd348d77e39a0f3c4c762ff2663b602d4c", + "sha256": "1d3lsrqvci4qz2hwjrcnd8h5vfkg8aypq3sjd4g3izbc8frwz5sm", + "type": "tarball", + "url": "https://github.com/nmattia/niv/archive/5830a4dd348d77e39a0f3c4c762ff2663b602d4c.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs": { + "branch": "release-20.03", + "description": "Nix Packages collection", + "homepage": "", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "eb73405ecceb1dc505b7cbbd234f8f94165e2696", + "sha256": "06k21wbyhhvq2f1xczszh3c2934p0m02by3l2ixvd6nkwrqklax7", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/eb73405ecceb1dc505b7cbbd234f8f94165e2696.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs-21.11": { + "branch": "nixos-21.11", + "description": "Nix Packages collection", + "homepage": "", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "6c4b9f1a2fd761e2d384ef86cff0d208ca27fdca", + "sha256": "1yl5gj0mzczhl1j8sl8iqpwa1jzsgr12fdszw9rq13cdig2a2r5f", + "type": "tarball", + "url": "https://github.com/nixos/nixpkgs/archive/6c4b9f1a2fd761e2d384ef86cff0d208ca27fdca.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "pypi-deps-db": { + "branch": "master", + "description": "Probably the most complete python dependency database", + "homepage": "", + "owner": "DavHau", + "repo": "pypi-deps-db", + "rev": "0f6de8bf1f186c275af862ec9667abb95aae8542", + "sha256": "1ygw9pywyl4p25hx761d1sbwl3qjhm630fa36gdf6b649im4mx8y", + "type": "tarball", + "url": "https://github.com/DavHau/pypi-deps-db/archive/0f6de8bf1f186c275af862ec9667abb95aae8542.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + } +} diff --git a/nix/sources.nix b/nix/sources.nix new file mode 100644 index 000000000..1938409dd --- /dev/null +++ b/nix/sources.nix @@ -0,0 +1,174 @@ +# This file has been generated by Niv. + +let + + # + # The fetchers. fetch_ fetches specs of type . + # + + fetch_file = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchurl { inherit (spec) url sha256; name = name'; } + else + pkgs.fetchurl { inherit (spec) url sha256; name = name'; }; + + fetch_tarball = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchTarball { name = name'; inherit (spec) url sha256; } + else + pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; + + fetch_git = name: spec: + let + ref = + if spec ? ref then spec.ref else + if spec ? branch then "refs/heads/${spec.branch}" else + if spec ? tag then "refs/tags/${spec.tag}" else + abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!"; + in + builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; }; + + fetch_local = spec: spec.path; + + fetch_builtin-tarball = name: throw + ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=tarball -a builtin=true''; + + fetch_builtin-url = name: throw + ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=file -a builtin=true''; + + # + # Various helpers + # + + # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 + sanitizeName = name: + ( + concatMapStrings (s: if builtins.isList s then "-" else s) + ( + builtins.split "[^[:alnum:]+._?=-]+" + ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) + ) + ); + + # The set of packages used when specs are fetched using non-builtins. + mkPkgs = sources: system: + let + sourcesNixpkgs = + import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; }; + hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; + hasThisAsNixpkgsPath = == ./.; + in + if builtins.hasAttr "nixpkgs" sources + then sourcesNixpkgs + else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then + import {} + else + abort + '' + Please specify either (through -I or NIX_PATH=nixpkgs=...) or + add a package called "nixpkgs" to your sources.json. + ''; + + # The actual fetching function. + fetch = pkgs: name: spec: + + if ! builtins.hasAttr "type" spec then + abort "ERROR: niv spec ${name} does not have a 'type' attribute" + else if spec.type == "file" then fetch_file pkgs name spec + else if spec.type == "tarball" then fetch_tarball pkgs name spec + else if spec.type == "git" then fetch_git name spec + else if spec.type == "local" then fetch_local spec + else if spec.type == "builtin-tarball" then fetch_builtin-tarball name + else if spec.type == "builtin-url" then fetch_builtin-url name + else + abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; + + # If the environment variable NIV_OVERRIDE_${name} is set, then use + # the path directly as opposed to the fetched source. + replace = name: drv: + let + saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name; + ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; + in + if ersatz == "" then drv else + # this turns the string into an actual Nix path (for both absolute and + # relative paths) + if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}"; + + # Ports of functions for older nix versions + + # a Nix version of mapAttrs if the built-in doesn't exist + mapAttrs = builtins.mapAttrs or ( + f: set: with builtins; + listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) + ); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 + range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 + stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 + stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); + concatMapStrings = f: list: concatStrings (map f list); + concatStrings = builtins.concatStringsSep ""; + + # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331 + optionalAttrs = cond: as: if cond then as else {}; + + # fetchTarball version that is compatible between all the versions of Nix + builtins_fetchTarball = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchTarball; + in + if lessThan nixVersion "1.12" then + fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) + else + fetchTarball attrs; + + # fetchurl version that is compatible between all the versions of Nix + builtins_fetchurl = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchurl; + in + if lessThan nixVersion "1.12" then + fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) + else + fetchurl attrs; + + # Create the final "sources" from the config + mkSources = config: + mapAttrs ( + name: spec: + if builtins.hasAttr "outPath" spec + then abort + "The values in sources.json should not have an 'outPath' attribute" + else + spec // { outPath = replace name (fetch config.pkgs name spec); } + ) config.sources; + + # The "config" used by the fetchers + mkConfig = + { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null + , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile) + , system ? builtins.currentSystem + , pkgs ? mkPkgs sources system + }: rec { + # The sources, i.e. the attribute set of spec name to spec + inherit sources; + + # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers + inherit pkgs; + }; + +in +mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); } From fae80e5da9c558a4700e1f4d78169359ebd02d0f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:23:54 -0500 Subject: [PATCH 56/87] Fix zfec packaging --- default.nix | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 4c9ed1cc9..b6918ecb7 100644 --- a/default.nix +++ b/default.nix @@ -46,7 +46,21 @@ mach-nix.buildPythonPackage { providers = { # Through zfec 1.5.5 the wheel has an incorrect runtime dependency # declared on argparse, not available for recent versions of Python 3. - # Force mach-nix to use the sdist instead, side-stepping this issue. + # Force mach-nix to use the sdist instead. This allows us to apply a + # patch that removes the offending declaration. zfec = "sdist"; }; + + # Define certain overrides to the way Python dependencies are built. + _ = { + # Apply the argparse declaration fix to zfec sdist. + zfec.patches = with pkgs; [ + (fetchpatch { + name = "fix-argparse.patch"; + url = "https://github.com/tahoe-lafs/zfec/commit/c3e736a72cccf44b8e1fb7d6c276400204c6bc1e.patch"; + sha256 = "1md9i2fx1ya7mgcj9j01z58hs3q9pj4ch5is5b5kq4v86cf6x33x"; + }) + ]; + }; + } From c21ca210e35cb40e79105507a9e91675a9fcfef0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:23:59 -0500 Subject: [PATCH 57/87] a note about providers --- default.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/default.nix b/default.nix index b6918ecb7..970fd75ca 100644 --- a/default.nix +++ b/default.nix @@ -43,6 +43,9 @@ mach-nix.buildPythonPackage { pyrsistent ''; + # Specify where mach-nix should find packages for our Python dependencies. + # There are some reasonable defaults so we only need to specify certain + # packages where the default configuration runs into some issue. providers = { # Through zfec 1.5.5 the wheel has an incorrect runtime dependency # declared on argparse, not available for recent versions of Python 3. From 86bcfaa14d3378802a441421649a42b8c7ac3cfd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:24:05 -0500 Subject: [PATCH 58/87] Update CircleCI configuration to the new packaging --- .circleci/config.yml | 21 +++++++++------------ nix/sources.json | 12 ++++++------ 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a650313ed..6fa1106fc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,10 +39,10 @@ workflows: - "centos-8": {} - - "nixos-19-09": + - "nixos-21-05": {} - - "nixos-21-05": + - "nixos-21-11": {} # Test against PyPy 2.7 @@ -441,15 +441,16 @@ jobs: image: "tahoelafsci/fedora:29-py" user: "nobody" - nixos-19-09: &NIXOS + nixos-21.05: &NIXOS docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH image: "nixorg/nix:circleci" environment: - NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09-small.tar.gz" - SOURCE: "nix/" + # Reference the name of a niv-managed nixpkgs source (see `niv show` and + # nix/sources.json) + NIXPKGS: "nixpkgs-21.05" steps: - "checkout" @@ -466,17 +467,13 @@ jobs: # build a couple simple little dependencies that don't take # advantage of multiple cores and we get a little speedup by doing # them in parallel. - nix-build --cores 3 --max-jobs 2 "$SOURCE" + nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "$NIXPKGS" - nixos-21-05: + nixos-21-11: <<: *NIXOS environment: - # Note this doesn't look more similar to the 19.09 NIX_PATH URL because - # there was some internal shuffling by the NixOS project about how they - # publish stable revisions. - NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs/archive/d32b07e6df276d78e3640eb43882b80c9b2b3459.tar.gz" - SOURCE: "nix/py3.nix" + NIXPKGS: "nixpkgs-21.11" typechecks: docker: diff --git a/nix/sources.json b/nix/sources.json index 1169911e2..e0235a3fb 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -23,23 +23,23 @@ "url": "https://github.com/nmattia/niv/archive/5830a4dd348d77e39a0f3c4c762ff2663b602d4c.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, - "nixpkgs": { - "branch": "release-20.03", + "nixpkgs-21.05": { + "branch": "nixos-21.05", "description": "Nix Packages collection", "homepage": "", "owner": "NixOS", "repo": "nixpkgs", - "rev": "eb73405ecceb1dc505b7cbbd234f8f94165e2696", - "sha256": "06k21wbyhhvq2f1xczszh3c2934p0m02by3l2ixvd6nkwrqklax7", + "rev": "0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00", + "sha256": "1mr2qgv5r2nmf6s3gqpcjj76zpsca6r61grzmqngwm0xlh958smx", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/eb73405ecceb1dc505b7cbbd234f8f94165e2696.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "nixpkgs-21.11": { "branch": "nixos-21.11", "description": "Nix Packages collection", "homepage": "", - "owner": "nixos", + "owner": "NixOS", "repo": "nixpkgs", "rev": "6c4b9f1a2fd761e2d384ef86cff0d208ca27fdca", "sha256": "1yl5gj0mzczhl1j8sl8iqpwa1jzsgr12fdszw9rq13cdig2a2r5f", From b47457646c4a6e1c2a83577628b0e0b13ab39d77 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:26:57 -0500 Subject: [PATCH 59/87] Correct naming of the CircleCI job --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6fa1106fc..50191555f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -441,7 +441,7 @@ jobs: image: "tahoelafsci/fedora:29-py" user: "nobody" - nixos-21.05: &NIXOS + nixos-21-05: &NIXOS docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH From 9c964f4acd46d71d03a4b5e753b439bc9b63cc88 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:52:10 -0500 Subject: [PATCH 60/87] generate the version info --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 50191555f..9d9b967f1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -454,6 +454,12 @@ jobs: steps: - "checkout" + - "run": + name: "Generation version" + command: | + # The Nix package doesn't know how to do this part, unfortunately. + nix-shell -p python --run 'python setup.py update_version' + - "run": name: "Build and Test" command: | From 5cab1f7a4c147b6079583582231b021e5a837d7a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:57:09 -0500 Subject: [PATCH 61/87] Get Python this way? --- .circleci/config.yml | 2 +- .circleci/python.nix | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 .circleci/python.nix diff --git a/.circleci/config.yml b/.circleci/config.yml index 9d9b967f1..e4d6c9e93 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -458,7 +458,7 @@ jobs: name: "Generation version" command: | # The Nix package doesn't know how to do this part, unfortunately. - nix-shell -p python --run 'python setup.py update_version' + nix-shell .circleci/python.nix --run 'python setup.py update_version' - "run": name: "Build and Test" diff --git a/.circleci/python.nix b/.circleci/python.nix new file mode 100644 index 000000000..a830ee61b --- /dev/null +++ b/.circleci/python.nix @@ -0,0 +1,11 @@ +# Define a helper environment for incidental Python tasks required on CI. +let + sources = import ../nix/sources.nix; +in +{ pkgs ? import sources."nixpkgs-21.11" { } +}: +pkgs.mkShell { + buildInputs = [ + pkgs.python3 + ]; +} From dea4c7e131c097f875118f363c3c6e4472b8dc86 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:59:32 -0500 Subject: [PATCH 62/87] get setuptools --- .circleci/python.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/python.nix b/.circleci/python.nix index a830ee61b..6e3d79cc1 100644 --- a/.circleci/python.nix +++ b/.circleci/python.nix @@ -6,6 +6,8 @@ in }: pkgs.mkShell { buildInputs = [ - pkgs.python3 + (pkgs.python3.withPackages (ps: [ + ps.setuptools + ])) ]; } From 013e1810e4b0c94bf89701980f830490f1c45e22 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:59:37 -0500 Subject: [PATCH 63/87] try to use a single nixpkgs in each job --- .circleci/config.yml | 6 ++++-- .circleci/python.nix | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e4d6c9e93..f76197bd7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -456,9 +456,11 @@ jobs: - "checkout" - "run": name: "Generation version" - command: | + command: >- # The Nix package doesn't know how to do this part, unfortunately. - nix-shell .circleci/python.nix --run 'python setup.py update_version' + nix-shell .circleci/python.nix + --argstr pkgsVersion "$NIXPKGS" + --run 'python setup.py update_version' - "run": name: "Build and Test" diff --git a/.circleci/python.nix b/.circleci/python.nix index 6e3d79cc1..ecaf9e27c 100644 --- a/.circleci/python.nix +++ b/.circleci/python.nix @@ -2,7 +2,8 @@ let sources = import ../nix/sources.nix; in -{ pkgs ? import sources."nixpkgs-21.11" { } +{ pkgsVersion +, pkgs ? import sources.${pkgsVersion} { } }: pkgs.mkShell { buildInputs = [ From 78c4b98b086241b43b08d8e7e2c531c1e63133f6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:01:40 -0500 Subject: [PATCH 64/87] that comment handles the >- yaml string type badly --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f76197bd7..01aa75e80 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -455,9 +455,9 @@ jobs: steps: - "checkout" - "run": + # The Nix package doesn't know how to do this part, unfortunately. name: "Generation version" command: >- - # The Nix package doesn't know how to do this part, unfortunately. nix-shell .circleci/python.nix --argstr pkgsVersion "$NIXPKGS" --run 'python setup.py update_version' From 5b7f5a9f889c77c2946bfb0027e3a298f9338d33 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:04:21 -0500 Subject: [PATCH 65/87] fix typo --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 01aa75e80..406a8f200 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -456,7 +456,7 @@ jobs: - "checkout" - "run": # The Nix package doesn't know how to do this part, unfortunately. - name: "Generation version" + name: "Generate version" command: >- nix-shell .circleci/python.nix --argstr pkgsVersion "$NIXPKGS" From b2acd0f7d0192bb9c03291e5fda56f9d76f3f43c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:05:59 -0500 Subject: [PATCH 66/87] >- and indentation changes don't interact well blackslashes are more likely to be understood, I guess --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 406a8f200..55c5730a5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -457,9 +457,9 @@ jobs: - "run": # The Nix package doesn't know how to do this part, unfortunately. name: "Generate version" - command: >- - nix-shell .circleci/python.nix - --argstr pkgsVersion "$NIXPKGS" + command: | + nix-shell .circleci/python.nix \ + --argstr pkgsVersion "$NIXPKGS" \ --run 'python setup.py update_version' - "run": From 83a172210c1edf4298aedaed08c73371c22743ae Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:22:35 -0500 Subject: [PATCH 67/87] Switch to Nix 2.3. mach-nix is not compatible with older versions. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 55c5730a5..3752fb7c0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -445,7 +445,7 @@ jobs: docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH - image: "nixorg/nix:circleci" + image: "nixos/nix:2.3.16" environment: # Reference the name of a niv-managed nixpkgs source (see `niv show` and From 5edd96ce6b018f93de2914e5c7a83d48e45f6998 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:31:56 -0500 Subject: [PATCH 68/87] Change around environment management so we can install ssh too The new image does not come with it --- .circleci/config.yml | 14 +++++++++++--- .circleci/{python.nix => env.nix} | 11 +++++------ 2 files changed, 16 insertions(+), 9 deletions(-) rename .circleci/{python.nix => env.nix} (63%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3752fb7c0..499bb16b6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -453,14 +453,22 @@ jobs: NIXPKGS: "nixpkgs-21.05" steps: + - "run": + name: "Install Basic Dependencies" + command: | + nix-env \ + -f .circleci/env.nix \ + --argstr pkgsVersion "$NIXPKGS" \ + --install \ + -A ssh python3 + - "checkout" + - "run": # The Nix package doesn't know how to do this part, unfortunately. name: "Generate version" command: | - nix-shell .circleci/python.nix \ - --argstr pkgsVersion "$NIXPKGS" \ - --run 'python setup.py update_version' + python setup.py update_version - "run": name: "Build and Test" diff --git a/.circleci/python.nix b/.circleci/env.nix similarity index 63% rename from .circleci/python.nix rename to .circleci/env.nix index ecaf9e27c..0225b00c8 100644 --- a/.circleci/python.nix +++ b/.circleci/env.nix @@ -5,10 +5,9 @@ in { pkgsVersion , pkgs ? import sources.${pkgsVersion} { } }: -pkgs.mkShell { - buildInputs = [ - (pkgs.python3.withPackages (ps: [ - ps.setuptools - ])) - ]; +{ + ssh = pkgs.openssh; + python = pkgs.python3.withPackages (ps: [ + ps.setuptools + ]); } From e7bba3dad0909c4c5585270de1deba8d53eae643 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:36:59 -0500 Subject: [PATCH 69/87] cannot use the source before we do the checkout... --- .circleci/config.yml | 11 +++++------ .circleci/env.nix | 13 ------------- 2 files changed, 5 insertions(+), 19 deletions(-) delete mode 100644 .circleci/env.nix diff --git a/.circleci/config.yml b/.circleci/config.yml index 499bb16b6..21e560a89 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -450,17 +450,16 @@ jobs: environment: # Reference the name of a niv-managed nixpkgs source (see `niv show` and # nix/sources.json) - NIXPKGS: "nixpkgs-21.05" + NIXPKGS: "21.05" steps: - "run": name: "Install Basic Dependencies" command: | nix-env \ - -f .circleci/env.nix \ - --argstr pkgsVersion "$NIXPKGS" \ + -I nixpkgs=https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz\ --install \ - -A ssh python3 + -A git openssh python3 - "checkout" @@ -483,13 +482,13 @@ jobs: # build a couple simple little dependencies that don't take # advantage of multiple cores and we get a little speedup by doing # them in parallel. - nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "$NIXPKGS" + nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-$NIXPKGS" nixos-21-11: <<: *NIXOS environment: - NIXPKGS: "nixpkgs-21.11" + NIXPKGS: "21.11" typechecks: docker: diff --git a/.circleci/env.nix b/.circleci/env.nix deleted file mode 100644 index 0225b00c8..000000000 --- a/.circleci/env.nix +++ /dev/null @@ -1,13 +0,0 @@ -# Define a helper environment for incidental Python tasks required on CI. -let - sources = import ../nix/sources.nix; -in -{ pkgsVersion -, pkgs ? import sources.${pkgsVersion} { } -}: -{ - ssh = pkgs.openssh; - python = pkgs.python3.withPackages (ps: [ - ps.setuptools - ]); -} From e4ed98fa64a55984098dc8e147d7eca150c8e366 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:39:30 -0500 Subject: [PATCH 70/87] maybe this is where they may be found --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 21e560a89..ed45e3c3b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -459,7 +459,7 @@ jobs: nix-env \ -I nixpkgs=https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz\ --install \ - -A git openssh python3 + -A nixos.git nixos.openssh nixos.python3 - "checkout" From 7ee55d07e570af74aceb59fac88c6fccad13a2b1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:47:43 -0500 Subject: [PATCH 71/87] Use nix-env less wrong, maybe --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ed45e3c3b..294eacaf4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -457,9 +457,9 @@ jobs: name: "Install Basic Dependencies" command: | nix-env \ - -I nixpkgs=https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz\ + --file https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz \ --install \ - -A nixos.git nixos.openssh nixos.python3 + -A git openssh python3 - "checkout" From 17d2119521b4adbb6d438e15e596f46494f663bf Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:55:34 -0500 Subject: [PATCH 72/87] get setuptools in there --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 294eacaf4..de1966c86 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -459,7 +459,7 @@ jobs: nix-env \ --file https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz \ --install \ - -A git openssh python3 + -A git openssh 'python3.withPackages (ps: [ ps.setuptools ])' - "checkout" From a8033e2c2f734477aa9882f858477be7cb4266fb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:59:29 -0500 Subject: [PATCH 73/87] cannot get python env that way we don't need python until later anyway --- .circleci/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index de1966c86..2b72a4e78 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -459,7 +459,7 @@ jobs: nix-env \ --file https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz \ --install \ - -A git openssh 'python3.withPackages (ps: [ ps.setuptools ])' + -A git openssh - "checkout" @@ -467,7 +467,9 @@ jobs: # The Nix package doesn't know how to do this part, unfortunately. name: "Generate version" command: | - python setup.py update_version + nix-shell \ + -p 'python3.withPackages (ps: [ ps.setuptools ])' \ + --run 'python setup.py update_version' - "run": name: "Build and Test" From 0fb56c9a4890347a1124fcb98f6b459f46699e3a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 15:03:21 -0500 Subject: [PATCH 74/87] I checked, git is there. --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2b72a4e78..11136e04e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -454,12 +454,14 @@ jobs: steps: - "run": + # The nixos/nix image does not include ssh. Install it so the + # `checkout` step will succeed. name: "Install Basic Dependencies" command: | nix-env \ --file https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz \ --install \ - -A git openssh + -A openssh - "checkout" From 136734c198d5bec305b1763aec41a36b321e3308 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 15:09:52 -0500 Subject: [PATCH 75/87] try to use cachix --- .circleci/config.yml | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 11136e04e..7153fd370 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -451,20 +451,33 @@ jobs: # Reference the name of a niv-managed nixpkgs source (see `niv show` and # nix/sources.json) NIXPKGS: "21.05" + # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us + # to push to CACHIX_NAME. + CACHIX_NAME: "tahoe-lafs-opensource" steps: - "run": # The nixos/nix image does not include ssh. Install it so the - # `checkout` step will succeed. + # `checkout` step will succeed. We also want cachix for + # Nix-friendly caching. name: "Install Basic Dependencies" command: | nix-env \ --file https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz \ --install \ - -A openssh + -A openssh cachix bash - "checkout" + - run: + name: "Cachix setup" + # Record the store paths that exist before we did much. There's no + # reason to cache these, they're either in the image or have to be + # retrieved before we can use cachix to restore from cache. + command: | + cachix use "${CACHIX_NAME}" + nix path-info --all > /tmp/store-path-pre-build + - "run": # The Nix package doesn't know how to do this part, unfortunately. name: "Generate version" @@ -488,6 +501,26 @@ jobs: # them in parallel. nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-$NIXPKGS" + - run: + # Send any new store objects to cachix. + name: "Push to Cachix" + when: "always" + command: | + # Cribbed from + # https://circleci.com/blog/managing-secrets-when-you-have-pull-requests-from-outside-contributors/ + if [ -n "$CIRCLE_PR_NUMBER" ]; then + # I'm sure you're thinking "CIRCLE_PR_NUMBER must just be the + # number of the PR being built". Sorry, dear reader, you have + # guessed poorly. It is also conditionally set based on whether + # this is a PR from a fork or not. + # + # https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables + echo "Skipping Cachix push for forked PR." + else + # https://docs.cachix.org/continuous-integration-setup/circleci.html + bash -c "comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_NAME" + fi + nixos-21-11: <<: *NIXOS From ccb6e65c0453ec38f582ae5c1163dee9ab49495e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 15:26:19 -0500 Subject: [PATCH 76/87] make sure CACHIX_NAME is set for both nixos jobs --- .circleci/config.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7153fd370..8e860d497 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -451,9 +451,6 @@ jobs: # Reference the name of a niv-managed nixpkgs source (see `niv show` and # nix/sources.json) NIXPKGS: "21.05" - # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us - # to push to CACHIX_NAME. - CACHIX_NAME: "tahoe-lafs-opensource" steps: - "run": @@ -471,6 +468,11 @@ jobs: - run: name: "Cachix setup" + environment: + # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and + # allows us to push to CACHIX_NAME. We only need this set for + # `cachix use` in this step. + CACHIX_NAME: "tahoe-lafs-opensource" # Record the store paths that exist before we did much. There's no # reason to cache these, they're either in the image or have to be # retrieved before we can use cachix to restore from cache. From f5e1af00c0689e60a54931fa0720f031a7e83f06 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 15:35:23 -0500 Subject: [PATCH 77/87] try using parameters to avoid environment collision the `cachix push` later on also needs CACHIX_NAME so defining it on a single step is not great --- .circleci/config.yml | 47 +++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8e860d497..1ef0e820d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,11 +39,11 @@ workflows: - "centos-8": {} - - "nixos-21-05": - {} + - "nixos": + nixpkgs: "21.05" - - "nixos-21-11": - {} + - "nixos": + nixpkgs: "21.11" # Test against PyPy 2.7 - "pypy27-buster": @@ -441,16 +441,24 @@ jobs: image: "tahoelafsci/fedora:29-py" user: "nobody" - nixos-21-05: &NIXOS + nixos: + parameters: + nixpkgs: + description: >- + Reference the name of a niv-managed nixpkgs source (see `niv show` + and nix/sources.json) + type: "string" + docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH image: "nixos/nix:2.3.16" environment: - # Reference the name of a niv-managed nixpkgs source (see `niv show` and - # nix/sources.json) - NIXPKGS: "21.05" + # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and + # allows us to push to CACHIX_NAME. We only need this set for + # `cachix use` in this step. + CACHIX_NAME: "tahoe-lafs-opensource" steps: - "run": @@ -460,7 +468,7 @@ jobs: name: "Install Basic Dependencies" command: | nix-env \ - --file https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz \ + --file https://github.com/nixos/nixpkgs/archive/nixos-<>.tar.gz \ --install \ -A openssh cachix bash @@ -468,11 +476,6 @@ jobs: - run: name: "Cachix setup" - environment: - # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and - # allows us to push to CACHIX_NAME. We only need this set for - # `cachix use` in this step. - CACHIX_NAME: "tahoe-lafs-opensource" # Record the store paths that exist before we did much. There's no # reason to cache these, they're either in the image or have to be # retrieved before we can use cachix to restore from cache. @@ -489,7 +492,7 @@ jobs: --run 'python setup.py update_version' - "run": - name: "Build and Test" + name: "Build" command: | # CircleCI build environment looks like it has a zillion and a # half cores. Don't let Nix autodetect this high core count @@ -501,7 +504,13 @@ jobs: # build a couple simple little dependencies that don't take # advantage of multiple cores and we get a little speedup by doing # them in parallel. - nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-$NIXPKGS" + nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-<>" + + - "run": + name: "Test" + command: | + # Let it go somewhat wild for the test suite itself + nix-build --cores 8 --argstr pkgsVersion "nixpkgs-<>" tests.nix - run: # Send any new store objects to cachix. @@ -523,12 +532,6 @@ jobs: bash -c "comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_NAME" fi - nixos-21-11: - <<: *NIXOS - - environment: - NIXPKGS: "21.11" - typechecks: docker: - <<: *DOCKERHUB_AUTH From 6154be1a96c967bacc5f507ccac4e6dbe003e737 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 15:37:12 -0500 Subject: [PATCH 78/87] Give the NixOS job instantiations nice names --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1ef0e820d..3b76c0fb9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,9 +40,11 @@ workflows: {} - "nixos": + name: "NixOS 21.05" nixpkgs: "21.05" - "nixos": + name: "NixOS 21.11" nixpkgs: "21.11" # Test against PyPy 2.7 From 60dd2ee413dcc5f2b2dc2bc484a9fc6a9a73e5d7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 16:17:37 -0500 Subject: [PATCH 79/87] Document the parameters and also accept an `extras` parameter --- default.nix | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/default.nix b/default.nix index 970fd75ca..d5acbbdd7 100644 --- a/default.nix +++ b/default.nix @@ -1,14 +1,27 @@ let sources = import nix/sources.nix; in -{ pkgsVersion ? "nixpkgs-21.11" -, pkgs ? import sources.${pkgsVersion} { } -, pypiData ? sources.pypi-deps-db -, pythonVersion ? "python37" -, mach-nix ? import sources.mach-nix { +{ + pkgsVersion ? "nixpkgs-21.11" # a string which choses a nixpkgs from the + # niv-managed sources data + +, pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself + +, pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use + # for dependency resolution + +, pythonVersion ? "python37" # a string chosing the python derivation from + # nixpkgs to target + +, extras ? [] # a list of strings identifying tahoe-lafs extras, the + # dependencies of which the resulting package will also depend + # on + +, mach-nix ? import sources.mach-nix { # the mach-nix package to use to build + # the tahoe-lafs package inherit pkgs pypiData; python = pythonVersion; - } +} }: # The project name, version, and most other metadata are automatically # extracted from the source. Some requirements are not properly extracted @@ -22,6 +35,9 @@ mach-nix.buildPythonPackage { # re-build when files that make no difference to the package have changed. src = pkgs.lib.cleanSource ./.; + # Select whichever package extras were requested. + inherit extras; + # Define some extra requirements that mach-nix does not automatically detect # from inspection of the source. We typically don't need to put version # constraints on any of these requirements. The pypi-deps-db we're From f03f5fb8d7d3205c25edc94baee2c8886b3092cb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 16:18:43 -0500 Subject: [PATCH 80/87] Add an expression for running the test suite --- default.nix | 6 +++++- tests.nix | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 tests.nix diff --git a/default.nix b/default.nix index d5acbbdd7..044f59e7b 100644 --- a/default.nix +++ b/default.nix @@ -80,6 +80,10 @@ mach-nix.buildPythonPackage { sha256 = "1md9i2fx1ya7mgcj9j01z58hs3q9pj4ch5is5b5kq4v86cf6x33x"; }) ]; - }; + # Remove a click-default-group patch for a test suite problem which no + # longer applies because the project apparently no longer has a test suite + # in its source distribution. + click-default-group.patches = []; + }; } diff --git a/tests.nix b/tests.nix new file mode 100644 index 000000000..364407e87 --- /dev/null +++ b/tests.nix @@ -0,0 +1,26 @@ +let + sources = import nix/sources.nix; +in +# See default.nix for documentation about parameters. +{ pkgsVersion ? "nixpkgs-21.11" +, pkgs ? import sources.${pkgsVersion} { } +, pypiData ? sources.pypi-deps-db +, pythonVersion ? "python37" +, mach-nix ? import sources.mach-nix { + inherit pkgs pypiData; + python = pythonVersion; + } +}@args: +let + # Get the package with all of its test requirements. + tahoe-lafs = import ./. (args // { extras = [ "test" ]; }); + + # Put it into a Python environment. + python-env = pkgs.${pythonVersion}.withPackages (ps: [ + tahoe-lafs + ]); +in +# Make a derivation that runs the unit test suite. +pkgs.runCommand "tahoe-lafs-tests" { } '' + ${python-env}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata +'' From 16fd427b153551f139a39c5e9d371a2611da3a39 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 16:27:10 -0500 Subject: [PATCH 81/87] Get undetected txi2p-tahoe test dependency into the test environment --- default.nix | 6 +++++- tests.nix | 12 +++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/default.nix b/default.nix index 044f59e7b..03ab89a4e 100644 --- a/default.nix +++ b/default.nix @@ -28,7 +28,7 @@ in # and those cases are handled below. The version can only be extracted if # `setup.py update_version` has been run (this is not at all ideal but it # seems difficult to fix) - so for now just be sure to run that first. -mach-nix.buildPythonPackage { +mach-nix.buildPythonPackage rec { # Define the location of the Tahoe-LAFS source to be packaged. Clean up all # as many of the non-source files (eg the `.git` directory, `~` backup # files, nix's own `result` symlink, etc) as possible to avoid needing to @@ -86,4 +86,8 @@ mach-nix.buildPythonPackage { # in its source distribution. click-default-group.patches = []; }; + + passthru.meta.mach-nix = { + inherit providers _; + }; } diff --git a/tests.nix b/tests.nix index 364407e87..5b6eae497 100644 --- a/tests.nix +++ b/tests.nix @@ -16,9 +16,15 @@ let tahoe-lafs = import ./. (args // { extras = [ "test" ]; }); # Put it into a Python environment. - python-env = pkgs.${pythonVersion}.withPackages (ps: [ - tahoe-lafs - ]); + python-env = mach-nix.mkPython { + inherit (tahoe-lafs.meta.mach-nix) providers _; + packagesExtra = [ tahoe-lafs ]; + requirements = '' + # txi2p-tahoe is another dependency with an environment marker that + # mach-nix doesn't automatically pick up. + txi2p-tahoe + ''; + }; in # Make a derivation that runs the unit test suite. pkgs.runCommand "tahoe-lafs-tests" { } '' From f5de5fc1271dc434e044fc3c04f523a031b80b05 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 16:41:38 -0500 Subject: [PATCH 82/87] produce and output so the build appears to be a success --- tests.nix | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests.nix b/tests.nix index 5b6eae497..3816d6ad8 100644 --- a/tests.nix +++ b/tests.nix @@ -29,4 +29,14 @@ in # Make a derivation that runs the unit test suite. pkgs.runCommand "tahoe-lafs-tests" { } '' ${python-env}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata + + # It's not cool to put the whole _trial_temp into $out because it has weird + # files in it we don't want in the store. Plus, even all of the less weird + # files are mostly just trash that's not meaningful if the test suite passes + # (which is the only way we get $out anyway). + # + # The build log itself is typically available from `nix-store --read-log` so + # we don't need to record that either. + echo "passed" >$out + '' From e4505cd7b88c981983229fc5f040bc02d4eb3f66 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Jan 2022 09:56:57 -0500 Subject: [PATCH 83/87] change the strategy for building the test environment it's not clear to me if this is conceptually better or worse than what it replaces but it is about 25% faster --- default.nix | 9 +++++--- tests.nix | 61 ++++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/default.nix b/default.nix index 03ab89a4e..7abaa4c5a 100644 --- a/default.nix +++ b/default.nix @@ -13,9 +13,12 @@ in , pythonVersion ? "python37" # a string chosing the python derivation from # nixpkgs to target -, extras ? [] # a list of strings identifying tahoe-lafs extras, the - # dependencies of which the resulting package will also depend - # on +, extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras, + # the dependencies of which the resulting package + # will also depend on. Include all of the runtime + # extras by default because the incremental cost of + # including them is a lot smaller than the cost of + # re-building the whole thing to add them. , mach-nix ? import sources.mach-nix { # the mach-nix package to use to build # the tahoe-lafs package diff --git a/tests.nix b/tests.nix index 3816d6ad8..53a8885c0 100644 --- a/tests.nix +++ b/tests.nix @@ -12,17 +12,62 @@ in } }@args: let - # Get the package with all of its test requirements. - tahoe-lafs = import ./. (args // { extras = [ "test" ]; }); + # We would like to know the test requirements but mach-nix does not directly + # expose this information to us. However, it is perfectly capable of + # determining it if we ask right... This is probably not meant to be a + # public mach-nix API but we pinned mach-nix so we can deal with mach-nix + # upgrade breakage in our own time. + mach-lib = import "${sources.mach-nix}/mach_nix/nix/lib.nix" { + inherit pkgs; + lib = pkgs.lib; + }; + tests_require = (mach-lib.extract "python37" ./. "extras_require" ).extras_require.test; - # Put it into a Python environment. + # Get the Tahoe-LAFS package itself. This does not include test + # requirements and we don't ask for test requirements so that we can just + # re-use the normal package if it is already built. + tahoe-lafs = import ./. args; + + # If we want to get tahoe-lafs into a Python environment with a bunch of + # *other* Python modules and let them interact in the usual way then we have + # to ask mach-nix for tahoe-lafs and those other Python modules in the same + # way - i.e., using `requirements`. The other tempting mechanism, + # `packagesExtra`, inserts an extra layer of Python environment and prevents + # normal interaction between Python modules (as well as usually producing + # file collisions in the packages that are both runtime and test + # dependencies). To get the tahoe-lafs we just built into the environment, + # put it into nixpkgs using an overlay and tell mach-nix to get tahoe-lafs + # from nixpkgs. + overridesPre = [(self: super: { inherit tahoe-lafs; })]; + providers = tahoe-lafs.meta.mach-nix.providers // { tahoe-lafs = "nixpkgs"; }; + + # Make the Python environment in which we can run the tests. python-env = mach-nix.mkPython { - inherit (tahoe-lafs.meta.mach-nix) providers _; - packagesExtra = [ tahoe-lafs ]; + # Get the packaging fixes we already know we need from putting together + # the runtime package. + inherit (tahoe-lafs.meta.mach-nix) _; + # Share the runtime package's provider configuration - combined with our + # own that causes the right tahoe-lafs to be picked up. + inherit providers overridesPre; requirements = '' - # txi2p-tahoe is another dependency with an environment marker that - # mach-nix doesn't automatically pick up. - txi2p-tahoe + # Here we pull in the Tahoe-LAFS package itself. + tahoe-lafs + + # Unfortunately mach-nix misses all of the Python dependencies of the + # tahoe-lafs satisfied from nixpkgs. Drag them in here. This gives a + # bit of a pyrrhic flavor to the whole endeavor but maybe mach-nix will + # fix this soon. + # + # https://github.com/DavHau/mach-nix/issues/123 + # https://github.com/DavHau/mach-nix/pull/386 + ${tahoe-lafs.requirements} + + # And then all of the test-only dependencies. + ${builtins.concatStringsSep "\n" tests_require} + + # txi2p-tahoe is another dependency with an environment marker that + # mach-nix doesn't automatically pick up. + txi2p-tahoe ''; }; in From d6e82d1d56e1a6787645c134de66f64b1048aa83 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Jan 2022 10:37:43 -0500 Subject: [PATCH 84/87] explain this unfortunate cache step --- .circleci/config.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3b76c0fb9..daf985567 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -530,7 +530,24 @@ jobs: # https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables echo "Skipping Cachix push for forked PR." else + # If this *isn't* a build from a fork then we have the Cachix + # write key in our environment and we can push any new objects + # to Cachix. + # + # To decide what to push, we inspect the list of store objects + # that existed before and after we did most of our work. Any + # that are new after the work is probably a useful thing to have + # around so push it to the cache. We exclude all derivation + # objects (.drv files) because they're cheap to reconstruct and + # by the time you know their cache key you've already done all + # the work anyway. + # + # This shell expression for finding the objects and pushing them + # was from the Cachix docs: + # # https://docs.cachix.org/continuous-integration-setup/circleci.html + # + # but they seem to have removed it now. bash -c "comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_NAME" fi From 005a7622699c74c594ecdbe2b5be53cfa84ba8e9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Jan 2022 10:46:10 -0500 Subject: [PATCH 85/87] spelling --- default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/default.nix b/default.nix index 7abaa4c5a..cecb5579a 100644 --- a/default.nix +++ b/default.nix @@ -2,7 +2,7 @@ let sources = import nix/sources.nix; in { - pkgsVersion ? "nixpkgs-21.11" # a string which choses a nixpkgs from the + pkgsVersion ? "nixpkgs-21.11" # a string which chooses a nixpkgs from the # niv-managed sources data , pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself @@ -10,7 +10,7 @@ in , pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use # for dependency resolution -, pythonVersion ? "python37" # a string chosing the python derivation from +, pythonVersion ? "python37" # a string choosing the python derivation from # nixpkgs to target , extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras, From 9ba17ba8d1731ba004587144d0e5b0b63f870fcd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Jan 2022 10:46:13 -0500 Subject: [PATCH 86/87] explain sources.nix a bit --- default.nix | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/default.nix b/default.nix index cecb5579a..a8a7ba1c8 100644 --- a/default.nix +++ b/default.nix @@ -1,4 +1,23 @@ let + # sources.nix contains information about which versions of some of our + # dependencies we should use. since we use it to pin nixpkgs and the PyPI + # package database, roughly all the rest of our dependencies are *also* + # pinned - indirectly. + # + # sources.nix is managed using a tool called `niv`. as an example, to + # update to the most recent version of nixpkgs from the 21.11 maintenance + # release, in the top-level tahoe-lafs checkout directory you run: + # + # niv update nixpkgs-21.11 + # + # or, to update the PyPI package database -- which is necessary to make any + # newly released packages visible -- you likewise run: + # + # niv update pypi-deps-db + # + # niv also supports chosing a specific revision, following a different + # branch, etc. find complete documentation for the tool at + # https://github.com/nmattia/niv sources = import nix/sources.nix; in { From 03bc39ed771f2c0446a511821d280d9772cfce2a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 31 Jan 2022 11:30:41 -0500 Subject: [PATCH 87/87] Try to fix nix builds. --- default.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/default.nix b/default.nix index a8a7ba1c8..095c54578 100644 --- a/default.nix +++ b/default.nix @@ -73,12 +73,13 @@ mach-nix.buildPythonPackage rec { # file. Tell it about them here. setuptools_rust - # mach-nix does not yet parse environment markers correctly. It misses - # all of our requirements which have an environment marker. Duplicate them - # here. + # mach-nix does not yet parse environment markers (e.g. "python > '3.0'") + # correctly. It misses all of our requirements which have an environment marker. + # Duplicate them here. foolscap eliot pyrsistent + collections-extended ''; # Specify where mach-nix should find packages for our Python dependencies.