From 4990d90ae65f60afbbf1b47147a3e14640690f44 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 13 Oct 2020 14:34:11 -0600 Subject: [PATCH 001/463] update NEWS.txt for release --- NEWS.rst | 46 +++++++++++++++++++++++++++++++++ newsfragments/1792.feature | 1 - newsfragments/2755.other | 1 - newsfragments/3247.minor | 0 newsfragments/3254.minor | 0 newsfragments/3263.other | 0 newsfragments/3277.minor | 0 newsfragments/3278.minor | 0 newsfragments/3284.removed | 1 - newsfragments/3287.minor | 0 newsfragments/3288.minor | 0 newsfragments/3289.minor | 0 newsfragments/3290.minor | 0 newsfragments/3291.minor | 0 newsfragments/3292.minor | 0 newsfragments/3293.minor | 0 newsfragments/3294.minor | 0 newsfragments/3296.installation | 1 - newsfragments/3297.minor | 0 newsfragments/3298.minor | 0 newsfragments/3299.minor | 0 newsfragments/3300.minor | 0 newsfragments/3302.minor | 0 newsfragments/3303.minor | 0 newsfragments/3304.minor | 0 newsfragments/3305.minor | 0 newsfragments/3306.minor | 0 newsfragments/3308.minor | 0 newsfragments/3309.minor | 0 newsfragments/3312.bugfix | 1 - newsfragments/3313.minor | 1 - newsfragments/3315.minor | 1 - newsfragments/3316.minor | 1 - newsfragments/3317.feature | 1 - newsfragments/3320.minor | 0 newsfragments/3323.removed | 1 - newsfragments/3324.other | 0 newsfragments/3325.minor | 0 newsfragments/3326.minor | 0 newsfragments/3328.installation | 1 - newsfragments/3329.minor | 0 newsfragments/3330.minor | 0 newsfragments/3331.minor | 0 newsfragments/3332.minor | 0 newsfragments/3333.minor | 0 newsfragments/3334.minor | 0 newsfragments/3335.minor | 0 newsfragments/3336.minor | 0 newsfragments/3338.minor | 0 newsfragments/3339.minor | 0 newsfragments/3340.minor | 0 newsfragments/3341.minor | 0 newsfragments/3342.minor | 0 newsfragments/3343.minor | 0 newsfragments/3344.minor | 0 newsfragments/3346.minor | 0 newsfragments/3348.bugfix | 1 - newsfragments/3349.bugfix | 1 - newsfragments/3351.minor | 0 newsfragments/3353.minor | 0 newsfragments/3354.minor | 1 - newsfragments/3355.other | 1 - newsfragments/3356.minor | 0 newsfragments/3357.minor | 1 - newsfragments/3358.minor | 0 newsfragments/3359.minor | 0 newsfragments/3361.minor | 0 newsfragments/3364.minor | 0 newsfragments/3365.minor | 0 newsfragments/3366.minor | 0 newsfragments/3367.minor | 0 newsfragments/3368.minor | 0 newsfragments/3370.minor | 0 newsfragments/3372.minor | 1 - newsfragments/3373.minor | 0 newsfragments/3374.minor | 0 newsfragments/3375.minor | 0 newsfragments/3376.minor | 0 newsfragments/3377.minor | 0 newsfragments/3378.minor | 0 newsfragments/3380.minor | 0 newsfragments/3381.minor | 0 newsfragments/3382.minor | 0 newsfragments/3383.minor | 0 newsfragments/3386.minor | 0 newsfragments/3387.minor | 0 newsfragments/3388.minor | 0 newsfragments/3389.minor | 0 newsfragments/3391.minor | 0 newsfragments/3392.minor | 0 newsfragments/3393.minor | 0 newsfragments/3394.minor | 0 newsfragments/3395.minor | 0 newsfragments/3396.minor | 0 newsfragments/3397.minor | 0 newsfragments/3398.minor | 1 - newsfragments/3401.minor | 0 newsfragments/3403.minor | 0 newsfragments/3406.minor | 0 newsfragments/3408.minor | 0 newsfragments/3409.minor | 0 newsfragments/3411.minor | 0 newsfragments/3415.minor | 0 newsfragments/3416.minor | 0 newsfragments/3417.minor | 0 newsfragments/3421.minor | 1 - newsfragments/3422.minor | 0 newsfragments/3423.minor | 0 newsfragments/3424.minor | 0 newsfragments/3425.minor | 0 newsfragments/3426.minor | 0 newsfragments/3427.minor | 0 newsfragments/3429.minor | 0 newsfragments/3430.minor | 0 newsfragments/3431.minor | 0 newsfragments/3436.minor | 0 newsfragments/3437.minor | 0 newsfragments/3438.minor | 0 newsfragments/3439.minor | 0 newsfragments/3440.minor | 0 newsfragments/3442.minor | 1 - newsfragments/3443.minor | 0 newsfragments/3446.minor | 0 newsfragments/3448.minor | 1 - newsfragments/3449.minor | 0 newsfragments/3450.minor | 0 newsfragments/3451.minor | 0 newsfragments/3452.minor | 0 newsfragments/3453.minor | 0 newsfragments/3455.minor | 1 - newsfragments/3456.minor | 0 newsfragments/3458.minor | 0 newsfragments/3462.minor | 0 newsfragments/3463.minor | 0 newsfragments/3464.minor | 1 - 135 files changed, 46 insertions(+), 23 deletions(-) delete mode 100644 newsfragments/1792.feature delete mode 100644 newsfragments/2755.other delete mode 100644 newsfragments/3247.minor delete mode 100644 newsfragments/3254.minor delete mode 100644 newsfragments/3263.other delete mode 100644 newsfragments/3277.minor delete mode 100644 newsfragments/3278.minor delete mode 100644 newsfragments/3284.removed delete mode 100644 newsfragments/3287.minor delete mode 100644 newsfragments/3288.minor delete mode 100644 newsfragments/3289.minor delete mode 100644 newsfragments/3290.minor delete mode 100644 newsfragments/3291.minor delete mode 100644 newsfragments/3292.minor delete mode 100644 newsfragments/3293.minor delete mode 100644 newsfragments/3294.minor delete mode 100644 newsfragments/3296.installation delete mode 100644 newsfragments/3297.minor delete mode 100644 newsfragments/3298.minor delete mode 100644 newsfragments/3299.minor delete mode 100644 newsfragments/3300.minor delete mode 100644 newsfragments/3302.minor delete mode 100644 newsfragments/3303.minor delete mode 100644 newsfragments/3304.minor delete mode 100644 newsfragments/3305.minor delete mode 100644 newsfragments/3306.minor delete mode 100644 newsfragments/3308.minor delete mode 100644 newsfragments/3309.minor delete mode 100644 newsfragments/3312.bugfix delete mode 100644 newsfragments/3313.minor delete mode 100644 newsfragments/3315.minor delete mode 100644 newsfragments/3316.minor delete mode 100644 newsfragments/3317.feature delete mode 100644 newsfragments/3320.minor delete mode 100644 newsfragments/3323.removed delete mode 100644 newsfragments/3324.other delete mode 100644 newsfragments/3325.minor delete mode 100644 newsfragments/3326.minor delete mode 100644 newsfragments/3328.installation delete mode 100644 newsfragments/3329.minor delete mode 100644 newsfragments/3330.minor delete mode 100644 newsfragments/3331.minor delete mode 100644 newsfragments/3332.minor delete mode 100644 newsfragments/3333.minor delete mode 100644 newsfragments/3334.minor delete mode 100644 newsfragments/3335.minor delete mode 100644 newsfragments/3336.minor delete mode 100644 newsfragments/3338.minor delete mode 100644 newsfragments/3339.minor delete mode 100644 newsfragments/3340.minor delete mode 100644 newsfragments/3341.minor delete mode 100644 newsfragments/3342.minor delete mode 100644 newsfragments/3343.minor delete mode 100644 newsfragments/3344.minor delete mode 100644 newsfragments/3346.minor delete mode 100644 newsfragments/3348.bugfix delete mode 100644 newsfragments/3349.bugfix delete mode 100644 newsfragments/3351.minor delete mode 100644 newsfragments/3353.minor delete mode 100644 newsfragments/3354.minor delete mode 100644 newsfragments/3355.other delete mode 100644 newsfragments/3356.minor delete mode 100644 newsfragments/3357.minor delete mode 100644 newsfragments/3358.minor delete mode 100644 newsfragments/3359.minor delete mode 100644 newsfragments/3361.minor delete mode 100644 newsfragments/3364.minor delete mode 100644 newsfragments/3365.minor delete mode 100644 newsfragments/3366.minor delete mode 100644 newsfragments/3367.minor delete mode 100644 newsfragments/3368.minor delete mode 100644 newsfragments/3370.minor delete mode 100644 newsfragments/3372.minor delete mode 100644 newsfragments/3373.minor delete mode 100644 newsfragments/3374.minor delete mode 100644 newsfragments/3375.minor delete mode 100644 newsfragments/3376.minor delete mode 100644 newsfragments/3377.minor delete mode 100644 newsfragments/3378.minor delete mode 100644 newsfragments/3380.minor delete mode 100644 newsfragments/3381.minor delete mode 100644 newsfragments/3382.minor delete mode 100644 newsfragments/3383.minor delete mode 100644 newsfragments/3386.minor delete mode 100644 newsfragments/3387.minor delete mode 100644 newsfragments/3388.minor delete mode 100644 newsfragments/3389.minor delete mode 100644 newsfragments/3391.minor delete mode 100644 newsfragments/3392.minor delete mode 100644 newsfragments/3393.minor delete mode 100644 newsfragments/3394.minor delete mode 100644 newsfragments/3395.minor delete mode 100644 newsfragments/3396.minor delete mode 100644 newsfragments/3397.minor delete mode 100644 newsfragments/3398.minor delete mode 100644 newsfragments/3401.minor delete mode 100644 newsfragments/3403.minor delete mode 100644 newsfragments/3406.minor delete mode 100644 newsfragments/3408.minor delete mode 100644 newsfragments/3409.minor delete mode 100644 newsfragments/3411.minor delete mode 100644 newsfragments/3415.minor delete mode 100644 newsfragments/3416.minor delete mode 100644 newsfragments/3417.minor delete mode 100644 newsfragments/3421.minor delete mode 100644 newsfragments/3422.minor delete mode 100644 newsfragments/3423.minor delete mode 100644 newsfragments/3424.minor delete mode 100644 newsfragments/3425.minor delete mode 100644 newsfragments/3426.minor delete mode 100644 newsfragments/3427.minor delete mode 100644 newsfragments/3429.minor delete mode 100644 newsfragments/3430.minor delete mode 100644 newsfragments/3431.minor delete mode 100644 newsfragments/3436.minor delete mode 100644 newsfragments/3437.minor delete mode 100644 newsfragments/3438.minor delete mode 100644 newsfragments/3439.minor delete mode 100644 newsfragments/3440.minor delete mode 100644 newsfragments/3442.minor delete mode 100644 newsfragments/3443.minor delete mode 100644 newsfragments/3446.minor delete mode 100644 newsfragments/3448.minor delete mode 100644 newsfragments/3449.minor delete mode 100644 newsfragments/3450.minor delete mode 100644 newsfragments/3451.minor delete mode 100644 newsfragments/3452.minor delete mode 100644 newsfragments/3453.minor delete mode 100644 newsfragments/3455.minor delete mode 100644 newsfragments/3456.minor delete mode 100644 newsfragments/3458.minor delete mode 100644 newsfragments/3462.minor delete mode 100644 newsfragments/3463.minor delete mode 100644 newsfragments/3464.minor diff --git a/NEWS.rst b/NEWS.rst index 2ca67a1f6..de6acb567 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,6 +5,52 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line +Release 1.14.0.post1530 (2020-10-13) +'''''''''''''''''''''''''''''''''''' + +Features +-------- + +- PyPy is now a supported platform. (`#1792 `_) +- allmydata.testing.web, a new module, now offers a supported Python API for testing Tahoe-LAFS web API clients. (`#3317 `_) + + +Bug Fixes +--------- + +- Make directory page links work. (`#3312 `_) +- Use last known revision of Chutney that is known to work with Python 2 for Tor integration tests. (`#3348 `_) +- Mutable files now use RSA exponent 65537 (`#3349 `_) + + +Dependency/Installation Changes +------------------------------- + +- Tahoe-LAFS now supports CentOS 8 and no longer supports CentOS 7. (`#3296 `_) +- Tahoe-LAFS now supports Ubuntu 20.04. (`#3328 `_) + + +Removed Features +---------------- + +- The Magic Folder frontend has been split out into a stand-alone project. The functionality is no longer part of Tahoe-LAFS itself. Learn more at . (`#3284 `_) +- Slackware 14.2 is no longer a Tahoe-LAFS supported platform. (`#3323 `_) + + +Other Changes +------------- + +- The Tahoe-LAFS project has adopted a formal code of conduct. (`#2755 `_) +- (`#3263 `_, `#3324 `_) +- The "coverage" tox environment has been replaced by the "py27-coverage" and "py36-coverage" environments. (`#3355 `_) + + +Misc/Other +---------- + +- `#3247 `_, `#3254 `_, `#3277 `_, `#3278 `_, `#3287 `_, `#3288 `_, `#3289 `_, `#3290 `_, `#3291 `_, `#3292 `_, `#3293 `_, `#3294 `_, `#3297 `_, `#3298 `_, `#3299 `_, `#3300 `_, `#3302 `_, `#3303 `_, `#3304 `_, `#3305 `_, `#3306 `_, `#3308 `_, `#3309 `_, `#3313 `_, `#3315 `_, `#3316 `_, `#3320 `_, `#3325 `_, `#3326 `_, `#3329 `_, `#3330 `_, `#3331 `_, `#3332 `_, `#3333 `_, `#3334 `_, `#3335 `_, `#3336 `_, `#3338 `_, `#3339 `_, `#3340 `_, `#3341 `_, `#3342 `_, `#3343 `_, `#3344 `_, `#3346 `_, `#3351 `_, `#3353 `_, `#3354 `_, `#3356 `_, `#3357 `_, `#3358 `_, `#3359 `_, `#3361 `_, `#3364 `_, `#3365 `_, `#3366 `_, `#3367 `_, `#3368 `_, `#3370 `_, `#3372 `_, `#3373 `_, `#3374 `_, `#3375 `_, `#3376 `_, `#3377 `_, `#3378 `_, `#3380 `_, `#3381 `_, `#3382 `_, `#3383 `_, `#3386 `_, `#3387 `_, `#3388 `_, `#3389 `_, `#3391 `_, `#3392 `_, `#3393 `_, `#3394 `_, `#3395 `_, `#3396 `_, `#3397 `_, `#3398 `_, `#3401 `_, `#3403 `_, `#3406 `_, `#3408 `_, `#3409 `_, `#3411 `_, `#3415 `_, `#3416 `_, `#3417 `_, `#3421 `_, `#3422 `_, `#3423 `_, `#3424 `_, `#3425 `_, `#3426 `_, `#3427 `_, `#3429 `_, `#3430 `_, `#3431 `_, `#3436 `_, `#3437 `_, `#3438 `_, `#3439 `_, `#3440 `_, `#3442 `_, `#3443 `_, `#3446 `_, `#3448 `_, `#3449 `_, `#3450 `_, `#3451 `_, `#3452 `_, `#3453 `_, `#3455 `_, `#3456 `_, `#3458 `_, `#3462 `_, `#3463 `_, `#3464 `_ + + Release 1.14.0 (2020-03-11) ''''''''''''''''''''''''''' diff --git a/newsfragments/1792.feature b/newsfragments/1792.feature deleted file mode 100644 index b2b839664..000000000 --- a/newsfragments/1792.feature +++ /dev/null @@ -1 +0,0 @@ -PyPy is now a supported platform. \ No newline at end of file diff --git a/newsfragments/2755.other b/newsfragments/2755.other deleted file mode 100644 index e3e31465d..000000000 --- a/newsfragments/2755.other +++ /dev/null @@ -1 +0,0 @@ -The Tahoe-LAFS project has adopted a formal code of conduct. diff --git a/newsfragments/3247.minor b/newsfragments/3247.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3254.minor b/newsfragments/3254.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3263.other b/newsfragments/3263.other deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3277.minor b/newsfragments/3277.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3278.minor b/newsfragments/3278.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3284.removed b/newsfragments/3284.removed deleted file mode 100644 index 7e31d352a..000000000 --- a/newsfragments/3284.removed +++ /dev/null @@ -1 +0,0 @@ -The Magic Folder frontend has been split out into a stand-alone project. The functionality is no longer part of Tahoe-LAFS itself. Learn more at . diff --git a/newsfragments/3287.minor b/newsfragments/3287.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3288.minor b/newsfragments/3288.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3289.minor b/newsfragments/3289.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3290.minor b/newsfragments/3290.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3291.minor b/newsfragments/3291.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3292.minor b/newsfragments/3292.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3293.minor b/newsfragments/3293.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3294.minor b/newsfragments/3294.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3296.installation b/newsfragments/3296.installation deleted file mode 100644 index 78cf83f60..000000000 --- a/newsfragments/3296.installation +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS now supports CentOS 8 and no longer supports CentOS 7. \ No newline at end of file diff --git a/newsfragments/3297.minor b/newsfragments/3297.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3298.minor b/newsfragments/3298.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3299.minor b/newsfragments/3299.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3300.minor b/newsfragments/3300.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3302.minor b/newsfragments/3302.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3303.minor b/newsfragments/3303.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3304.minor b/newsfragments/3304.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3305.minor b/newsfragments/3305.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3306.minor b/newsfragments/3306.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3308.minor b/newsfragments/3308.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3309.minor b/newsfragments/3309.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3312.bugfix b/newsfragments/3312.bugfix deleted file mode 100644 index 9939fe1f0..000000000 --- a/newsfragments/3312.bugfix +++ /dev/null @@ -1 +0,0 @@ -Make directory page links work. diff --git a/newsfragments/3313.minor b/newsfragments/3313.minor deleted file mode 100644 index c4eecd956..000000000 --- a/newsfragments/3313.minor +++ /dev/null @@ -1 +0,0 @@ -Replace nevow with twisted.web in web.operations.OphandleTable diff --git a/newsfragments/3315.minor b/newsfragments/3315.minor deleted file mode 100644 index 0536c297a..000000000 --- a/newsfragments/3315.minor +++ /dev/null @@ -1 +0,0 @@ -Replace nevow with twisted.web in web.operations.ReloadMixin diff --git a/newsfragments/3316.minor b/newsfragments/3316.minor deleted file mode 100644 index 9457b486e..000000000 --- a/newsfragments/3316.minor +++ /dev/null @@ -1 +0,0 @@ -Port checker result pages' rendering from nevow to twisted web templates. diff --git a/newsfragments/3317.feature b/newsfragments/3317.feature deleted file mode 100644 index 2a7048397..000000000 --- a/newsfragments/3317.feature +++ /dev/null @@ -1 +0,0 @@ -allmydata.testing.web, a new module, now offers a supported Python API for testing Tahoe-LAFS web API clients. \ No newline at end of file diff --git a/newsfragments/3320.minor b/newsfragments/3320.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3323.removed b/newsfragments/3323.removed deleted file mode 100644 index 356b4b2af..000000000 --- a/newsfragments/3323.removed +++ /dev/null @@ -1 +0,0 @@ -Slackware 14.2 is no longer a Tahoe-LAFS supported platform. diff --git a/newsfragments/3324.other b/newsfragments/3324.other deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3325.minor b/newsfragments/3325.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3326.minor b/newsfragments/3326.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3328.installation b/newsfragments/3328.installation deleted file mode 100644 index 7b08ffdc4..000000000 --- a/newsfragments/3328.installation +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS now supports Ubuntu 20.04. \ No newline at end of file diff --git a/newsfragments/3329.minor b/newsfragments/3329.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3330.minor b/newsfragments/3330.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3331.minor b/newsfragments/3331.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3332.minor b/newsfragments/3332.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3333.minor b/newsfragments/3333.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3334.minor b/newsfragments/3334.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3335.minor b/newsfragments/3335.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3336.minor b/newsfragments/3336.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3338.minor b/newsfragments/3338.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3339.minor b/newsfragments/3339.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3340.minor b/newsfragments/3340.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3341.minor b/newsfragments/3341.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3342.minor b/newsfragments/3342.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3343.minor b/newsfragments/3343.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3344.minor b/newsfragments/3344.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3346.minor b/newsfragments/3346.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3348.bugfix b/newsfragments/3348.bugfix deleted file mode 100644 index e0f1f6c5e..000000000 --- a/newsfragments/3348.bugfix +++ /dev/null @@ -1 +0,0 @@ -Use last known revision of Chutney that is known to work with Python 2 for Tor integration tests. diff --git a/newsfragments/3349.bugfix b/newsfragments/3349.bugfix deleted file mode 100644 index 08f2d7314..000000000 --- a/newsfragments/3349.bugfix +++ /dev/null @@ -1 +0,0 @@ -Mutable files now use RSA exponent 65537 diff --git a/newsfragments/3351.minor b/newsfragments/3351.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3353.minor b/newsfragments/3353.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3354.minor b/newsfragments/3354.minor deleted file mode 100644 index 8b1378917..000000000 --- a/newsfragments/3354.minor +++ /dev/null @@ -1 +0,0 @@ - diff --git a/newsfragments/3355.other b/newsfragments/3355.other deleted file mode 100644 index 4e854e4dd..000000000 --- a/newsfragments/3355.other +++ /dev/null @@ -1 +0,0 @@ -The "coverage" tox environment has been replaced by the "py27-coverage" and "py36-coverage" environments. diff --git a/newsfragments/3356.minor b/newsfragments/3356.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3357.minor b/newsfragments/3357.minor deleted file mode 100644 index 8b1378917..000000000 --- a/newsfragments/3357.minor +++ /dev/null @@ -1 +0,0 @@ - diff --git a/newsfragments/3358.minor b/newsfragments/3358.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3359.minor b/newsfragments/3359.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3361.minor b/newsfragments/3361.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3364.minor b/newsfragments/3364.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3365.minor b/newsfragments/3365.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3366.minor b/newsfragments/3366.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3367.minor b/newsfragments/3367.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3368.minor b/newsfragments/3368.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3370.minor b/newsfragments/3370.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3372.minor b/newsfragments/3372.minor deleted file mode 100644 index 8b1378917..000000000 --- a/newsfragments/3372.minor +++ /dev/null @@ -1 +0,0 @@ - diff --git a/newsfragments/3373.minor b/newsfragments/3373.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3374.minor b/newsfragments/3374.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3375.minor b/newsfragments/3375.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3376.minor b/newsfragments/3376.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3377.minor b/newsfragments/3377.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3378.minor b/newsfragments/3378.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3380.minor b/newsfragments/3380.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3381.minor b/newsfragments/3381.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3382.minor b/newsfragments/3382.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3383.minor b/newsfragments/3383.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3386.minor b/newsfragments/3386.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3387.minor b/newsfragments/3387.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3388.minor b/newsfragments/3388.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3389.minor b/newsfragments/3389.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3391.minor b/newsfragments/3391.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3392.minor b/newsfragments/3392.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3393.minor b/newsfragments/3393.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3394.minor b/newsfragments/3394.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3395.minor b/newsfragments/3395.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3396.minor b/newsfragments/3396.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3397.minor b/newsfragments/3397.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3398.minor b/newsfragments/3398.minor deleted file mode 100644 index 477c141fd..000000000 --- a/newsfragments/3398.minor +++ /dev/null @@ -1 +0,0 @@ -Added pre-commit config to run flake8 checks on commit/push. diff --git a/newsfragments/3401.minor b/newsfragments/3401.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3403.minor b/newsfragments/3403.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3406.minor b/newsfragments/3406.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3408.minor b/newsfragments/3408.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3409.minor b/newsfragments/3409.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3411.minor b/newsfragments/3411.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3415.minor b/newsfragments/3415.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3416.minor b/newsfragments/3416.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3417.minor b/newsfragments/3417.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3421.minor b/newsfragments/3421.minor deleted file mode 100644 index d6f70f6d9..000000000 --- a/newsfragments/3421.minor +++ /dev/null @@ -1 +0,0 @@ -Various, minor development `./Makefile` cleanup and improvement. diff --git a/newsfragments/3422.minor b/newsfragments/3422.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3423.minor b/newsfragments/3423.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3424.minor b/newsfragments/3424.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3425.minor b/newsfragments/3425.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3426.minor b/newsfragments/3426.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3427.minor b/newsfragments/3427.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3429.minor b/newsfragments/3429.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3430.minor b/newsfragments/3430.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3431.minor b/newsfragments/3431.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3436.minor b/newsfragments/3436.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3437.minor b/newsfragments/3437.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3438.minor b/newsfragments/3438.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3439.minor b/newsfragments/3439.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3440.minor b/newsfragments/3440.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3442.minor b/newsfragments/3442.minor deleted file mode 100644 index d67c7fb62..000000000 --- a/newsfragments/3442.minor +++ /dev/null @@ -1 +0,0 @@ -Minor test runner improvements and docs. diff --git a/newsfragments/3443.minor b/newsfragments/3443.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3446.minor b/newsfragments/3446.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3448.minor b/newsfragments/3448.minor deleted file mode 100644 index 4a5f7243f..000000000 --- a/newsfragments/3448.minor +++ /dev/null @@ -1 +0,0 @@ -Convert modules that only reference `unicode` to use `str`. \ No newline at end of file diff --git a/newsfragments/3449.minor b/newsfragments/3449.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3450.minor b/newsfragments/3450.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3451.minor b/newsfragments/3451.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3452.minor b/newsfragments/3452.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3453.minor b/newsfragments/3453.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3455.minor b/newsfragments/3455.minor deleted file mode 100644 index d7af32b64..000000000 --- a/newsfragments/3455.minor +++ /dev/null @@ -1 +0,0 @@ -Begin porting the `node` module to Python 3. diff --git a/newsfragments/3456.minor b/newsfragments/3456.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3458.minor b/newsfragments/3458.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3462.minor b/newsfragments/3462.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3463.minor b/newsfragments/3463.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3464.minor b/newsfragments/3464.minor deleted file mode 100644 index bc79dee53..000000000 --- a/newsfragments/3464.minor +++ /dev/null @@ -1 +0,0 @@ -Cleanup comments that don't match the project convention. From 2e4d5990e0c5cb1313d939106322b6d1e487ff1e Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 13 Oct 2020 14:41:57 -0600 Subject: [PATCH 002/463] proper release name --- NEWS.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index de6acb567..5f164c825 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,8 +5,8 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line -Release 1.14.0.post1530 (2020-10-13) -'''''''''''''''''''''''''''''''''''' +Release 1.15.0 (2020-10-13) +''''''''''''''''''''''''''' Features -------- From b7789e1ac4b4dbb4ff42551060d49e6d60493e91 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 13 Oct 2020 14:54:17 -0600 Subject: [PATCH 003/463] fix relnotes.txt for 1.15.0 --- relnotes.txt | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/relnotes.txt b/relnotes.txt index a46996dfb..fb0acfd61 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -1,6 +1,6 @@ -ANNOUNCING Tahoe, the Least-Authority File Store, v1.14.0 +ANNOUNCING Tahoe, the Least-Authority File Store, v1.15.0 -The Tahoe-LAFS team is pleased to announce version 1.14.0 of +The Tahoe-LAFS team is pleased to announce version 1.15.0 of Tahoe-LAFS, an extremely reliable decentralized storage system. Get it with "pip install tahoe-lafs", or download a tarball here: @@ -15,21 +15,24 @@ unique security and fault-tolerance properties: https://tahoe-lafs.readthedocs.org/en/latest/about.html -The previous stable release of Tahoe-LAFS was v1.12.1, -released on January 18, 2017. +The previous stable release of Tahoe-LAFS was v1.14.0, released on +April 21, 2020. -The v1.14.0 release: makes several Magic Folder improvements (MacOS -support, better logs, fewer conflict cases); adds an Eliot streaming -logs endpoint; adds an extension point for storage customization; -makes a bunch of bug-fixes and cleanups. NixOS is a supported -platform; Fedora 29 is no longer a supported platform. Several early -parts of Python3 porting have landed. +In this release: RSA exponent is changed to 65537 for mutable files; +magic-folder has been split to a stand-alone project. A formal code of +conduct has been adopted. -DEPRECATED: ``tahoe start``, ``tahoe stop``, ``tahoe restart`` and -``tahoe daemonize`` are all deprecated in favour of using ``tahoe -run`` (along with a suitable process manager if desired). +Platform support has changed for this release. No longer supported +are: Slackware 14.2 and CentOS 7. Newly supported are: the PyPy +interpreter (on MacOS, Windows and Linux); CentOS 8; Ubuntu 20.04. -Please see ``NEWS.rst`` for a more complete list of changes. +Note that Python3 porting is underway but not yet complete in this +release. Developers may notice python3 as new targets for certain +tools. + +In addition, 121 other minor tickets have been completed since the +last release. Please see ``NEWS.rst`` for a more complete list of +changes. WHAT IS IT GOOD FOR? @@ -146,19 +149,19 @@ solely as a labor of love by volunteers. Thank you very much to the team of "hackers in the public interest" who make Tahoe-LAFS possible. -Brian Warner +meejah on behalf of the Tahoe-LAFS team -May 17, 2018 -San Francisco, California, USA +October 13, 2020 +Planet Earth -[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.14.0/NEWS.rst +[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.15.0/NEWS.rst [2] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/docs/known_issues.rst [3] https://tahoe-lafs.org/trac/tahoe-lafs/wiki/RelatedProjects -[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.14.0/COPYING.GPL -[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.14.0/COPYING.TGPPL.rst -[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.14.0/INSTALL.html +[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.15.0/COPYING.GPL +[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.15.0/COPYING.TGPPL.rst +[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.15.0/INSTALL.html [7] https://tahoe-lafs.org/cgi-bin/mailman/listinfo/tahoe-dev [8] https://tahoe-lafs.org/trac/tahoe-lafs/roadmap [9] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/CREDITS From cb29293ec57cf596f7c70292495418495482bed0 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 13 Oct 2020 15:01:01 -0600 Subject: [PATCH 004/463] update CREDITS --- CREDITS | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/CREDITS b/CREDITS index b07a0b3e2..788e247b4 100644 --- a/CREDITS +++ b/CREDITS @@ -201,3 +201,28 @@ N: meejah E: meejah@meejah.ca P: 0xC2602803128069A7, 9D5A 2BD5 688E CB88 9DEB CD3F C260 2803 1280 69A7 D: various bug-fixes and features + +N: Chad Whitacre +E: chad@zetaweb.com +D: Python3 porting + +N: Itamar Turner-Trauring +E: itamar@itamarst.org +D: Python3 porting + +N: Jason R. Coombs +E: jaraco@jaraco.com +D: Python3 porting + +N: Maciej Fijalkowski +E: fijall@gmail.com +D: Python3 porting + +N: Ross Patterson +E: me@rpatterson.net +D: Python3 porting + +N: Sajith Sasidharan +E: sajith@hcoop.net +E: sajith@nonzen.in +D: Python3 porting From 107909997102d502551b808a8c8a6e210c8bc136 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 13 Oct 2020 15:03:53 -0600 Subject: [PATCH 005/463] 1.14.0 -> 1.15.0 --- docs/INSTALL.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/INSTALL.rst b/docs/INSTALL.rst index ab9b5a743..1f021d576 100644 --- a/docs/INSTALL.rst +++ b/docs/INSTALL.rst @@ -163,7 +163,7 @@ from PyPI with ``venv/bin/pip install tahoe-lafs``. After installation, run Successfully installed ... % venv/bin/tahoe --version - tahoe-lafs: 1.14.0 + tahoe-lafs: 1.15.0 foolscap: ... % @@ -183,14 +183,14 @@ You can also install directly from the source tarball URL:: New python executable in ~/venv/bin/python2.7 Installing setuptools, pip, wheel...done. - % venv/bin/pip install https://tahoe-lafs.org/downloads/tahoe-lafs-1.14.0.tar.bz2 - Collecting https://tahoe-lafs.org/downloads/tahoe-lafs-1.14.0.tar.bz2 + % venv/bin/pip install https://tahoe-lafs.org/downloads/tahoe-lafs-1.15.0.tar.bz2 + Collecting https://tahoe-lafs.org/downloads/tahoe-lafs-1.15.0.tar.bz2 ... Installing collected packages: ... Successfully installed ... % venv/bin/tahoe --version - tahoe-lafs: 1.14.0 + tahoe-lafs: 1.15.0 ... Extras @@ -224,7 +224,7 @@ the additional libraries needed to run the unit tests:: Successfully installed ... % venv/bin/tahoe --version - tahoe-lafs: 1.14.0.post34.dev0 + tahoe-lafs: 1.15.0 ... This way, you won't have to re-run the ``pip install`` step each time you @@ -273,7 +273,7 @@ result in a "all tests passed" mesage:: % tox GLOB sdist-make: ~/tahoe-lafs/setup.py py27 recreate: ~/tahoe-lafs/.tox/py27 - py27 inst: ~/tahoe-lafs/.tox/dist/tahoe-lafs-1.14.0.post8.dev0.zip + py27 inst: ~/tahoe-lafs/.tox/dist/tahoe-lafs-1.15.0.zip py27 runtests: commands[0] | tahoe --version py27 runtests: commands[1] | trial --rterrors allmydata allmydata.test.test_auth From 21b0eefed998d9007d07f48ebdc3dadd03ae183d Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 13 Oct 2020 16:38:25 -0600 Subject: [PATCH 006/463] newsfragment --- newsfragments/3469.minor | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3469.minor diff --git a/newsfragments/3469.minor b/newsfragments/3469.minor new file mode 100644 index 000000000..af22f6e10 --- /dev/null +++ b/newsfragments/3469.minor @@ -0,0 +1 @@ +1.15.0 release From b6bd58b4d0099d5be1211349e60080b25c011765 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 13 Oct 2020 16:58:16 -0600 Subject: [PATCH 007/463] CREDITS details --- CREDITS | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CREDITS b/CREDITS index 788e247b4..af8da934e 100644 --- a/CREDITS +++ b/CREDITS @@ -207,7 +207,7 @@ E: chad@zetaweb.com D: Python3 porting N: Itamar Turner-Trauring -E: itamar@itamarst.org +E: itamar@pythonspeed.com D: Python3 porting N: Jason R. Coombs @@ -226,3 +226,7 @@ N: Sajith Sasidharan E: sajith@hcoop.net E: sajith@nonzen.in D: Python3 porting + +N: Pete Fein +E: pete@snake.dev +D: Python3 porting From b05ae87a28dda5a9c279dbbcb788a9f979fe3eef Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 14 Oct 2020 19:14:05 -0600 Subject: [PATCH 008/463] non-relevant email --- CREDITS | 1 - 1 file changed, 1 deletion(-) diff --git a/CREDITS b/CREDITS index af8da934e..dd247757e 100644 --- a/CREDITS +++ b/CREDITS @@ -224,7 +224,6 @@ D: Python3 porting N: Sajith Sasidharan E: sajith@hcoop.net -E: sajith@nonzen.in D: Python3 porting N: Pete Fein From d28e172b5fc9e6327bc1bb8e2e6efe6d177aab85 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 9 Mar 2021 15:02:32 -0500 Subject: [PATCH 009/463] Update notes about reading/writing docs --- docs/README.txt | 39 +++++++++++++++++++++++++++++++++------ newsfragments/3630.minor | 0 2 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 newsfragments/3630.minor diff --git a/docs/README.txt b/docs/README.txt index 87c9583d9..063d2d772 100644 --- a/docs/README.txt +++ b/docs/README.txt @@ -1,7 +1,34 @@ +If you are reading Tahoe-LAFS documentation +------------------------------------------- -Note: http://tahoe-lafs.readthedocs.io/en/latest/ is the preferred place to -read this documentation (GitHub doesn't render cross-document links or -images). If you're reading this on https://github.com/tahoe-lafs/tahoe-lafs , -or from a checked-out source tree, then either run `tox -e docs` and open -_build/html/index.html in your browser, or view the pre-rendered trunk copy -at http://tahoe-lafs.readthedocs.io/en/latest/ +If you are reading Tahoe-LAFS documentation at a code hosting site or +from a checked-out source tree, the preferred place to view the docs +is http://tahoe-lafs.readthedocs.io/en/latest/. Code-hosting sites do +not render cross-document links or images correctly. + + +If you are writing Tahoe-LAFS documentation +------------------------------------------- + +To edit Tahoe-LAFS docs, you will need a checked-out source tree. You +can edit the `.rst` files in this directory using a text editor, and +then generate HTML output using Sphinx, a program that can produce its +output in HTML and other formats. + +The files with `.rst` extension use reStructuredText markup format, +which is the format Sphinx natively handles. To learn more about +Sphinx, and for a friendly primer on reStructuredText, please see +Sphinx project's documentation, available at: + +https://www.sphinx-doc.org/ + +If you have tox installed, you can run `tox -e docs` and then open the +resulting docs/_build/html/index.html in your web browser. + +If you have Sphinx and Make installed, you can also run `make html` +within docs directory. You may also need to install some additional +Python packages, such as setuptools and recommonmark. + +Note that Sphinx can also process comments in Python source code to +generate API documentation. Tahoe-LAFS currently does not use Sphinx +for this purpose. diff --git a/newsfragments/3630.minor b/newsfragments/3630.minor new file mode 100644 index 000000000..e69de29bb From 61fc96181ee0a154eba1906ddefd033eefd3ff63 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 9 Mar 2021 16:57:20 -0500 Subject: [PATCH 010/463] Build docs on CI --- .circleci/config.yml | 12 ++++++++++++ newsfragments/3632.minor | 0 2 files changed, 12 insertions(+) create mode 100644 newsfragments/3632.minor diff --git a/.circleci/config.yml b/.circleci/config.yml index b00bcdcec..2daf3b93f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -458,6 +458,18 @@ jobs: command: | /tmp/venv/bin/tox -e typechecks + docs: + docker: + - <<: *DOCKERHUB_AUTH + image: "tahoelafsci/ubuntu:18.04-py3" + + steps: + - "checkout" + - run: + name: "Build documentation" + command: | + /tmp/venv/bin/tox -e docs + build-image: &BUILD_IMAGE # This is a template for a job to build a Docker image that has as much of # the setup as we can manage already done and baked in. This cuts down on diff --git a/newsfragments/3632.minor b/newsfragments/3632.minor new file mode 100644 index 000000000..e69de29bb From 70291cd468b2ee5de6dab574508fdee3aa4d5d1a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 19 Mar 2021 15:55:19 -0400 Subject: [PATCH 011/463] Add "docs" job to CircleCI workflow --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2daf3b93f..1fb7558de 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,6 +89,9 @@ workflows: - "typechecks": <<: *DOCKERHUB_CONTEXT + - "docs": + <<: *DOCKERHUB_CONTEXT + images: # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. From af539a6f5c78f9f372f445712c1eb8280ae4fa50 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 22 Mar 2021 13:44:44 -0400 Subject: [PATCH 012/463] news fragment --- newsfragments/3037.other | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3037.other diff --git a/newsfragments/3037.other b/newsfragments/3037.other new file mode 100644 index 000000000..947dc8f60 --- /dev/null +++ b/newsfragments/3037.other @@ -0,0 +1 @@ +The "Great Black Swamp" proposed specification has been expanded to include two lease management APIs. \ No newline at end of file From 6520d3a505daf6ac6cd7c0eddc525c1f424e10db Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 22 Mar 2021 13:44:48 -0400 Subject: [PATCH 013/463] add the APIs --- docs/proposed/http-storage-node-protocol.rst | 57 ++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 1bdc774de..36c718c56 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -257,6 +257,62 @@ For example:: "application-version": "1.13.0" } +``PUT /v1/lease/:storage_index`` +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +Create a new lease that applies to all shares for the given storage index. +The details of the lease are encoded in the request body. +For example:: + + {"renew-secret": "abcd", "cancel-secret": "efgh"} + +If there are no shares for the given ``storage_index`` +then do nothing and return ``NO CONTENT``. + +If the ``renew-secret`` value matches an existing lease +then that lease will be renewed instead. + +The lease expires after 31 days. + +Discussion +`````````` + +Several behaviors here are blindly copied from the Foolscap-based storage server protocol. + +* There is a cancel secret but there is no API to use it to cancel a lease. +* The lease period is hard-coded at 31 days. +* There is no way to differentiate between success and an unknown **storage index**. +* There are separate **add** and **renew** lease APIs. + +These are not necessarily ideal behaviors +but they are adopted to avoid any *semantic* changes between the Foolscap- and HTTP-based protocols. +It is expected that some or all of these behaviors may change in a future revision of the HTTP-based protocol. + +``POST /v1/lease/:storage_index`` +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +Renew an existing lease for all shares for the given storage index. +The details of the lease are encoded in the request body. +For example:: + + {"renew-secret": "abcd"} + +If there are no shares for the given ``storage_index`` +then ``NOT FOUND`` is returned. + +If there is no lease with a matching ``renew-secret`` value on the given storage index +then ``NOT FOUND`` is returned. +In this case, +if the storage index refers to mutable data +then the response also includes a list of nodeids where the lease can be renewed. +For example:: + + {"nodeids": ["aaa...", "bbb..."]} + +Othewise, +the matching lease's expiration time is changed to be 31 days from the time of this operation +and ``NO CONTENT`` is returned. + Immutable --------- @@ -268,6 +324,7 @@ Writing Initialize an immutable storage index with some buckets. The buckets may have share data written to them once. +A lease is also created for the shares. Details of the buckets to create are encoded in the request body. For example:: From 9d64c881998b0cdb6962fd8abe18b86d73e31c39 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 22 Mar 2021 20:28:18 -0400 Subject: [PATCH 014/463] news fragment --- newsfragments/3645.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3645.minor diff --git a/newsfragments/3645.minor b/newsfragments/3645.minor new file mode 100644 index 000000000..e69de29bb From 31c838d5848eec9a65e63a63bb01b9a24446cb42 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 22 Mar 2021 20:28:24 -0400 Subject: [PATCH 015/463] the expanded motivation --- docs/proposed/http-storage-node-protocol.rst | 48 ++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 36c718c56..17da870b1 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -13,6 +13,54 @@ Specifically, it should be possible to implement a Tahoe-LAFS storage server wit The Tahoe-LAFS client will also need to change but it is not expected that it will be noticably simplified by this change (though this may be the first step towards simplifying it). +Motivation +---------- + +Foolscap +~~~~~~~~ + +Foolscap is a remote method invocation protocol with several distinctive features. +At its core it allows separate processes to refer each other's objects and methods using a capability-based model. +This allows for extremely fine-grained access control in a system that remains highly securable without becoming overwhelmingly complicated. +Supporting this is a flexible and extensible serialization system which allows data to be exchanged between processes in carefully controlled ways. + +Tahoe avails itself of only a small portion of these features. +A Tahoe storage server typically only exposes one object with a fixed set of methods to clients. +A Tahoe introducer node does roughly the same. +Tahoe exchanges simple data structures that have many common, standard serialized representations. + +In exchange for this slight use of Foolscap's sophisticated mechanisms, +Tahoe pays a substantial price: + +* Foolscap is implemented only for Python. + Tahoe is thus limited to being implemented on in Python. +* There is only one Python implementation of Foolscap. + The implementation is therefore the de facto standard and understanding of the protocol often relies on understanding that implementation. +* The Foolscap developer community is very small. + The implementation therefore advances very little and some non-trivial part of the maintenance cost falls on the Tahoe project. +* The extensible serialization system imposes substantial overhead for the simple data structures Tahoe exchanges. + Tahoe therefore presents a more sluggish experience to users and taxes servers more greatly than is necessary. + +HTTP +~~~~ + +HTTP is a request/response protocol that has become the lingua franca of the internet. +Combined with the principles of Representational state transfer (REST) it is widely employed to create, update, and delete data in collections on the internet. +HTTP itself provides only modest functionality in comparison to Foolscap. +However its simplicity and widespread use have led to a diverse and almost overwhelming ecosystem of libraries, frameworks, toolkits, and so on. + +By adopting HTTP in place of Foolscap Tahoe can realize the following concrete benefits: + +* Practically every language or runtime has an HTTP protocol implementation (or a dozen of them) available. + This change paves the way for new Tahoe implementations using tools better suited for certain situations + (mobile client implementations, high-performance server implementations, easily distributed desktop clients, etc). +* The simplicity of and vast quantity of resources about HTTP make it a very easy protocol to learn and use. + This change reduces the barrier to entry for developers to contribute improvements to Tahoe's network interactions. +* For any given language there is very likely an HTTP implementation with a large and active developer community. + Tahoe can therefore benefit from the large effort being put into making better libraries for using HTTP. +* One of the core features of HTTP is the mundane transfer of bulk data and implementions are often capable of doing this with extreme efficiency. + The alignment of this core feature with a core activity of Tahoe of transferring bulk data means that a substantial barrier to improved Tahoe runtime performance will be eliminated. + Requirements ------------ From 8b1aa50b6967f32aaa6f8e19b5db129f7b1349b6 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 23 Mar 2021 10:17:20 -0400 Subject: [PATCH 016/463] Use one space after period --- docs/README.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/README.txt b/docs/README.txt index 063d2d772..571e28bdd 100644 --- a/docs/README.txt +++ b/docs/README.txt @@ -3,20 +3,20 @@ If you are reading Tahoe-LAFS documentation If you are reading Tahoe-LAFS documentation at a code hosting site or from a checked-out source tree, the preferred place to view the docs -is http://tahoe-lafs.readthedocs.io/en/latest/. Code-hosting sites do +is http://tahoe-lafs.readthedocs.io/en/latest/. Code-hosting sites do not render cross-document links or images correctly. If you are writing Tahoe-LAFS documentation ------------------------------------------- -To edit Tahoe-LAFS docs, you will need a checked-out source tree. You +To edit Tahoe-LAFS docs, you will need a checked-out source tree. You can edit the `.rst` files in this directory using a text editor, and then generate HTML output using Sphinx, a program that can produce its output in HTML and other formats. The files with `.rst` extension use reStructuredText markup format, -which is the format Sphinx natively handles. To learn more about +which is the format Sphinx natively handles. To learn more about Sphinx, and for a friendly primer on reStructuredText, please see Sphinx project's documentation, available at: @@ -26,9 +26,9 @@ If you have tox installed, you can run `tox -e docs` and then open the resulting docs/_build/html/index.html in your web browser. If you have Sphinx and Make installed, you can also run `make html` -within docs directory. You may also need to install some additional +within docs directory. You may also need to install some additional Python packages, such as setuptools and recommonmark. Note that Sphinx can also process comments in Python source code to -generate API documentation. Tahoe-LAFS currently does not use Sphinx +generate API documentation. Tahoe-LAFS currently does not use Sphinx for this purpose. From c71b72758e8b368981d75330b9ff5324f6c776f5 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 23 Mar 2021 10:18:25 -0400 Subject: [PATCH 017/463] "The files with ..." to "Files with ..." --- docs/README.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/README.txt b/docs/README.txt index 571e28bdd..868e4cd2b 100644 --- a/docs/README.txt +++ b/docs/README.txt @@ -15,10 +15,10 @@ can edit the `.rst` files in this directory using a text editor, and then generate HTML output using Sphinx, a program that can produce its output in HTML and other formats. -The files with `.rst` extension use reStructuredText markup format, -which is the format Sphinx natively handles. To learn more about -Sphinx, and for a friendly primer on reStructuredText, please see -Sphinx project's documentation, available at: +Files with `.rst` extension use reStructuredText markup format, which +is the format Sphinx natively handles. To learn more about Sphinx, and +for a friendly primer on reStructuredText, please see Sphinx project's +documentation, available at: https://www.sphinx-doc.org/ From 72ffc6211cdd06ee6299eb15ce2629411166d35c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 23 Mar 2021 10:34:46 -0400 Subject: [PATCH 018/463] tox to `tox` --- docs/README.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/README.txt b/docs/README.txt index 868e4cd2b..d710470f7 100644 --- a/docs/README.txt +++ b/docs/README.txt @@ -22,8 +22,8 @@ documentation, available at: https://www.sphinx-doc.org/ -If you have tox installed, you can run `tox -e docs` and then open the -resulting docs/_build/html/index.html in your web browser. +If you have `tox` installed, you can run `tox -e docs` and then open +the resulting docs/_build/html/index.html in your web browser. If you have Sphinx and Make installed, you can also run `make html` within docs directory. You may also need to install some additional From db02800232d2394154837384a2d049d774c28b1b Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 23 Mar 2021 10:35:30 -0400 Subject: [PATCH 019/463] `docs directory` to `the docs directory` --- docs/README.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/README.txt b/docs/README.txt index d710470f7..987014518 100644 --- a/docs/README.txt +++ b/docs/README.txt @@ -26,8 +26,8 @@ If you have `tox` installed, you can run `tox -e docs` and then open the resulting docs/_build/html/index.html in your web browser. If you have Sphinx and Make installed, you can also run `make html` -within docs directory. You may also need to install some additional -Python packages, such as setuptools and recommonmark. +within the docs directory. You may also need to install some +additional Python packages, such as setuptools and recommonmark. Note that Sphinx can also process comments in Python source code to generate API documentation. Tahoe-LAFS currently does not use Sphinx From 517cfad4df33d667cdef411026140e7a041eaffc Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 23 Mar 2021 10:36:06 -0400 Subject: [PATCH 020/463] `setuptools` and `recommonmark` --- docs/README.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README.txt b/docs/README.txt index 987014518..25460bbaa 100644 --- a/docs/README.txt +++ b/docs/README.txt @@ -27,7 +27,7 @@ the resulting docs/_build/html/index.html in your web browser. If you have Sphinx and Make installed, you can also run `make html` within the docs directory. You may also need to install some -additional Python packages, such as setuptools and recommonmark. +additional Python packages, such as `setuptools` and `recommonmark`. Note that Sphinx can also process comments in Python source code to generate API documentation. Tahoe-LAFS currently does not use Sphinx From 4005d90024bac064a89fd1ad09802a302bd5c7d7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 Mar 2021 10:42:29 -0400 Subject: [PATCH 021/463] Ensure the fake matches the real Wormhole interface. --- src/allmydata/test/cli/test_invite.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index f356e18de..05bfed128 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -16,6 +16,8 @@ class _FakeWormhole(object): def __init__(self, outgoing_messages): self.messages = [] + for o in outgoing_messages: + assert isinstance(o, bytes) self._outgoing = outgoing_messages def get_code(self): @@ -26,15 +28,16 @@ class _FakeWormhole(object): def get_welcome(self): return defer.succeed( - json.dumps({ + { u"welcome": {}, - }) + } ) def allocate_code(self): return None def send_message(self, msg): + assert isinstance(msg, bytes) self.messages.append(msg) def get_message(self): From 5ac52da14c07800772cec480f1914e97d19e83f6 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 23 Mar 2021 10:45:56 -0400 Subject: [PATCH 022/463] `comments in Python source code` to `Python docstrings` --- docs/README.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/README.txt b/docs/README.txt index 25460bbaa..53f905e90 100644 --- a/docs/README.txt +++ b/docs/README.txt @@ -29,6 +29,6 @@ If you have Sphinx and Make installed, you can also run `make html` within the docs directory. You may also need to install some additional Python packages, such as `setuptools` and `recommonmark`. -Note that Sphinx can also process comments in Python source code to -generate API documentation. Tahoe-LAFS currently does not use Sphinx -for this purpose. +Note that Sphinx can also process Python docstrings to generate API +documentation. Tahoe-LAFS currently does not use Sphinx for this +purpose. From 5b39940d47239726a3ee3328503fc1c55fd2ff4f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 23 Mar 2021 10:52:51 -0400 Subject: [PATCH 023/463] Drop the note about running `make html` in docs --- docs/README.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/README.txt b/docs/README.txt index 53f905e90..b571a2077 100644 --- a/docs/README.txt +++ b/docs/README.txt @@ -25,10 +25,6 @@ https://www.sphinx-doc.org/ If you have `tox` installed, you can run `tox -e docs` and then open the resulting docs/_build/html/index.html in your web browser. -If you have Sphinx and Make installed, you can also run `make html` -within the docs directory. You may also need to install some -additional Python packages, such as `setuptools` and `recommonmark`. - Note that Sphinx can also process Python docstrings to generate API documentation. Tahoe-LAFS currently does not use Sphinx for this purpose. From d182ba82838c3dce6113d6c9b106981cff4ec1ce Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 Mar 2021 10:53:10 -0400 Subject: [PATCH 024/463] Utility to dump JSON to bytes. --- src/allmydata/test/test_util.py | 8 +++++++- src/allmydata/util/jsonbytes.py | 10 +++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 5f5db82bd..a14adb787 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -507,7 +507,6 @@ class JSONBytes(unittest.TestCase): self.assertEqual(json.loads(encoded), expected) self.assertEqual(jsonbytes.loads(encoded), expected) - def test_encode_unicode(self): """BytesJSONEncoder encodes Unicode string as usual.""" expected = { @@ -515,3 +514,10 @@ class JSONBytes(unittest.TestCase): } encoded = jsonbytes.dumps(expected) self.assertEqual(json.loads(encoded), expected) + + def test_dumps_bytes(self): + """jsonbytes.dumps_bytes always returns bytes.""" + x = {u"def\N{SNOWMAN}\uFF00": 123} + encoded = jsonbytes.dumps_bytes(x) + self.assertIsInstance(encoded, bytes) + self.assertEqual(json.loads(encoded, encoding="utf-8"), x) diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index 935187d29..c46a932d0 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -9,7 +9,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from future.utils import PY2 +from future.utils import PY2, PY3 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 @@ -51,6 +51,14 @@ def dumps(obj, *args, **kwargs): return json.dumps(obj, cls=BytesJSONEncoder, *args, **kwargs) +def dumps_bytes(obj, *args, **kwargs): + """Encode to JSON, then encode as bytes.""" + result = dumps(obj, *args, **kwargs) + if PY3: + result = result.encode("utf-8") + return result + + # To make this module drop-in compatible with json module: loads = json.loads From e140dc06ea2d5c2dc77f2001acaff4535513bf61 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 Mar 2021 10:53:53 -0400 Subject: [PATCH 025/463] Tests pass on Python 3. --- src/allmydata/scripts/create_node.py | 7 +++---- src/allmydata/scripts/tahoe_invite.py | 7 +++---- src/allmydata/test/cli/test_invite.py | 6 ++++++ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index 3afbeeb0d..9bcfc2fe0 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -5,13 +5,12 @@ from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals -from future.utils import PY2 +from future.utils import PY2, PY3 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import io import os -import json try: from allmydata.scripts.types_ import SubCommands @@ -32,7 +31,7 @@ from allmydata.scripts.common import ( from allmydata.scripts.default_nodedir import _default_nodedir from allmydata.util.assertutil import precondition from allmydata.util.encodingutil import listdir_unicode, argv_to_unicode, quote_local_unicode_path, get_io_encoding -from allmydata.util import fileutil, i2p_provider, iputil, tor_provider +from allmydata.util import fileutil, i2p_provider, iputil, tor_provider, jsonbytes as json from wormhole import wormhole @@ -389,7 +388,7 @@ def _get_config_via_wormhole(config): "client-v1": {}, } } - wh.send_message(json.dumps(intro)) + wh.send_message(json.dumps_bytes(intro)) server_intro = yield wh.get_message() server_intro = json.loads(server_intro) diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py index 884536ec2..403481029 100644 --- a/src/allmydata/scripts/tahoe_invite.py +++ b/src/allmydata/scripts/tahoe_invite.py @@ -1,7 +1,5 @@ from __future__ import print_function -import json - try: from allmydata.scripts.types_ import SubCommands except ImportError: @@ -13,6 +11,7 @@ from twisted.internet import defer, reactor from wormhole import wormhole from allmydata.util.encodingutil import argv_to_abspath +from allmydata.util import jsonbytes as json from allmydata.scripts.common import get_default_nodedir, get_introducer_furl from allmydata.node import read_config @@ -54,7 +53,7 @@ def _send_config_via_wormhole(options, config): code = yield wh.get_code() print("Invite Code for client: {}".format(code), file=out) - wh.send_message(json.dumps({ + wh.send_message(json.dumps_bytes({ u"abilities": { u"server-v1": {}, } @@ -71,7 +70,7 @@ def _send_config_via_wormhole(options, config): defer.returnValue(1) print(" transmitting configuration", file=out) - wh.send_message(json.dumps(config)) + wh.send_message(json.dumps_bytes(config)) yield wh.close() diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 05bfed128..452acbc9c 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -1,3 +1,5 @@ +from past.builtins import unicode + import os import mock import json @@ -48,6 +50,10 @@ class _FakeWormhole(object): def _create_fake_wormhole(outgoing_messages): + outgoing_messages = [ + m.encode("utf-8") if isinstance(m, unicode) else m + for m in outgoing_messages + ] return _FakeWormhole(outgoing_messages) From 15d6ab610da3f9fb62d491fca5c5dd0e50e6e7ad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 Mar 2021 10:55:14 -0400 Subject: [PATCH 026/463] Port to Python 3. --- src/allmydata/test/cli/test_invite.py | 14 ++++++++++++-- src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 452acbc9c..36021af13 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -1,4 +1,14 @@ -from past.builtins import unicode +""" +Ported to Pythn 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os import mock @@ -51,7 +61,7 @@ class _FakeWormhole(object): def _create_fake_wormhole(outgoing_messages): outgoing_messages = [ - m.encode("utf-8") if isinstance(m, unicode) else m + m.encode("utf-8") if isinstance(m, str) else m for m in outgoing_messages ] return _FakeWormhole(outgoing_messages) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 8cf6a61b3..5a29020b8 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -145,6 +145,7 @@ PORTED_MODULES = [ PORTED_TEST_MODULES = [ "allmydata.test.cli.test_alias", "allmydata.test.cli.test_create", + "allmydata.test.cli.test_invite", "allmydata.test.cli.test_status", "allmydata.test.mutable.test_checker", From 7e889eb60a41382b307dce023ca6076855ceabb9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 Mar 2021 11:02:14 -0400 Subject: [PATCH 027/463] news file. --- newsfragments/3647.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3647.minor diff --git a/newsfragments/3647.minor b/newsfragments/3647.minor new file mode 100644 index 000000000..e69de29bb From 4b0aa412567a1d47c4d0be6c46c242a558036109 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 Mar 2021 11:04:00 -0400 Subject: [PATCH 028/463] Fix flake. --- src/allmydata/scripts/create_node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index 9bcfc2fe0..0258a0074 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -5,7 +5,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals -from future.utils import PY2, PY3 +from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 From 36cc66525d388b4fb4bd652a7186dc85f5746bbc Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 1 Mar 2021 13:47:39 -0500 Subject: [PATCH 029/463] Drop txi2p for Python 3 entirely PEP-508 "URL lookups" are unsupported in packages released on PyPI --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 4151545f7..d021b4512 100644 --- a/setup.py +++ b/setup.py @@ -151,9 +151,7 @@ tor_requires = [ i2p_requires = [ # txi2p has Python 3 support, but it's unreleased: https://github.com/str4d/txi2p/issues/10. - # URL lookups are in PEP-508 (via https://stackoverflow.com/a/54794506). - # Also see the comment in tor_requires. - "txi2p @ git+https://github.com/str4d/txi2p@0611b9a86172cb70d2f5e415a88eee9f230590b3#egg=txi2p", + "txi2p; python_version < '3.0'", ] if len(sys.argv) > 1 and sys.argv[1] == '--fakedependency': From a888b0bde90a0a3731690a4618c34b5dbb9e1bea Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 1 Mar 2021 13:29:31 -0500 Subject: [PATCH 030/463] news fragment --- newsfragments/3608.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3608.minor diff --git a/newsfragments/3608.minor b/newsfragments/3608.minor new file mode 100644 index 000000000..e69de29bb From ea7bd9b880c1c97dc4ab0cd37ee635e47f5cf8bc Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 23 Mar 2021 11:12:32 -0600 Subject: [PATCH 031/463] update NEWS.txt for release --- NEWS.rst | 9 +++++++++ newsfragments/3469.minor | 1 - newsfragments/3608.minor | 0 3 files changed, 9 insertions(+), 1 deletion(-) delete mode 100644 newsfragments/3469.minor delete mode 100644 newsfragments/3608.minor diff --git a/NEWS.rst b/NEWS.rst index 5f164c825..20fc05b4a 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,6 +5,15 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line +Release 1.15.0.post2 (2021-03-23) +''''''''''''''''''''''''''''''''' + +Misc/Other +---------- + +- `#3469 `_, `#3608 `_ + + Release 1.15.0 (2020-10-13) ''''''''''''''''''''''''''' diff --git a/newsfragments/3469.minor b/newsfragments/3469.minor deleted file mode 100644 index af22f6e10..000000000 --- a/newsfragments/3469.minor +++ /dev/null @@ -1 +0,0 @@ -1.15.0 release diff --git a/newsfragments/3608.minor b/newsfragments/3608.minor deleted file mode 100644 index e69de29bb..000000000 From 36ae60a3a58e0c0e7355d2361f1f438651d0b2f7 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 23 Mar 2021 11:16:45 -0600 Subject: [PATCH 032/463] update release-notes --- NEWS.rst | 4 ++-- relnotes.txt | 31 ++++++++++++------------------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 20fc05b4a..88d231826 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,8 +5,8 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line -Release 1.15.0.post2 (2021-03-23) -''''''''''''''''''''''''''''''''' +Release 1.15.1 +'''''''''''''' Misc/Other ---------- diff --git a/relnotes.txt b/relnotes.txt index fb0acfd61..d5552e24b 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -1,6 +1,6 @@ -ANNOUNCING Tahoe, the Least-Authority File Store, v1.15.0 +ANNOUNCING Tahoe, the Least-Authority File Store, v1.15.1 -The Tahoe-LAFS team is pleased to announce version 1.15.0 of +The Tahoe-LAFS team is pleased to announce version 1.15.1 of Tahoe-LAFS, an extremely reliable decentralized storage system. Get it with "pip install tahoe-lafs", or download a tarball here: @@ -15,24 +15,17 @@ unique security and fault-tolerance properties: https://tahoe-lafs.readthedocs.org/en/latest/about.html -The previous stable release of Tahoe-LAFS was v1.14.0, released on -April 21, 2020. +The previous stable release of Tahoe-LAFS was v1.15.0, released on +January 19, 2021. -In this release: RSA exponent is changed to 65537 for mutable files; -magic-folder has been split to a stand-alone project. A formal code of -conduct has been adopted. - -Platform support has changed for this release. No longer supported -are: Slackware 14.2 and CentOS 7. Newly supported are: the PyPy -interpreter (on MacOS, Windows and Linux); CentOS 8; Ubuntu 20.04. +In this release: PyPI does not accept uploads of packages that use +PEP-508 version specifiers. Note that Python3 porting is underway but not yet complete in this release. Developers may notice python3 as new targets for certain tools. -In addition, 121 other minor tickets have been completed since the -last release. Please see ``NEWS.rst`` for a more complete list of -changes. +Please see ``NEWS.rst`` for a more complete list of changes. WHAT IS IT GOOD FOR? @@ -152,16 +145,16 @@ Tahoe-LAFS possible. meejah on behalf of the Tahoe-LAFS team -October 13, 2020 +March 23, 2021 Planet Earth -[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.15.0/NEWS.rst +[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.15.1/NEWS.rst [2] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/docs/known_issues.rst [3] https://tahoe-lafs.org/trac/tahoe-lafs/wiki/RelatedProjects -[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.15.0/COPYING.GPL -[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.15.0/COPYING.TGPPL.rst -[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.15.0/INSTALL.html +[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.15.1/COPYING.GPL +[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.15.1/COPYING.TGPPL.rst +[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.15.1/INSTALL.html [7] https://tahoe-lafs.org/cgi-bin/mailman/listinfo/tahoe-dev [8] https://tahoe-lafs.org/trac/tahoe-lafs/roadmap [9] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/CREDITS From 034f577ff06234bc2a008a62d3fd03a30a447844 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 23 Mar 2021 11:17:28 -0600 Subject: [PATCH 033/463] bump release version --- docs/INSTALL.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/INSTALL.rst b/docs/INSTALL.rst index 1f021d576..515b0a6cc 100644 --- a/docs/INSTALL.rst +++ b/docs/INSTALL.rst @@ -163,7 +163,7 @@ from PyPI with ``venv/bin/pip install tahoe-lafs``. After installation, run Successfully installed ... % venv/bin/tahoe --version - tahoe-lafs: 1.15.0 + tahoe-lafs: 1.15.1 foolscap: ... % @@ -183,14 +183,14 @@ You can also install directly from the source tarball URL:: New python executable in ~/venv/bin/python2.7 Installing setuptools, pip, wheel...done. - % venv/bin/pip install https://tahoe-lafs.org/downloads/tahoe-lafs-1.15.0.tar.bz2 - Collecting https://tahoe-lafs.org/downloads/tahoe-lafs-1.15.0.tar.bz2 + % venv/bin/pip install https://tahoe-lafs.org/downloads/tahoe-lafs-1.15.1.tar.bz2 + Collecting https://tahoe-lafs.org/downloads/tahoe-lafs-1.15.1.tar.bz2 ... Installing collected packages: ... Successfully installed ... % venv/bin/tahoe --version - tahoe-lafs: 1.15.0 + tahoe-lafs: 1.15.1 ... Extras @@ -224,7 +224,7 @@ the additional libraries needed to run the unit tests:: Successfully installed ... % venv/bin/tahoe --version - tahoe-lafs: 1.15.0 + tahoe-lafs: 1.15.1 ... This way, you won't have to re-run the ``pip install`` step each time you @@ -273,7 +273,7 @@ result in a "all tests passed" mesage:: % tox GLOB sdist-make: ~/tahoe-lafs/setup.py py27 recreate: ~/tahoe-lafs/.tox/py27 - py27 inst: ~/tahoe-lafs/.tox/dist/tahoe-lafs-1.15.0.zip + py27 inst: ~/tahoe-lafs/.tox/dist/tahoe-lafs-1.15.1.zip py27 runtests: commands[0] | tahoe --version py27 runtests: commands[1] | trial --rterrors allmydata allmydata.test.test_auth From 5854723a0538cf71412ca8b9b72f20a9f8179f22 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 23 Mar 2021 12:05:56 -0600 Subject: [PATCH 034/463] make CI happy --- newsfragments/3648.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3648.minor diff --git a/newsfragments/3648.minor b/newsfragments/3648.minor new file mode 100644 index 000000000..e69de29bb From 9a066d29d4cade860278e44e49e85249573e44ff Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 23 Mar 2021 15:25:00 -0600 Subject: [PATCH 035/463] remove LGTM --- .lgtm.yml | 22 ---------------------- newsfragments/3649.minor | 0 2 files changed, 22 deletions(-) delete mode 100644 .lgtm.yml create mode 100644 newsfragments/3649.minor diff --git a/.lgtm.yml b/.lgtm.yml deleted file mode 100644 index efc2479ca..000000000 --- a/.lgtm.yml +++ /dev/null @@ -1,22 +0,0 @@ -extraction: - python: - after_prepare: - - | - # https://discuss.lgtm.com/t/determination-of-python-requirements/974/4 - sed -i 's/\("pyOpenSSL\)/\# Dependency removed for lgtm (see .lgtm.yml): \1/g' src/allmydata/_auto_deps.py - -queries: - # This generates spurious errors for calls by interface because of the - # zope.interface choice to exclude self from method signatures. So, turn it - # off. - - exclude: "py/call/wrong-arguments" - - # The premise of this query is broken. The errors it produces are nonsense. - # There is no such thing as a "procedure" in Python and "None" is not - # meaningless. - - exclude: "py/procedure-return-value-used" - - # It is true that this query identifies things which are sometimes mistakes. - # However, it also identifies things which are entirely valid. Therefore, - # it produces noisy results. - - exclude: "py/implicit-string-concatenation-in-list" \ No newline at end of file diff --git a/newsfragments/3649.minor b/newsfragments/3649.minor new file mode 100644 index 000000000..e69de29bb From 20d9b84b0ed9dbdc132c44b1b18fc73ba4e6710f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 24 Mar 2021 13:14:58 -0400 Subject: [PATCH 036/463] news fragment --- newsfragments/3650.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3650.bugfix diff --git a/newsfragments/3650.bugfix b/newsfragments/3650.bugfix new file mode 100644 index 000000000..a747a1246 --- /dev/null +++ b/newsfragments/3650.bugfix @@ -0,0 +1 @@ +``tahoe invite`` will now read share encoding/placement configuration from a Tahoe client node configuration file if they are not given on the command line, instead of raising an unhandled exception. From 3963979fbd14f5bdf8041018c762ff85720e82ec Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 24 Mar 2021 13:23:19 -0400 Subject: [PATCH 037/463] Test and fix `tahoe invite` w/o share arguments --- src/allmydata/scripts/tahoe_invite.py | 8 +- src/allmydata/test/cli/test_invite.py | 113 +++++++++++++++++++------- 2 files changed, 89 insertions(+), 32 deletions(-) diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py index 884536ec2..5ef05e2ac 100644 --- a/src/allmydata/scripts/tahoe_invite.py +++ b/src/allmydata/scripts/tahoe_invite.py @@ -14,7 +14,7 @@ from wormhole import wormhole from allmydata.util.encodingutil import argv_to_abspath from allmydata.scripts.common import get_default_nodedir, get_introducer_furl -from allmydata.node import read_config +from allmydata.client import read_config class InviteOptions(usage.Options): @@ -94,9 +94,9 @@ def invite(options): nick = options['nick'] remote_config = { - "shares-needed": options["shares-needed"] or config.get('client', 'shares.needed'), - "shares-total": options["shares-total"] or config.get('client', 'shares.total'), - "shares-happy": options["shares-happy"] or config.get('client', 'shares.happy'), + "shares-needed": options["shares-needed"] or config.get_config('client', 'shares.needed'), + "shares-total": options["shares-total"] or config.get_config('client', 'shares.total'), + "shares-happy": options["shares-happy"] or config.get_config('client', 'shares.happy'), "nickname": nick, "introducer": introducer_furl, } diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index f356e18de..cc610d409 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -3,6 +3,11 @@ import mock import json from os.path import join +try: + from typing import Optional, Tuple +except ImportError: + pass + from twisted.trial import unittest from twisted.internet import defer from ..common_util import run_cli @@ -144,10 +149,17 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): intro_dir, ) - @defer.inlineCallbacks - def test_invite_success(self): + + def _invite_success(self, extra_args=(), tahoe_config=None): + # type: (Tuple[bytes], Optional[bytes]) -> defer.Deferred """ - successfully send an invite + Exercise an expected-success case of ``tahoe invite``. + + :param extra_args: Positional arguments to pass to ``tahoe invite`` + before the nickname. + + :param tahoe_config: If given, bytes to write to the node's + ``tahoe.cfg`` before running ``tahoe invite. """ intro_dir = os.path.join(self.basedir, "introducer") # we've never run the introducer, so it hasn't created @@ -155,6 +167,9 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): priv_dir = join(intro_dir, "private") with open(join(priv_dir, "introducer.furl"), "w") as f: f.write("pb://fooblam\n") + if tahoe_config is not None: + with open(join(intro_dir, "tahoe.cfg"), "w") as f: + f.write(tahoe_config) with mock.patch('allmydata.scripts.tahoe_invite.wormhole') as w: fake_wh = _create_fake_wormhole([ @@ -162,34 +177,76 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): ]) w.create = mock.Mock(return_value=fake_wh) - rc, out, err = yield run_cli( + def done(result): + rc, out, err = result + self.assertEqual(2, len(fake_wh.messages)) + self.assertEqual( + json.loads(fake_wh.messages[0]), + { + "abilities": + { + "server-v1": {} + }, + }, + ) + invite = json.loads(fake_wh.messages[1]) + self.assertEqual( + invite["nickname"], "foo", + ) + self.assertEqual( + invite["introducer"], "pb://fooblam", + ) + return invite + d = run_cli( "-d", intro_dir, "invite", - "--shares-needed", "1", - "--shares-happy", "1", - "--shares-total", "1", - "foo", - ) - self.assertEqual(2, len(fake_wh.messages)) - self.assertEqual( - json.loads(fake_wh.messages[0]), - { - "abilities": - { - "server-v1": {} - }, - }, - ) - self.assertEqual( - json.loads(fake_wh.messages[1]), - { - "shares-needed": "1", - "shares-total": "1", - "nickname": "foo", - "introducer": "pb://fooblam", - "shares-happy": "1", - }, + *(extra_args + ("foo",)) ) + d.addCallback(done) + return d + + @defer.inlineCallbacks + def test_invite_success(self): + """ + successfully send an invite + """ + invite = yield self._invite_success(( + "--shares-needed", "1", + "--shares-happy", "2", + "--shares-total", "3", + )) + self.assertEqual( + invite["shares-needed"], "1", + ) + self.assertEqual( + invite["shares-happy"], "2", + ) + self.assertEqual( + invite["shares-total"], "3", + ) + + @defer.inlineCallbacks + def test_invite_success_read_share_config(self): + """ + If ``--shares-{needed,happy,total}`` are not given on the command line + then the invitation is generated using the configured values. + """ + invite = yield self._invite_success(tahoe_config=""" +[client] +shares.needed = 2 +shares.happy = 4 +shares.total = 6 +""") + self.assertEqual( + invite["shares-needed"], "2", + ) + self.assertEqual( + invite["shares-happy"], "4", + ) + self.assertEqual( + invite["shares-total"], "6", + ) + @defer.inlineCallbacks def test_invite_no_furl(self): From 00de3b9e7e9abed7ab41887105516750233613ca Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 24 Mar 2021 13:54:10 -0400 Subject: [PATCH 038/463] type cleanups --- src/allmydata/test/cli/test_invite.py | 35 +++++++++++++++------------ 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index cc610d409..eae082a2b 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -4,7 +4,7 @@ import json from os.path import join try: - from typing import Optional, Tuple + from typing import Optional, Sequence except ImportError: pass @@ -149,9 +149,8 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): intro_dir, ) - def _invite_success(self, extra_args=(), tahoe_config=None): - # type: (Tuple[bytes], Optional[bytes]) -> defer.Deferred + # type: (Sequence[bytes], Optional[bytes]) -> defer.Deferred """ Exercise an expected-success case of ``tahoe invite``. @@ -165,11 +164,12 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): # we've never run the introducer, so it hasn't created # introducer.furl yet priv_dir = join(intro_dir, "private") - with open(join(priv_dir, "introducer.furl"), "w") as f: - f.write("pb://fooblam\n") + with open(join(priv_dir, "introducer.furl"), "w") as fobj_intro: + fobj_intro.write("pb://fooblam\n") if tahoe_config is not None: - with open(join(intro_dir, "tahoe.cfg"), "w") as f: - f.write(tahoe_config) + assert isinstance(tahoe_config, bytes) + with open(join(intro_dir, "tahoe.cfg"), "wb") as fobj_cfg: + fobj_cfg.write(tahoe_config) with mock.patch('allmydata.scripts.tahoe_invite.wormhole') as w: fake_wh = _create_fake_wormhole([ @@ -177,6 +177,14 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): ]) w.create = mock.Mock(return_value=fake_wh) + extra_args = tuple(extra_args) + + d = run_cli( + "-d", intro_dir, + "invite", + *(extra_args + ("foo",)) + ) + def done(result): rc, out, err = result self.assertEqual(2, len(fake_wh.messages)) @@ -197,11 +205,6 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): invite["introducer"], "pb://fooblam", ) return invite - d = run_cli( - "-d", intro_dir, - "invite", - *(extra_args + ("foo",)) - ) d.addCallback(done) return d @@ -211,9 +214,9 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): successfully send an invite """ invite = yield self._invite_success(( - "--shares-needed", "1", - "--shares-happy", "2", - "--shares-total", "3", + b"--shares-needed", b"1", + b"--shares-happy", b"2", + b"--shares-total", b"3", )) self.assertEqual( invite["shares-needed"], "1", @@ -231,7 +234,7 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): If ``--shares-{needed,happy,total}`` are not given on the command line then the invitation is generated using the configured values. """ - invite = yield self._invite_success(tahoe_config=""" + invite = yield self._invite_success(tahoe_config=b""" [client] shares.needed = 2 shares.happy = 4 From 03f27ae9eb7ad32eaf0452384dae9e544c9079dc Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 24 Mar 2021 15:58:00 -0400 Subject: [PATCH 039/463] fix grammar --- newsfragments/3650.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3650.bugfix b/newsfragments/3650.bugfix index a747a1246..09a810239 100644 --- a/newsfragments/3650.bugfix +++ b/newsfragments/3650.bugfix @@ -1 +1 @@ -``tahoe invite`` will now read share encoding/placement configuration from a Tahoe client node configuration file if they are not given on the command line, instead of raising an unhandled exception. +``tahoe invite`` will now read share encoding/placement configuration values from a Tahoe client node configuration file if they are not given on the command line, instead of raising an unhandled exception. From 0c278a5f3c1eb012817b2ec0306291caea55b0d5 Mon Sep 17 00:00:00 2001 From: May-Lee Sia Date: Thu, 25 Mar 2021 17:56:59 +0100 Subject: [PATCH 040/463] Add May-Lee Added my name and my pronouns JIC. --- docs/CODE_OF_CONDUCT.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CODE_OF_CONDUCT.md b/docs/CODE_OF_CONDUCT.md index 3b14f2d70..174157e8c 100644 --- a/docs/CODE_OF_CONDUCT.md +++ b/docs/CODE_OF_CONDUCT.md @@ -45,6 +45,7 @@ The following community members have made themselves available for conduct issue - Jean-Paul Calderone (jean-paul at leastauthority dot com) - meejah (meejah at meejah dot ca) +- May-Lee Sia(she/her) (tahoe dot lafs dot community at gmail dot com) This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.3.0, available at From bfef77a39683d698450d69151059bc15ac507c45 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 15 Feb 2021 14:38:11 -0500 Subject: [PATCH 041/463] Port runner to Python 3. --- src/allmydata/scripts/runner.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index ada3c2dfc..9d5df478d 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -1,5 +1,11 @@ from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import warnings import os, sys from six.moves import StringIO From 110e77b56031b111f48f1827826927dfc9b0f26f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 26 Mar 2021 09:48:46 -0400 Subject: [PATCH 042/463] Mark module as ported --- src/allmydata/util/_python3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index f26b77185..7b3226bc5 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -77,6 +77,7 @@ PORTED_MODULES = [ "allmydata.node", "allmydata.nodemaker", "allmydata.scripts.create_node", + "allmydata.scripts.runner", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From a6147b05b1d52a32e981521e6e3fa07e339ff394 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 26 Mar 2021 11:24:39 -0400 Subject: [PATCH 043/463] Fix test failure in test_unicode_arguments_and_output on Python 2. --- src/allmydata/scripts/runner.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 9d5df478d..dc83e2ff0 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -22,7 +22,7 @@ from twisted.internet import defer, task, threads from allmydata.scripts.common import get_default_nodedir from allmydata.scripts import debug, create_node, cli, \ admin, tahoe_run, tahoe_invite -from allmydata.util.encodingutil import quote_local_unicode_path +from allmydata.util.encodingutil import quote_local_unicode_path, argv_to_unicode from allmydata.util.eliotutil import ( opt_eliot_destination, opt_help_eliot_destinations, @@ -118,6 +118,22 @@ def parse_options(argv, config=None): config.parseOptions(argv) # may raise usage.error return config + +def _safety_encode(msg): + """ + Previously, on Python 2, argv was utf-8 encoded bytes and + Options.parseOptions would pass through those unicode + characters (such as in a UsageError unknown command). + + In a Unicode-preferred world, the argv is decoded to + Unicode early and that's what's passed through, but + print still needs an encoded value. + """ + if PY2: + return msg.encode('utf-8') + return msg + + def parse_or_exit_with_explanation(argv, stdout=sys.stdout): config = Options() try: @@ -127,7 +143,7 @@ def parse_or_exit_with_explanation(argv, stdout=sys.stdout): while hasattr(c, 'subOptions'): c = c.subOptions print(str(c), file=stdout) - print("%s: %s\n" % (sys.argv[0], e), file=stdout) + print(_safety_encode("%s: %s\n" % (sys.argv[0], e)), file=stdout) sys.exit(1) return config @@ -235,7 +251,8 @@ def _run_with_reactor(reactor): _setup_coverage(reactor) - d = defer.maybeDeferred(parse_or_exit_with_explanation, sys.argv[1:]) + argv = list(map(argv_to_unicode, sys.argv[1:])) + d = defer.maybeDeferred(parse_or_exit_with_explanation, argv) d.addCallback(_maybe_enable_eliot_logging, reactor) d.addCallback(dispatch) def _show_exception(f): From f8dddcd1d031b6e187e215a464a4118882bf8612 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 26 Mar 2021 11:27:53 -0400 Subject: [PATCH 044/463] Add newsfragment --- newsfragments/3603.minor.2 | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3603.minor.2 diff --git a/newsfragments/3603.minor.2 b/newsfragments/3603.minor.2 new file mode 100644 index 000000000..e69de29bb From 32b2e93302393b8b5b8b1cb59ad48ed1d01001a3 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 27 Mar 2021 10:03:12 -0400 Subject: [PATCH 045/463] Use newer version of coveralls for reporting --- .github/workflows/ci.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e14268f23..1b75f95b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,10 +87,26 @@ jobs: # Action for this, as of Jan 2021 it does not support Python coverage # files - only lcov files. Therefore, we use coveralls-python, the # coveralls.io-supplied Python reporter, for this. + # + # It is coveralls-python 1.x that has maintained compatibility + # with Python 2, while coveralls-python 3.x is compatible with + # Python 3. Sadly we can't use them both in the same workflow. + # + # The two versions of coveralls-python are somewhat mutually + # incompatible. Mixing these two different versions when + # reporting coverage to coveralls.io will lead to grief, since + # they get job IDs in different fashion. If we use both + # versions of coveralls in the same workflow, the finalizing + # step will be able to mark only part of the jobs as done, and + # the other part will be left hanging, never marked as done. It + # doesn't matter if we make an API call or `coveralls --finish`. + # + # So we just use the newer coveralls-python that is available to + # Python 3 throughout this workflow. - name: "Report Coverage to Coveralls" run: | - pip install coveralls - python -m coveralls + pip3 install --upgrade coveralls==3.0.1 + python3 -m coveralls env: # Some magic value required for some magic reason. GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" From 626eba53b81f6fcbc2a7b6366c370fba9ffaa1c3 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 27 Mar 2021 10:17:22 -0400 Subject: [PATCH 046/463] Run `coveralls --finish` to indicate completion --- .github/workflows/ci.yml | 77 +++++----------------------------------- 1 file changed, 9 insertions(+), 68 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b75f95b9..8eaef0d07 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,80 +129,21 @@ jobs: # a single report, we have to tell Coveralls when we've uploaded all of the # data files. This does it. We make sure it runs last by making it depend # on *all* of the coverage-collecting jobs. + # + # See notes about parallel builds on GitHub Actions at + # https://coveralls-python.readthedocs.io/en/latest/usage/configuration.html finish-coverage-report: - # There happens to just be one coverage-collecting job at the moment. If - # the coverage reports are broken and someone added more - # coverage-collecting jobs to this workflow but didn't update this, that's - # why. - needs: - - "coverage" - runs-on: "ubuntu-latest" + needs: coverage + runs-on: ubuntu-latest + container: python:3-slim steps: - - name: "Check out Tahoe-LAFS sources" - uses: "actions/checkout@v2" - - - name: "Finish Coveralls Reporting" + - name: "Indicate completion to coveralls.io" run: | - # coveralls-python does have a `--finish` option but it doesn't seem - # to work, at least for us. - # https://github.com/coveralls-clients/coveralls-python/issues/248 - # - # But all it does is this simple POST so we can just send it - # ourselves. The only hard part is guessing what the POST - # parameters mean. And I've done that for you already. - # - # Since the build is done I'm going to guess that "done" is a fine - # value for status. - # - # That leaves "build_num". The coveralls documentation gives some - # hints about it. It suggests using $CIRCLE_WORKFLOW_ID if your job - # is on CircleCI. CircleCI documentation says this about - # CIRCLE_WORKFLOW_ID: - # - # Observation of the coveralls.io web interface, logs from the - # coveralls command in action, and experimentation suggests the - # value for PRs is something more like: - # - # -PR- - # - # For branches, it's just the git branch tip hash. - - # For pull requests, refs/pull//merge was just checked out - # by so HEAD will refer to the right revision. For branches, HEAD - # is also the tip of the branch. - REV=$(git rev-parse HEAD) - - # We can get the PR number from the "context". - # - # https://docs.github.com/en/free-pro-team@latest/developers/webhooks-and-events/webhook-events-and-payloads#pull_request - # - # (via ). - # - # If this is a pull request, `github.event` is a `pull_request` - # structure which has `number` right in it. - # - # If this is a push, `github.event` is a `push` instead but we only - # need the revision to construct the build_num. - - PR=${{ github.event.number }} - - if [ "${PR}" = "" ]; then - BUILD_NUM=$REV - else - BUILD_NUM=$REV-PR-$PR - fi - REPO_NAME=$GITHUB_REPOSITORY - - curl \ - -k \ - https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN \ - -d \ - "payload[build_num]=$BUILD_NUM&payload[status]=done&payload[repo_name]=$REPO_NAME" + pip3 install --upgrade coveralls==3.0.1 + python3 -m coveralls --finish env: # Some magic value required for some magic reason. GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - # Help coveralls identify our project. - COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o" integration: runs-on: ${{ matrix.os }} From ed203817d72972054121078a3d4d692eef0bd0fd Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 27 Mar 2021 10:17:42 -0400 Subject: [PATCH 047/463] Add newsfragment --- newsfragments/3653.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3653.minor diff --git a/newsfragments/3653.minor b/newsfragments/3653.minor new file mode 100644 index 000000000..e69de29bb From 4e4faee5fd21f21604c7c6b64d8e610c39855dad Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 27 Mar 2021 10:21:34 -0400 Subject: [PATCH 048/463] Update notes about coveralls versions --- .github/workflows/ci.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8eaef0d07..d565ede47 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,11 +98,14 @@ jobs: # they get job IDs in different fashion. If we use both # versions of coveralls in the same workflow, the finalizing # step will be able to mark only part of the jobs as done, and - # the other part will be left hanging, never marked as done. It - # doesn't matter if we make an API call or `coveralls --finish`. + # the other part will be left hanging, never marked as done: it + # does not matter if we make an API call or `coveralls --finish` + # to indicate that CI has finished running. # - # So we just use the newer coveralls-python that is available to - # Python 3 throughout this workflow. + # So we try to use the newer coveralls-python that is available + # via Python 3 (which is present in GitHub Actions tool cache, + # even when we're running Python 2.7 tests) throughout this + # workflow. - name: "Report Coverage to Coveralls" run: | pip3 install --upgrade coveralls==3.0.1 From 415f3b286647fc4a88a848fdaca3cf029703f616 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 27 Mar 2021 10:46:33 -0400 Subject: [PATCH 049/463] Add newsfragment --- newsfragments/3654.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3654.minor diff --git a/newsfragments/3654.minor b/newsfragments/3654.minor new file mode 100644 index 000000000..e69de29bb From c392a334fc70dc83e56a3683d42434cc6fec8f52 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 27 Mar 2021 11:26:27 -0400 Subject: [PATCH 050/463] Pacify Python 3 `b"foo"` is `str` in Python 2, `bytes` in Python 3 --- src/allmydata/test/cli/test_invite.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index f3f8859b5..20d012995 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -233,9 +233,9 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): successfully send an invite """ invite = yield self._invite_success(( - b"--shares-needed", b"1", - b"--shares-happy", b"2", - b"--shares-total", b"3", + "--shares-needed", "1", + "--shares-happy", "2", + "--shares-total", "3", )) self.assertEqual( invite["shares-needed"], "1", From 43f1f115cbe8b288a83a6bfe4b47121a0ab565a8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 30 Mar 2021 09:46:56 -0400 Subject: [PATCH 051/463] Simplify. --- src/allmydata/scripts/runner.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index dc83e2ff0..7122e499e 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -119,21 +119,6 @@ def parse_options(argv, config=None): return config -def _safety_encode(msg): - """ - Previously, on Python 2, argv was utf-8 encoded bytes and - Options.parseOptions would pass through those unicode - characters (such as in a UsageError unknown command). - - In a Unicode-preferred world, the argv is decoded to - Unicode early and that's what's passed through, but - print still needs an encoded value. - """ - if PY2: - return msg.encode('utf-8') - return msg - - def parse_or_exit_with_explanation(argv, stdout=sys.stdout): config = Options() try: @@ -143,7 +128,10 @@ def parse_or_exit_with_explanation(argv, stdout=sys.stdout): while hasattr(c, 'subOptions'): c = c.subOptions print(str(c), file=stdout) - print(_safety_encode("%s: %s\n" % (sys.argv[0], e)), file=stdout) + # On Python 2 the string may turn into a unicode string, e.g. the error + # may be unicode, in which case it will print funny. Once we're on + # Python 3 we can just drop the ensure_str(). + print(six.ensure_str("%s: %s\n" % (sys.argv[0], e)), file=stdout) sys.exit(1) return config From e97043eeae0fc38202d5f3f56760a81b511ca5f0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 30 Mar 2021 09:58:22 -0400 Subject: [PATCH 052/463] News file. --- newsfragments/3655.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3655.minor diff --git a/newsfragments/3655.minor b/newsfragments/3655.minor new file mode 100644 index 000000000..e69de29bb From 0485eeb126f9c6a1fe43e52d7cdf50e6587c4747 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 30 Mar 2021 10:03:03 -0400 Subject: [PATCH 053/463] Attempt to explain inclusion of lease secrets in the body --- docs/proposed/http-storage-node-protocol.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 36c718c56..dd778473e 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -234,6 +234,19 @@ Because of the simple types used throughout and the equivalence described in `RFC 7049`_ these examples should be representative regardless of which of these two encodings is chosen. +HTTP Design +~~~~~~~~~~~ + +The HTTP interface described here is informed by the ideas of REST +(Representational State Transfer). +For ``GET`` requests query parameters are preferred over values encoded in the request body. +For other requests query parameters are encoded into the message body. + +Many branches of the resource tree are conceived as homogenous containers: +one branch contains all of the share data; +another branch contains all of the lease data; +etc. + General ~~~~~~~ @@ -277,6 +290,9 @@ The lease expires after 31 days. Discussion `````````` +We considered an alternative where ``renew-secret`` and ``cancel-secret`` are placed in query arguments on the request path. +We chose to put these values into the request body to make the URL simpler. + Several behaviors here are blindly copied from the Foolscap-based storage server protocol. * There is a cancel secret but there is no API to use it to cancel a lease. From 90c393b8b22607a986ce596b4e70f8a7ce707843 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 30 Mar 2021 10:04:28 -0400 Subject: [PATCH 054/463] Port __init__.py modules to Python 3 (or just mark them as ported if they're empty). --- src/allmydata/introducer/__init__.py | 13 +++++++++++++ src/allmydata/test/__init__.py | 12 ++++++++++-- src/allmydata/util/_python3.py | 14 ++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/allmydata/introducer/__init__.py b/src/allmydata/introducer/__init__.py index c4a644e76..bfc960e05 100644 --- a/src/allmydata/introducer/__init__.py +++ b/src/allmydata/introducer/__init__.py @@ -1,3 +1,16 @@ +""" +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from allmydata.introducer.server import create_introducer diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py index e3ac48290..a1cbb76a1 100644 --- a/src/allmydata/test/__init__.py +++ b/src/allmydata/test/__init__.py @@ -12,9 +12,17 @@ Some setup that should apply across the entire test suite. Rather than defining interesting APIs for other code to use, this just causes some side-effects which make things better when the test suite runs. -""" -from future.utils import PY3 +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2, PY3 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import warnings from traceback import extract_stack, format_list diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index f26b77185..db2b3307a 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -40,9 +40,11 @@ PORTED_MODULES = [ "allmydata.crypto.util", "allmydata.deep_stats", "allmydata.dirnode", + "allmydata.frontends", "allmydata.frontends.sftpd", "allmydata.hashtree", "allmydata.history", + "allmydata.immutable", "allmydata.immutable.checker", "allmydata.immutable.downloader", "allmydata.immutable.downloader.common", @@ -61,11 +63,13 @@ PORTED_MODULES = [ "allmydata.immutable.repairer", "allmydata.immutable.upload", "allmydata.interfaces", + "allmydata.introducer", "allmydata.introducer.client", "allmydata.introducer.common", "allmydata.introducer.interfaces", "allmydata.introducer.server", "allmydata.monitor", + "allmydata.mutable", "allmydata.mutable.checker", "allmydata.mutable.common", "allmydata.mutable.filenode", @@ -76,10 +80,12 @@ PORTED_MODULES = [ "allmydata.mutable.servermap", "allmydata.node", "allmydata.nodemaker", + "allmydata.scripts", "allmydata.scripts.create_node", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", + "allmydata.storage", "allmydata.storage.common", "allmydata.storage.crawler", "allmydata.storage.expirer", @@ -88,12 +94,18 @@ PORTED_MODULES = [ "allmydata.storage.mutable", "allmydata.storage.server", "allmydata.storage.shares", + "allmydata.test", + "allmydata.test.cli", "allmydata.test.no_network", "allmydata.test.matchers", + "allmydata.test.mutable", "allmydata.test.mutable.util", + "allmydata.test.web", + "allmydata.testing", "allmydata.testing.web", "allmydata.unknown", "allmydata.uri", + "allmydata.util", "allmydata.util._python3", "allmydata.util.abbreviate", "allmydata.util.assertutil", @@ -125,6 +137,7 @@ PORTED_MODULES = [ "allmydata.util.statistics", "allmydata.util.time_format", "allmydata.util.tor_provider", + "allmydata.web", "allmydata.web.check_results", "allmydata.web.common", "allmydata.web.directory", @@ -140,6 +153,7 @@ PORTED_MODULES = [ "allmydata.web.storage_plugins", "allmydata.web.unlinked", "allmydata.webish", + "allmydata.windows", ] PORTED_TEST_MODULES = [ From 45e21f8f703f26a8e98af04c3042428de5d80e1b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 30 Mar 2021 11:05:49 -0400 Subject: [PATCH 055/463] Port to Python 3. --- src/allmydata/__init__.py | 10 ++++++++++ src/allmydata/test/__init__.py | 2 +- src/allmydata/util/_python3.py | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index 3157c8c80..b29868c05 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -3,6 +3,16 @@ Decentralized storage grid. community web site: U{https://tahoe-lafs.org/} """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + # Don't import future str() so we don't break Foolscap serialization on Python 2. + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 + from past.builtins import unicode as str __all__ = [ "__version__", diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py index a1cbb76a1..c75f8d003 100644 --- a/src/allmydata/test/__init__.py +++ b/src/allmydata/test/__init__.py @@ -132,4 +132,4 @@ if sys.platform == "win32": from eliot import to_file from allmydata.util.jsonbytes import BytesJSONEncoder -to_file(open("eliot.log", "w"), encoder=BytesJSONEncoder) +to_file(open("eliot.log", "wb"), encoder=BytesJSONEncoder) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index db2b3307a..0d7f1561c 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -24,6 +24,7 @@ if PY2: # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ + "allmydata", "allmydata.__main__", "allmydata._auto_deps", "allmydata._monkeypatch", From 835c050e6c5cee9cb272bad3de3f17e2e8c9d68a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 30 Mar 2021 12:39:22 -0400 Subject: [PATCH 056/463] fix word-o --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 8d65e121d..5e46480d3 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -33,7 +33,7 @@ In exchange for this slight use of Foolscap's sophisticated mechanisms, Tahoe pays a substantial price: * Foolscap is implemented only for Python. - Tahoe is thus limited to being implemented on in Python. + Tahoe is thus limited to being implemented only in Python. * There is only one Python implementation of Foolscap. The implementation is therefore the de facto standard and understanding of the protocol often relies on understanding that implementation. * The Foolscap developer community is very small. From e843206524b5904f8a49ce3ed6614da67db4ca49 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 30 Mar 2021 12:39:27 -0400 Subject: [PATCH 057/463] expand on justification for performance complaints --- docs/proposed/http-storage-node-protocol.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 5e46480d3..c2a68b3ee 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -39,7 +39,10 @@ Tahoe pays a substantial price: * The Foolscap developer community is very small. The implementation therefore advances very little and some non-trivial part of the maintenance cost falls on the Tahoe project. * The extensible serialization system imposes substantial overhead for the simple data structures Tahoe exchanges. - Tahoe therefore presents a more sluggish experience to users and taxes servers more greatly than is necessary. +* Foolscap encourages a "remote object" style of protocol design with involves many client-server interactions. + However, Foolscap does not implement "promise pipelining". + The result is that Foolscap encourages a protocol that requires many round-trips between client and server. +* The serialization overhead combined with the many round-trips result in Tahoe presenting a more sluggish experience to users and taxes servers more greatly than is necessary. HTTP ~~~~ From 0f0398aeaf24ac7d2ea54c6fc9f06661f552f45a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 30 Mar 2021 12:39:50 -0400 Subject: [PATCH 058/463] minor typography --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index c2a68b3ee..6a2f5e933 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -48,7 +48,7 @@ HTTP ~~~~ HTTP is a request/response protocol that has become the lingua franca of the internet. -Combined with the principles of Representational state transfer (REST) it is widely employed to create, update, and delete data in collections on the internet. +Combined with the principles of Representational State Transfer (REST) it is widely employed to create, update, and delete data in collections on the internet. HTTP itself provides only modest functionality in comparison to Foolscap. However its simplicity and widespread use have led to a diverse and almost overwhelming ecosystem of libraries, frameworks, toolkits, and so on. From 94b92202c3f448561318f03f9b070eea5f14ef2a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 30 Mar 2021 12:45:22 -0400 Subject: [PATCH 059/463] Try to clarify requirement for some security properties --- docs/proposed/http-storage-node-protocol.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 6a2f5e933..a760ed7d9 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -64,6 +64,14 @@ By adopting HTTP in place of Foolscap Tahoe can realize the following concrete b * One of the core features of HTTP is the mundane transfer of bulk data and implementions are often capable of doing this with extreme efficiency. The alignment of this core feature with a core activity of Tahoe of transferring bulk data means that a substantial barrier to improved Tahoe runtime performance will be eliminated. +TLS +~~~ + +The Foolscap-based protocol provides *some* of Tahoe's confidentiality, integrity, and authentication properties by leveraging TLS. +An HTTP-based protocol can make use of TLS in largely the same way to provide the same properties. +Provision of these properties *is* dependant on implementers following Great Black Swamp's rules for x509 certificate validation +(rather than the standard "web" rules for validation). + Requirements ------------ From cef53c40d12e27472e0f1f0e3e44ed302e327d92 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 31 Mar 2021 09:05:18 -0400 Subject: [PATCH 060/463] New ticket number. --- newsfragments/{3603.minor.2 => 3656.minor} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{3603.minor.2 => 3656.minor} (100%) diff --git a/newsfragments/3603.minor.2 b/newsfragments/3656.minor similarity index 100% rename from newsfragments/3603.minor.2 rename to newsfragments/3656.minor From fbcb9bef2922c7b9406daa706253effcbb95b0e5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 31 Mar 2021 10:09:54 -0400 Subject: [PATCH 061/463] Delete unused code. --- src/allmydata/util/sibpath.py | 24 --- src/allmydata/util/verlib.py | 336 ---------------------------------- 2 files changed, 360 deletions(-) delete mode 100644 src/allmydata/util/sibpath.py delete mode 100644 src/allmydata/util/verlib.py diff --git a/src/allmydata/util/sibpath.py b/src/allmydata/util/sibpath.py deleted file mode 100644 index 80a2801a3..000000000 --- a/src/allmydata/util/sibpath.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import sys -from twisted.python.util import sibpath as tsibpath - -def sibpath(path, sibling): - """ - Looks for a named sibling relative to the given path. If such a file - exists, its path will be returned, otherwise a second search will be - made for the named sibling relative to the path of the executable - currently running. This is useful in the case that something built - with py2exe, for example, needs to find data files relative to its - install. Note hence that care should be taken not to search for - private package files whose names might collide with files which might - be found installed alongside the python interpreter itself. If no - file is found in either place, the sibling relative to the given path - is returned, likely leading to a file not found error. - """ - sib = tsibpath(path, sibling) - if not os.path.exists(sib): - exe_sib = tsibpath(sys.executable, sibling) - if os.path.exists(exe_sib): - return exe_sib - return sib - diff --git a/src/allmydata/util/verlib.py b/src/allmydata/util/verlib.py deleted file mode 100644 index 2dfc24a1b..000000000 --- a/src/allmydata/util/verlib.py +++ /dev/null @@ -1,336 +0,0 @@ -""" -"Rational" version definition and parsing for DistutilsVersionFight -discussion at PyCon 2009. - -Ported to Python 3. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -import re - - -class IrrationalVersionError(Exception): - """This is an irrational version.""" - pass - -class HugeMajorVersionNumError(IrrationalVersionError): - """An irrational version because the major version number is huge - (often because a year or date was used). - - See `error_on_huge_major_num` option in `NormalizedVersion` for details. - This guard can be disabled by setting that option False. - """ - pass - -# A marker used in the second and third parts of the `parts` tuple, for -# versions that don't have those segments, to sort properly. An example -# of versions in sort order ('highest' last): -# 1.0b1 ((1,0), ('b',1), ('f',)) -# 1.0.dev345 ((1,0), ('f',), ('dev', 345)) -# 1.0 ((1,0), ('f',), ('f',)) -# 1.0.post256.dev345 ((1,0), ('f',), ('f', 'post', 256, 'dev', 345)) -# 1.0.post345 ((1,0), ('f',), ('f', 'post', 345, 'f')) -# ^ ^ ^ -# 'b' < 'f' ---------------------/ | | -# | | -# 'dev' < 'f' < 'post' -------------------/ | -# | -# 'dev' < 'f' ----------------------------------------------/ -# Other letters would do, but 'f' for 'final' is kind of nice. -FINAL_MARKER = ('f',) - -VERSION_RE = re.compile(r''' - ^ - (?P\d+\.\d+) # minimum 'N.N' - (?P(?:\.\d+)*) # any number of extra '.N' segments - (?: - (?P[abc]|rc) # 'a'=alpha, 'b'=beta, 'c'=release candidate - # 'rc'= alias for release candidate - (?P\d+(?:\.\d+)*) - )? - (?P(\.post(?P\d+))?(\.dev(?P\d+))?)? - $''', re.VERBOSE) - -class NormalizedVersion(object): - """A rational version. - - Good: - 1.2 # equivalent to "1.2.0" - 1.2.0 - 1.2a1 - 1.2.3a2 - 1.2.3b1 - 1.2.3c1 - 1.2.3.4 - TODO: fill this out - - Bad: - 1 # mininum two numbers - 1.2a # release level must have a release serial - 1.2.3b - """ - def __init__(self, s, error_on_huge_major_num=True): - """Create a NormalizedVersion instance from a version string. - - @param s {str} The version string. - @param error_on_huge_major_num {bool} Whether to consider an - apparent use of a year or full date as the major version number - an error. Default True. One of the observed patterns on PyPI before - the introduction of `NormalizedVersion` was version numbers like this: - 2009.01.03 - 20040603 - 2005.01 - This guard is here to strongly encourage the package author to - use an alternate version, because a release deployed into PyPI - and, e.g. downstream Linux package managers, will forever remove - the possibility of using a version number like "1.0" (i.e. - where the major number is less than that huge major number). - """ - self._parse(s, error_on_huge_major_num) - - @classmethod - def from_parts(cls, version, prerelease=FINAL_MARKER, - devpost=FINAL_MARKER): - return cls(cls.parts_to_str((version, prerelease, devpost))) - - def _parse(self, s, error_on_huge_major_num=True): - """Parses a string version into parts.""" - match = VERSION_RE.search(s) - if not match: - raise IrrationalVersionError(s) - - groups = match.groupdict() - parts = [] - - # main version - block = self._parse_numdots(groups['version'], s, False, 2) - extraversion = groups.get('extraversion') - if extraversion not in ('', None): - block += self._parse_numdots(extraversion[1:], s) - parts.append(tuple(block)) - - # prerelease - prerel = groups.get('prerel') - if prerel is not None: - block = [prerel] - block += self._parse_numdots(groups.get('prerelversion'), s, - pad_zeros_length=1) - parts.append(tuple(block)) - else: - parts.append(FINAL_MARKER) - - # postdev - if groups.get('postdev'): - post = groups.get('post') - dev = groups.get('dev') - postdev = [] - if post is not None: - postdev.extend([FINAL_MARKER[0], 'post', int(post)]) - if dev is None: - postdev.append(FINAL_MARKER[0]) - if dev is not None: - postdev.extend(['dev', int(dev)]) - parts.append(tuple(postdev)) - else: - parts.append(FINAL_MARKER) - self.parts = tuple(parts) - if error_on_huge_major_num and self.parts[0][0] > 1980: - raise HugeMajorVersionNumError("huge major version number, %r, " - "which might cause future problems: %r" % (self.parts[0][0], s)) - - def _parse_numdots(self, s, full_ver_str, drop_trailing_zeros=True, - pad_zeros_length=0): - """Parse 'N.N.N' sequences, return a list of ints. - - @param s {str} 'N.N.N...' sequence to be parsed - @param full_ver_str {str} The full version string from which this - comes. Used for error strings. - @param drop_trailing_zeros {bool} Whether to drop trailing zeros - from the returned list. Default True. - @param pad_zeros_length {int} The length to which to pad the - returned list with zeros, if necessary. Default 0. - """ - nums = [] - for n in s.split("."): - if len(n) > 1 and n[0] == '0': - raise IrrationalVersionError("cannot have leading zero in " - "version number segment: '%s' in %r" % (n, full_ver_str)) - nums.append(int(n)) - if drop_trailing_zeros: - while nums and nums[-1] == 0: - nums.pop() - while len(nums) < pad_zeros_length: - nums.append(0) - return nums - - def __str__(self): - return self.parts_to_str(self.parts) - - @classmethod - def parts_to_str(cls, parts): - """Transforms a version expressed in tuple into its string - representation.""" - # XXX This doesn't check for invalid tuples - main, prerel, postdev = parts - s = '.'.join(str(v) for v in main) - if prerel is not FINAL_MARKER: - s += prerel[0] - s += '.'.join(str(v) for v in prerel[1:]) - if postdev and postdev is not FINAL_MARKER: - if postdev[0] == 'f': - postdev = postdev[1:] - i = 0 - while i < len(postdev): - if i % 2 == 0: - s += '.' - s += str(postdev[i]) - i += 1 - return s - - def __repr__(self): - return "%s('%s')" % (self.__class__.__name__, self) - - def _cannot_compare(self, other): - raise TypeError("cannot compare %s and %s" - % (type(self).__name__, type(other).__name__)) - - def __eq__(self, other): - if not isinstance(other, NormalizedVersion): - self._cannot_compare(other) - return self.parts == other.parts - - def __lt__(self, other): - if not isinstance(other, NormalizedVersion): - self._cannot_compare(other) - return self.parts < other.parts - - def __ne__(self, other): - return not self.__eq__(other) - - def __gt__(self, other): - return not (self.__lt__(other) or self.__eq__(other)) - - def __le__(self, other): - return self.__eq__(other) or self.__lt__(other) - - def __ge__(self, other): - return self.__eq__(other) or self.__gt__(other) - -def suggest_normalized_version(s): - """Suggest a normalized version close to the given version string. - - If you have a version string that isn't rational (i.e. NormalizedVersion - doesn't like it) then you might be able to get an equivalent (or close) - rational version from this function. - - This does a number of simple normalizations to the given string, based - on observation of versions currently in use on PyPI. Given a dump of - those version during PyCon 2009, 4287 of them: - - 2312 (53.93%) match NormalizedVersion without change - - with the automatic suggestion - - 3474 (81.04%) match when using this suggestion method - - @param s {str} An irrational version string. - @returns A rational version string, or None, if couldn't determine one. - """ - try: - NormalizedVersion(s) - return s # already rational - except IrrationalVersionError: - pass - - rs = s.lower() - - # part of this could use maketrans - for orig, repl in (('-alpha', 'a'), ('-beta', 'b'), ('alpha', 'a'), - ('beta', 'b'), ('rc', 'c'), ('-final', ''), - ('-pre', 'c'), - ('-release', ''), ('.release', ''), ('-stable', ''), - ('+', '.'), ('_', '.'), (' ', ''), ('.final', ''), - ('final', '')): - rs = rs.replace(orig, repl) - - # if something ends with dev or pre, we add a 0 - rs = re.sub(r"pre$", r"pre0", rs) - rs = re.sub(r"dev$", r"dev0", rs) - - # if we have something like "b-2" or "a.2" at the end of the - # version, that is pobably beta, alpha, etc - # let's remove the dash or dot - rs = re.sub(r"([abc]|rc)[\-\.](\d+)$", r"\1\2", rs) - - # 1.0-dev-r371 -> 1.0.dev371 - # 0.1-dev-r79 -> 0.1.dev79 - rs = re.sub(r"[\-\.](dev)[\-\.]?r?(\d+)$", r".\1\2", rs) - - # Clean: 2.0.a.3, 2.0.b1, 0.9.0~c1 - rs = re.sub(r"[.~]?([abc])\.?", r"\1", rs) - - # Clean: v0.3, v1.0 - if rs.startswith('v'): - rs = rs[1:] - - # Clean leading '0's on numbers. - #TODO: unintended side-effect on, e.g., "2003.05.09" - # PyPI stats: 77 (~2%) better - rs = re.sub(r"\b0+(\d+)(?!\d)", r"\1", rs) - - # Clean a/b/c with no version. E.g. "1.0a" -> "1.0a0". Setuptools infers - # zero. - # PyPI stats: 245 (7.56%) better - rs = re.sub(r"(\d+[abc])$", r"\g<1>0", rs) - - # the 'dev-rNNN' tag is a dev tag - rs = re.sub(r"\.?(dev-r|dev\.r)\.?(\d+)$", r".dev\2", rs) - - # clean the - when used as a pre delimiter - rs = re.sub(r"-(a|b|c)(\d+)$", r"\1\2", rs) - - # a terminal "dev" or "devel" can be changed into ".dev0" - rs = re.sub(r"[\.\-](dev|devel)$", r".dev0", rs) - - # a terminal "dev" can be changed into ".dev0" - rs = re.sub(r"(?![\.\-])dev$", r".dev0", rs) - - # a terminal "final" or "stable" can be removed - rs = re.sub(r"(final|stable)$", "", rs) - - # The 'r' and the '-' tags are post release tags - # 0.4a1.r10 -> 0.4a1.post10 - # 0.9.33-17222 -> 0.9.33.post17222 - # 0.9.33-r17222 -> 0.9.33.post17222 - rs = re.sub(r"\.?(r|-|-r)\.?(\d+)$", r".post\2", rs) - - # Clean 'r' instead of 'dev' usage: - # 0.9.33+r17222 -> 0.9.33.dev17222 - # 1.0dev123 -> 1.0.dev123 - # 1.0.git123 -> 1.0.dev123 - # 1.0.bzr123 -> 1.0.dev123 - # 0.1a0dev.123 -> 0.1a0.dev123 - # PyPI stats: ~150 (~4%) better - rs = re.sub(r"\.?(dev|git|bzr)\.?(\d+)$", r".dev\2", rs) - - # Clean '.pre' (normalized from '-pre' above) instead of 'c' usage: - # 0.2.pre1 -> 0.2c1 - # 0.2-c1 -> 0.2c1 - # 1.0preview123 -> 1.0c123 - # PyPI stats: ~21 (0.62%) better - rs = re.sub(r"\.?(pre|preview|-c)(\d+)$", r"c\g<2>", rs) - - - # Tcl/Tk uses "px" for their post release markers - rs = re.sub(r"p(\d+)$", r".post\1", rs) - - try: - NormalizedVersion(rs) - return rs # already rational - except IrrationalVersionError: - pass - return None From 1ead68d06128960c16dfda77413cafc4045e79d6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 31 Mar 2021 10:10:05 -0400 Subject: [PATCH 062/463] Start of tests for consumer.py. --- src/allmydata/test/test_consumer.py | 80 +++++++++++++++++++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 81 insertions(+) create mode 100644 src/allmydata/test/test_consumer.py diff --git a/src/allmydata/test/test_consumer.py b/src/allmydata/test/test_consumer.py new file mode 100644 index 000000000..98f419764 --- /dev/null +++ b/src/allmydata/test/test_consumer.py @@ -0,0 +1,80 @@ +""" +Tests for allmydata.util.consumer. + +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from zope.interface import implementer +from twisted.trial.unittest import TestCase +from twisted.internet.interfaces import IPushProducer, IPullProducer + +from allmydata.util.consumer import MemoryConsumer, download_to_data + + +@implementer(IPushProducer) +@implementer(IPullProducer) +class Producer: + """Can be used as either streaming or non-streaming producer. + + If used as streaming, the test should call iterate() manually. + """ + + def __init__(self, consumer): + self.data = [b"abc", b"def", b"ghi"] + self.consumer = consumer + self.done = False + + def resumeProducing(self): + """Kick off streaming.""" + self.iterate() + + def iterate(self): + """Do another iteration of writing.""" + if self.done: + raise RuntimeError( + "There's a bug somewhere, shouldn't iterate after being done" + ) + if self.data: + self.consumer.write(self.data.pop(0)) + else: + self.done = True + self.consumer.unregisterProducer() + + +class MemoryConsumerTests(TestCase): + """Tests for MemoryConsumer.""" + + def test_push_producer(self): + """ + A MemoryConsumer accumulates all data sent by a streaming producer. + """ + consumer = MemoryConsumer() + producer = Producer(consumer) + consumer.registerProducer(producer, True) + self.assertEqual(consumer.chunks, [b"abc"]) + producer.iterate() + producer.iterate() + self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"]) + self.assertEqual(consumer.done, False) + producer.iterate() + self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"]) + self.assertEqual(consumer.done, True) + + def test_pull_producer(self): + """ + A MemoryConsumer accumulates all data sent by a non-streaming producer. + """ + consumer = MemoryConsumer() + producer = Producer(consumer) + consumer.registerProducer(producer, False) + self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"]) + self.assertEqual(consumer.done, True) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 2f6857cad..61c444dc1 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -191,6 +191,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_configutil", "allmydata.test.test_connections", "allmydata.test.test_connection_status", + "allmydata.test.test_consumer", "allmydata.test.test_crawler", "allmydata.test.test_crypto", From 0d0dd4dee9d8c88575c8d26aaf5f88d1aea6a431 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 31 Mar 2021 10:35:25 -0400 Subject: [PATCH 063/463] Rip out all references to the unused IProgress API. --- src/allmydata/blacklist.py | 2 +- src/allmydata/dirnode.py | 4 +-- src/allmydata/immutable/encode.py | 12 +------ src/allmydata/immutable/filenode.py | 4 +-- src/allmydata/immutable/literal.py | 5 +-- src/allmydata/immutable/offloaded.py | 4 +-- src/allmydata/immutable/upload.py | 30 +++++------------ src/allmydata/interfaces.py | 48 +++------------------------- src/allmydata/mutable/filenode.py | 16 +++++----- src/allmydata/test/common.py | 10 +++--- src/allmydata/test/test_dirnode.py | 2 +- src/allmydata/util/consumer.py | 11 +++---- src/allmydata/util/progress.py | 37 --------------------- 13 files changed, 39 insertions(+), 146 deletions(-) delete mode 100644 src/allmydata/util/progress.py diff --git a/src/allmydata/blacklist.py b/src/allmydata/blacklist.py index 570d4723e..43eb36cc6 100644 --- a/src/allmydata/blacklist.py +++ b/src/allmydata/blacklist.py @@ -142,7 +142,7 @@ class ProhibitedNode(object): def get_best_readable_version(self): raise FileProhibited(self.reason) - def download_best_version(self, progress=None): + def download_best_version(self): raise FileProhibited(self.reason) def get_best_mutable_version(self): diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index ea8e84e05..fdf373b45 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -646,7 +646,7 @@ class DirectoryNode(object): return d - def add_file(self, namex, uploadable, metadata=None, overwrite=True, progress=None): + def add_file(self, namex, uploadable, metadata=None, overwrite=True): """I upload a file (using the given IUploadable), then attach the resulting FileNode to the directory at the given name. I return a Deferred that fires (with the IFileNode of the uploaded file) when @@ -657,7 +657,7 @@ class DirectoryNode(object): d = DeferredContext(defer.fail(NotWriteableError())) else: # XXX should pass reactor arg - d = DeferredContext(self._uploader.upload(uploadable, progress=progress)) + d = DeferredContext(self._uploader.upload(uploadable)) d.addCallback(lambda results: self._create_and_validate_node(results.get_uri(), None, name)) diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index bbb7f8123..42fc18077 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -90,7 +90,7 @@ PiB=1024*TiB @implementer(IEncoder) class Encoder(object): - def __init__(self, log_parent=None, upload_status=None, progress=None): + def __init__(self, log_parent=None, upload_status=None): object.__init__(self) self.uri_extension_data = {} self._codec = None @@ -102,7 +102,6 @@ class Encoder(object): self._log_number = log.msg("creating Encoder %s" % self, facility="tahoe.encoder", parent=log_parent) self._aborted = False - self._progress = progress def __repr__(self): if hasattr(self, "_storage_index"): @@ -123,8 +122,6 @@ class Encoder(object): def _got_size(size): self.log(format="file size: %(size)d", size=size) self.file_size = size - if self._progress: - self._progress.set_progress_total(self.file_size) d.addCallback(_got_size) d.addCallback(lambda res: eu.get_all_encoding_parameters()) d.addCallback(self._got_all_encoding_parameters) @@ -462,13 +459,6 @@ class Encoder(object): dl = self._gather_responses(dl) - def do_progress(ign): - done = self.segment_size * (segnum + 1) - if self._progress: - self._progress.set_progress(done) - return ign - dl.addCallback(do_progress) - def _logit(res): self.log("%s uploaded %s / %s bytes (%d%%) of your file." % (self, diff --git a/src/allmydata/immutable/filenode.py b/src/allmydata/immutable/filenode.py index 9e13e1337..6962c31a4 100644 --- a/src/allmydata/immutable/filenode.py +++ b/src/allmydata/immutable/filenode.py @@ -337,13 +337,13 @@ class ImmutableFileNode(object): """ return defer.succeed(self) - def download_best_version(self, progress=None): + def download_best_version(self): """ Download the best version of this file, returning its contents as a bytestring. Since there is only one version of an immutable file, we download and return the contents of this file. """ - d = consumer.download_to_data(self, progress=progress) + d = consumer.download_to_data(self) return d # for an immutable file, download_to_data (specified in IReadable) diff --git a/src/allmydata/immutable/literal.py b/src/allmydata/immutable/literal.py index 6ed5571b9..544a205e1 100644 --- a/src/allmydata/immutable/literal.py +++ b/src/allmydata/immutable/literal.py @@ -113,10 +113,7 @@ class LiteralFileNode(_ImmutableFileNodeBase): return defer.succeed(self) - def download_best_version(self, progress=None): - if progress is not None: - progress.set_progress_total(len(self.u.data)) - progress.set_progress(len(self.u.data)) + def download_best_version(self): return defer.succeed(self.u.data) diff --git a/src/allmydata/immutable/offloaded.py b/src/allmydata/immutable/offloaded.py index 2c4b9db78..8ce51782c 100644 --- a/src/allmydata/immutable/offloaded.py +++ b/src/allmydata/immutable/offloaded.py @@ -154,8 +154,8 @@ class CHKUploadHelper(Referenceable, upload.CHKUploader): # type: ignore # warn def __init__(self, storage_index, helper, storage_broker, secret_holder, incoming_file, encoding_file, - log_number, progress=None): - upload.CHKUploader.__init__(self, storage_broker, secret_holder, progress=progress) + log_number): + upload.CHKUploader.__init__(self, storage_broker, secret_holder) self._storage_index = storage_index self._helper = helper self._incoming_file = incoming_file diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 3d794abf1..cb332dfdf 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -48,7 +48,7 @@ from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.interfaces import IUploadable, IUploader, IUploadResults, \ IEncryptedUploadable, RIEncryptedUploadable, IUploadStatus, \ NoServersError, InsufficientVersionError, UploadUnhappinessError, \ - DEFAULT_MAX_SEGMENT_SIZE, IProgress, IPeerSelector + DEFAULT_MAX_SEGMENT_SIZE, IPeerSelector from allmydata.immutable import layout from io import BytesIO @@ -945,7 +945,7 @@ class EncryptAnUploadable(object): IEncryptedUploadable.""" CHUNKSIZE = 50*1024 - def __init__(self, original, log_parent=None, progress=None, chunk_size=None): + def __init__(self, original, log_parent=None, chunk_size=None): """ :param chunk_size: The number of bytes to read from the uploadable at a time, or None for some default. @@ -962,7 +962,6 @@ class EncryptAnUploadable(object): self._file_size = None self._ciphertext_bytes_read = 0 self._status = None - self._progress = progress if chunk_size is not None: self.CHUNKSIZE = chunk_size @@ -985,8 +984,6 @@ class EncryptAnUploadable(object): self._file_size = size if self._status: self._status.set_size(size) - if self._progress: - self._progress.set_progress_total(size) return size d.addCallback(_got_size) return d @@ -1229,7 +1226,7 @@ class UploadStatus(object): class CHKUploader(object): - def __init__(self, storage_broker, secret_holder, progress=None, reactor=None): + def __init__(self, storage_broker, secret_holder, reactor=None): # server_selector needs storage_broker and secret_holder self._storage_broker = storage_broker self._secret_holder = secret_holder @@ -1239,7 +1236,6 @@ class CHKUploader(object): self._upload_status = UploadStatus() self._upload_status.set_helper(False) self._upload_status.set_active(True) - self._progress = progress self._reactor = reactor # locate_all_shareholders() will create the following attribute: @@ -1294,7 +1290,6 @@ class CHKUploader(object): self._encoder = encode.Encoder( self._log_number, self._upload_status, - progress=self._progress, ) # this just returns itself yield self._encoder.set_encrypted_uploadable(eu) @@ -1428,13 +1423,12 @@ def read_this_many_bytes(uploadable, size, prepend_data=[]): class LiteralUploader(object): - def __init__(self, progress=None): + def __init__(self): self._status = s = UploadStatus() s.set_storage_index(None) s.set_helper(False) s.set_progress(0, 1.0) s.set_active(False) - self._progress = progress def start(self, uploadable): uploadable = IUploadable(uploadable) @@ -1442,8 +1436,6 @@ class LiteralUploader(object): def _got_size(size): self._size = size self._status.set_size(size) - if self._progress: - self._progress.set_progress_total(size) return read_this_many_bytes(uploadable, size) d.addCallback(_got_size) d.addCallback(lambda data: uri.LiteralFileURI(b"".join(data))) @@ -1467,8 +1459,6 @@ class LiteralUploader(object): self._status.set_progress(1, 1.0) self._status.set_progress(2, 1.0) self._status.set_results(ur) - if self._progress: - self._progress.set_progress(self._size) return ur def close(self): @@ -1867,13 +1857,12 @@ class Uploader(service.MultiService, log.PrefixingLogMixin): name = "uploader" URI_LIT_SIZE_THRESHOLD = 55 - def __init__(self, helper_furl=None, stats_provider=None, history=None, progress=None): + def __init__(self, helper_furl=None, stats_provider=None, history=None): self._helper_furl = helper_furl self.stats_provider = stats_provider self._history = history self._helper = None self._all_uploads = weakref.WeakKeyDictionary() # for debugging - self._progress = progress log.PrefixingLogMixin.__init__(self, facility="tahoe.immutable.upload") service.MultiService.__init__(self) @@ -1907,13 +1896,12 @@ class Uploader(service.MultiService, log.PrefixingLogMixin): return (self._helper_furl, bool(self._helper)) - def upload(self, uploadable, progress=None, reactor=None): + def upload(self, uploadable, reactor=None): """ Returns a Deferred that will fire with the UploadResults instance. """ assert self.parent assert self.running - assert progress is None or IProgress.providedBy(progress) uploadable = IUploadable(uploadable) d = uploadable.get_size() @@ -1922,15 +1910,13 @@ class Uploader(service.MultiService, log.PrefixingLogMixin): precondition(isinstance(default_params, dict), default_params) precondition("max_segment_size" in default_params, default_params) uploadable.set_default_encoding_parameters(default_params) - if progress: - progress.set_progress_total(size) if self.stats_provider: self.stats_provider.count('uploader.files_uploaded', 1) self.stats_provider.count('uploader.bytes_uploaded', size) if size <= self.URI_LIT_SIZE_THRESHOLD: - uploader = LiteralUploader(progress=progress) + uploader = LiteralUploader() return uploader.start(uploadable) else: eu = EncryptAnUploadable(uploadable, self._parentmsgid) @@ -1943,7 +1929,7 @@ class Uploader(service.MultiService, log.PrefixingLogMixin): else: storage_broker = self.parent.get_storage_broker() secret_holder = self.parent._secret_holder - uploader = CHKUploader(storage_broker, secret_holder, progress=progress, reactor=reactor) + uploader = CHKUploader(storage_broker, secret_holder, reactor=reactor) d2.addCallback(lambda x: uploader.start(eu)) self._all_uploads[uploader] = None diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 96d3e813c..2e055a888 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -733,38 +733,6 @@ class MustNotBeUnknownRWError(CapConstraintError): """Cannot add an unknown child cap specified in a rw_uri field.""" -class IProgress(Interface): - """ - Remembers progress measured in arbitrary units. Users of these - instances must call ``set_progress_total`` at least once before - progress can be valid, and must use the same units for both - ``set_progress_total`` and ``set_progress calls``. - - See also: - :class:`allmydata.util.progress.PercentProgress` - """ - - progress = Attribute( - "Current amount of progress (in percentage)" - ) - - def set_progress(value): - """ - Sets the current amount of progress. - - Arbitrary units, but must match units used for - set_progress_total. - """ - - def set_progress_total(value): - """ - Sets the total amount of expected progress - - Arbitrary units, but must be same units as used when calling - set_progress() on this instance).. - """ - - class IReadable(Interface): """I represent a readable object -- either an immutable file, or a specific version of a mutable file. @@ -794,11 +762,9 @@ class IReadable(Interface): def get_size(): """Return the length (in bytes) of this readable object.""" - def download_to_data(progress=None): + def download_to_data(): """Download all of the file contents. I return a Deferred that fires with the contents as a byte string. - - :param progress: None or IProgress implementer """ def read(consumer, offset=0, size=None): @@ -1106,13 +1072,11 @@ class IFileNode(IFilesystemNode): the Deferred will errback with an UnrecoverableFileError. """ - def download_best_version(progress=None): + def download_best_version(): """Download the contents of the version that would be returned by get_best_readable_version(). This is equivalent to calling download_to_data() on the IReadable given by that method. - progress is anything that implements IProgress - I return a Deferred that fires with a byte string when the file has been fully downloaded. To support streaming download, use the 'read' method of IReadable. If no version is recoverable, @@ -1258,7 +1222,7 @@ class IMutableFileNode(IFileNode): everything) to get increased visibility. """ - def upload(new_contents, servermap, progress=None): + def upload(new_contents, servermap): """Replace the contents of the file with new ones. This requires a servermap that was previously updated with MODE_WRITE. @@ -1279,8 +1243,6 @@ class IMutableFileNode(IFileNode): operation. If I do not signal UncoordinatedWriteError, then I was able to write the new version without incident. - ``progress`` is either None or an IProgress provider - I return a Deferred that fires (with a PublishStatus object) when the publish has completed. I will update the servermap in-place with the location of all new shares. @@ -1471,14 +1433,12 @@ class IDirectoryNode(IFilesystemNode): equivalent to calling set_node() multiple times, but is much more efficient.""" - def add_file(name, uploadable, metadata=None, overwrite=True, progress=None): + def add_file(name, uploadable, metadata=None, overwrite=True): """I upload a file (using the given IUploadable), then attach the resulting ImmutableFileNode to the directory at the given name. I set metadata the same way as set_uri and set_node. The child name must be a unicode string. - ``progress`` either provides IProgress or is None - I return a Deferred that fires (with the IFileNode of the uploaded file) when the operation completes.""" diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index baedacb23..cd8cb0dc7 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -418,21 +418,21 @@ class MutableFileNode(object): return d.addCallback(_get_version, version) - def download_best_version(self, progress=None): + def download_best_version(self): """ I return a Deferred that fires with the contents of the best version of this mutable file. """ - return self._do_serialized(self._download_best_version, progress=progress) + return self._do_serialized(self._download_best_version) - def _download_best_version(self, progress=None): + def _download_best_version(self): """ I am the serialized sibling of download_best_version. """ d = self.get_best_readable_version() d.addCallback(self._record_size) - d.addCallback(lambda version: version.download_to_data(progress=progress)) + d.addCallback(lambda version: version.download_to_data()) # It is possible that the download will fail because there # aren't enough shares to be had. If so, we will try again after @@ -447,7 +447,7 @@ class MutableFileNode(object): d = self.get_best_mutable_version() d.addCallback(self._record_size) - d.addCallback(lambda version: version.download_to_data(progress=progress)) + d.addCallback(lambda version: version.download_to_data()) return d d.addErrback(_maybe_retry) @@ -564,7 +564,7 @@ class MutableFileNode(object): return d - def upload(self, new_contents, servermap, progress=None): + def upload(self, new_contents, servermap): """ I overwrite the contents of the best recoverable version of this mutable file with new_contents, using servermap instead of @@ -951,13 +951,13 @@ class MutableFileVersion(object): return self._servermap.size_of_version(self._version) - def download_to_data(self, fetch_privkey=False, progress=None): # type: ignore # fixme + def download_to_data(self, fetch_privkey=False): # type: ignore # fixme """ I return a Deferred that fires with the contents of this readable object as a byte string. """ - c = consumer.MemoryConsumer(progress=progress) + c = consumer.MemoryConsumer() d = self.read(c, fetch_privkey=fetch_privkey) d.addCallback(lambda mc: b"".join(mc.chunks)) return d diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 94a59f5c9..d7f00554d 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -539,8 +539,8 @@ class FakeCHKFileNode(object): # type: ignore # incomplete implementation return defer.succeed(self) - def download_to_data(self, progress=None): - return download_to_data(self, progress=progress) + def download_to_data(self): + return download_to_data(self) download_best_version = download_to_data @@ -717,11 +717,11 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation d.addCallback(_done) return d - def download_best_version(self, progress=None): - return defer.succeed(self._download_best_version(progress=progress)) + def download_best_version(self): + return defer.succeed(self._download_best_version()) - def _download_best_version(self, ignored=None, progress=None): + def _download_best_version(self, ignored=None): if isinstance(self.my_uri, uri.LiteralFileURI): return self.my_uri.data if self.storage_index not in self.all_contents: diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 23a0fd76b..67d331430 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -1592,7 +1592,7 @@ class FakeMutableFile(object): # type: ignore # incomplete implementation def get_write_uri(self): return self.uri.to_string() - def download_best_version(self, progress=None): + def download_best_version(self): return defer.succeed(self.data) def get_writekey(self): diff --git a/src/allmydata/util/consumer.py b/src/allmydata/util/consumer.py index 393d7dec5..a8eededcc 100644 --- a/src/allmydata/util/consumer.py +++ b/src/allmydata/util/consumer.py @@ -9,10 +9,9 @@ from twisted.internet.interfaces import IConsumer @implementer(IConsumer) class MemoryConsumer(object): - def __init__(self, progress=None): + def __init__(self): self.chunks = [] self.done = False - self._progress = progress def registerProducer(self, p, streaming): self.producer = p @@ -25,16 +24,14 @@ class MemoryConsumer(object): def write(self, data): self.chunks.append(data) - if self._progress is not None: - self._progress.set_progress(sum([len(c) for c in self.chunks])) def unregisterProducer(self): self.done = True -def download_to_data(n, offset=0, size=None, progress=None): +def download_to_data(n, offset=0, size=None): """ - :param progress: None or an IProgress implementer + Return Deferred that fires with results of reading from the given filenode. """ - d = n.read(MemoryConsumer(progress=progress), offset, size) + d = n.read(MemoryConsumer(), offset, size) d.addCallback(lambda mc: b"".join(mc.chunks)) return d diff --git a/src/allmydata/util/progress.py b/src/allmydata/util/progress.py deleted file mode 100644 index 3618c88dd..000000000 --- a/src/allmydata/util/progress.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -Utilities relating to computing progress information. - -Ties in with the "consumer" module also -""" - -from allmydata.interfaces import IProgress -from zope.interface import implementer - - -@implementer(IProgress) -class PercentProgress(object): - """ - Represents progress as a percentage, from 0.0 to 100.0 - """ - - def __init__(self, total_size=None): - self._value = 0.0 - self.set_progress_total(total_size) - - def set_progress(self, value): - "IProgress API" - self._value = value - - def set_progress_total(self, size): - "IProgress API" - if size is not None: - size = float(size) - self._total_size = size - - @property - def progress(self): - if self._total_size is None: - return 0 # or 1.0? - if self._total_size <= 0.0: - return 0 - return (self._value / self._total_size) * 100.0 From cf63a3a83aac548b1fff5ad923b7b7b807312c36 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 31 Mar 2021 10:35:51 -0400 Subject: [PATCH 064/463] News file. --- newsfragments/3658.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3658.minor diff --git a/newsfragments/3658.minor b/newsfragments/3658.minor new file mode 100644 index 000000000..e69de29bb From 1ef33d3d69de4a66ef70eb9451ddeb149ef69c7f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 31 Mar 2021 10:50:50 -0400 Subject: [PATCH 065/463] Note download_to_data() is tested elsewhere. --- src/allmydata/test/test_consumer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_consumer.py b/src/allmydata/test/test_consumer.py index 98f419764..b6c1d1d28 100644 --- a/src/allmydata/test/test_consumer.py +++ b/src/allmydata/test/test_consumer.py @@ -78,3 +78,7 @@ class MemoryConsumerTests(TestCase): consumer.registerProducer(producer, False) self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"]) self.assertEqual(consumer.done, True) + + +# download_to_data() is effectively tested by some of the filenode tests, e.g. +# test_immutable.py. From 8439f2820bf00902b185f749c36f5ea7ec9cd643 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 31 Mar 2021 10:53:02 -0400 Subject: [PATCH 066/463] Port to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/util/consumer.py | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 61c444dc1..e3fee8dc6 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -115,6 +115,7 @@ PORTED_MODULES = [ "allmydata.util.base62", "allmydata.util.configutil", "allmydata.util.connection_status", + "allmydata.util.consumer", "allmydata.util.deferredutil", "allmydata.util.dictutil", "allmydata.util.eliotutil", diff --git a/src/allmydata/util/consumer.py b/src/allmydata/util/consumer.py index a8eededcc..3de82974d 100644 --- a/src/allmydata/util/consumer.py +++ b/src/allmydata/util/consumer.py @@ -1,11 +1,22 @@ - -"""This file defines a basic download-to-memory consumer, suitable for use in -a filenode's read() method. See download_to_data() for an example of its use. """ +This file defines a basic download-to-memory consumer, suitable for use in +a filenode's read() method. See download_to_data() for an example of its use. + +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from zope.interface import implementer from twisted.internet.interfaces import IConsumer + @implementer(IConsumer) class MemoryConsumer(object): @@ -28,6 +39,7 @@ class MemoryConsumer(object): def unregisterProducer(self): self.done = True + def download_to_data(n, offset=0, size=None): """ Return Deferred that fires with results of reading from the given filenode. From 88e3005abbea9df8d57104352c122e90d785d771 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 31 Mar 2021 11:53:02 -0400 Subject: [PATCH 067/463] Quote some YAML strings Following PR review feedback. Some parts of GitHub Actions configuration follows convention in GA's documentation, in which YAML strings are not quoted, but that probably is not a good idea. We also don't want to change all the strings in this unrelated set of changes. So we will quote strings as we go, in the blocks we touch. --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d565ede47..39cd114d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,9 +136,10 @@ jobs: # See notes about parallel builds on GitHub Actions at # https://coveralls-python.readthedocs.io/en/latest/usage/configuration.html finish-coverage-report: - needs: coverage - runs-on: ubuntu-latest - container: python:3-slim + needs: + - "coverage" + runs-on: "ubuntu-latest" + container: "python:3-slim" steps: - name: "Indicate completion to coveralls.io" run: | From bb8295ac6111da904b49a771e155d05a929cd9f9 Mon Sep 17 00:00:00 2001 From: May-Lee Sia Date: Wed, 31 Mar 2021 21:14:50 +0200 Subject: [PATCH 068/463] Add newsfragment --- newsfragments/3651.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3651.minor diff --git a/newsfragments/3651.minor b/newsfragments/3651.minor new file mode 100644 index 000000000..e69de29bb From 94358d4587e28999cfa306fe32db2a9fbbf61acd Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 31 Mar 2021 15:54:42 -0400 Subject: [PATCH 069/463] Add newsfragment --- newsfragments/3616.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3616.minor diff --git a/newsfragments/3616.minor b/newsfragments/3616.minor new file mode 100644 index 000000000..e69de29bb From aebbc52f067faf3176809af4736ed162f53db505 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 31 Mar 2021 15:54:14 -0400 Subject: [PATCH 070/463] Add Python 3.6 to GitHub Actions test matrix Let us send more coverage reports to coveralls.io --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39cd114d3..c3e789ede 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,7 @@ jobs: - windows-latest python-version: - 2.7 + - 3.6 steps: From dd3b95a0bd039ec5b337780f1ecc1adf9040b8cb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 31 Mar 2021 15:54:31 -0400 Subject: [PATCH 071/463] Add ubuntu-latest to GitHub Actions matrix --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3e789ede..c45ceaa63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,7 @@ jobs: os: - macos-latest - windows-latest + - ubuntu-latest python-version: - 2.7 - 3.6 From 1351a62ac4b48a4b9c2a05635c1a676151e63cfb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 31 Mar 2021 16:13:41 -0400 Subject: [PATCH 072/463] Expect coverage tests to fail on Python 3.6 + Windows --- .github/workflows/ci.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c45ceaa63..3e460e722 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,17 @@ jobs: - ubuntu-latest python-version: - 2.7 - - 3.6 + + # Also run coverage tests with Python 3.6. Expect failures on + # Windows, until we've had a chance to deal with them. + include: + - python-version: 3.6 + os: + - macos-latest + - ubuntu-latest + - python-version: 3.6 + os: windows-latest + experimental: true steps: From 2e67e070811f42825f24f238cd813bfe18a2dca0 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 31 Mar 2021 16:42:39 -0400 Subject: [PATCH 073/463] Just don't run coverage tests with Python 3.6 + Windows Another test matrix setup I tried is this: jobs: coverage: matrix: os: - macos-latest - windows-latest - ubuntu-latest python-version: - 2.7 include: - python-version: 3.6 os: - macos-latest - ubuntu-latest - python-version: 3.6 os: windows-latest experimental: true But that failed on Python 3.6 + macOS with a simple error message, and no further explanation: "This check failed". Huh? Might simply exclude Windows altogether, because that approach sort of worked in another experiment. --- .github/workflows/ci.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e460e722..dcbab9461 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,17 +23,13 @@ jobs: - ubuntu-latest python-version: - 2.7 - - # Also run coverage tests with Python 3.6. Expect failures on - # Windows, until we've had a chance to deal with them. - include: - - python-version: 3.6 - os: - - macos-latest - - ubuntu-latest + - 3.6 + exclude: + # Do not run coverage tests with Python 3.6 on Windows for + # now. They will fail. Dealing with them separately would + # be simpler. - python-version: 3.6 os: windows-latest - experimental: true steps: From d17f3d36c2e7712f67cceb8b73ebce5899bbb3d6 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 31 Mar 2021 17:57:44 -0400 Subject: [PATCH 074/463] Run coveralls verbosely --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dcbab9461..4c12c7f7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,7 +117,7 @@ jobs: - name: "Report Coverage to Coveralls" run: | pip3 install --upgrade coveralls==3.0.1 - python3 -m coveralls + python3 -m coveralls --verbose env: # Some magic value required for some magic reason. GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -152,7 +152,7 @@ jobs: - name: "Indicate completion to coveralls.io" run: | pip3 install --upgrade coveralls==3.0.1 - python3 -m coveralls --finish + python3 -m coveralls --verbose --finish env: # Some magic value required for some magic reason. GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" From 2a620863230e170cd4fdd51e0c509953690ad0b7 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 31 Mar 2021 19:16:58 -0400 Subject: [PATCH 075/463] Turn down coveralls verbosity --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c12c7f7f..dcbab9461 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,7 +117,7 @@ jobs: - name: "Report Coverage to Coveralls" run: | pip3 install --upgrade coveralls==3.0.1 - python3 -m coveralls --verbose + python3 -m coveralls env: # Some magic value required for some magic reason. GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -152,7 +152,7 @@ jobs: - name: "Indicate completion to coveralls.io" run: | pip3 install --upgrade coveralls==3.0.1 - python3 -m coveralls --verbose --finish + python3 -m coveralls --finish env: # Some magic value required for some magic reason. GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" From de2609151e2add0b505f62c8644a5e86a79e7117 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Apr 2021 10:01:14 -0400 Subject: [PATCH 076/463] Special logic for roundtripping Unicode to Unicode is only necessary on Python 2. --- src/allmydata/test/test_util.py | 7 ++++++- src/allmydata/util/yamlutil.py | 29 +++++++++++++++++++++++------ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index a14adb787..4dc2793a4 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -480,7 +480,12 @@ class EqButNotIs(object): class YAML(unittest.TestCase): def test_convert(self): - data = yaml.safe_dump(["str", u"unicode", u"\u1234nicode"]) + """ + Unicode and (ASCII) native strings get roundtripped to Unicode strings. + """ + data = yaml.safe_dump( + [six.ensure_str("str"), u"unicode", u"\u1234nicode"] + ) back = yamlutil.safe_load(data) self.assertIsInstance(back[0], str) self.assertIsInstance(back[1], str) diff --git a/src/allmydata/util/yamlutil.py b/src/allmydata/util/yamlutil.py index 40c38fa30..f7eb8004f 100644 --- a/src/allmydata/util/yamlutil.py +++ b/src/allmydata/util/yamlutil.py @@ -1,11 +1,28 @@ +from future.utils import PY2 + import yaml -# Announcements contain unicode, because they come from JSON. We tell PyYAML -# to give us unicode instead of str/bytes. -def construct_unicode(loader, node): - return node.value -yaml.SafeLoader.add_constructor("tag:yaml.org,2002:str", - construct_unicode) +if PY2: + # On Python 2 the way pyyaml deals with Unicode strings is inconsistent. + # + # >>> yaml.safe_load(yaml.safe_dump(u"hello")) + # 'hello' + # >>> yaml.safe_load(yaml.safe_dump(u"hello\u1234")) + # u'hello\u1234' + # + # In other words, Unicode strings get roundtripped to byte strings, but + # only sometimes. + # + # In order to ensure unicode stays unicode, we add a configuration saying + # that the YAML String Language-Independent Type ("a sequence of zero or + # more Unicode characters") should be the underlying Unicode string object, + # rather than converting to bytes when possible. + # + # Reference: https://yaml.org/type/str.html + def construct_unicode(loader, node): + return node.value + yaml.SafeLoader.add_constructor("tag:yaml.org,2002:str", + construct_unicode) def safe_load(f): return yaml.safe_load(f) From 5ac631047c26fee933d6407f209ea0733fad3a99 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Apr 2021 10:04:01 -0400 Subject: [PATCH 077/463] Port to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/util/yamlutil.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index e3fee8dc6..e8783aee7 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -140,6 +140,7 @@ PORTED_MODULES = [ "allmydata.util.statistics", "allmydata.util.time_format", "allmydata.util.tor_provider", + "allmydata.util.yamlutil", "allmydata.web", "allmydata.web.check_results", "allmydata.web.common", diff --git a/src/allmydata/util/yamlutil.py b/src/allmydata/util/yamlutil.py index f7eb8004f..fd9fc73e2 100644 --- a/src/allmydata/util/yamlutil.py +++ b/src/allmydata/util/yamlutil.py @@ -1,7 +1,18 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import yaml + if PY2: # On Python 2 the way pyyaml deals with Unicode strings is inconsistent. # From f606420d7823b29e5fb359d5bf91c87b0d4006f1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Apr 2021 10:14:05 -0400 Subject: [PATCH 078/463] Make it new-style. --- src/allmydata/test/test_consumer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_consumer.py b/src/allmydata/test/test_consumer.py index b6c1d1d28..46b87e477 100644 --- a/src/allmydata/test/test_consumer.py +++ b/src/allmydata/test/test_consumer.py @@ -22,7 +22,7 @@ from allmydata.util.consumer import MemoryConsumer, download_to_data @implementer(IPushProducer) @implementer(IPullProducer) -class Producer: +class Producer(object): """Can be used as either streaming or non-streaming producer. If used as streaming, the test should call iterate() manually. From 9f02de688ca4baffb1dee7fd3d599d20776c96b7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Apr 2021 10:19:16 -0400 Subject: [PATCH 079/463] We don't support old Foolscap versions anymore. --- src/allmydata/introducer/common.py | 6 ++++-- src/allmydata/util/rrefutil.py | 13 ------------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/src/allmydata/introducer/common.py b/src/allmydata/introducer/common.py index f67aad203..dd9bb527c 100644 --- a/src/allmydata/introducer/common.py +++ b/src/allmydata/introducer/common.py @@ -11,9 +11,11 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re + +from foolscap.api import SturdyRef from allmydata.crypto.util import remove_prefix from allmydata.crypto import ed25519 -from allmydata.util import base32, rrefutil, jsonbytes as json +from allmydata.util import base32, jsonbytes as json def get_tubid_string_from_ann(ann): @@ -127,6 +129,6 @@ class AnnouncementDescriptor(object): self.serverid = key_s furl = ann_d.get("anonymous-storage-FURL") if furl: - self.connection_hints = rrefutil.connection_hints_for_furl(furl) + self.connection_hints = list(SturdyRef(furl).locationHints) else: self.connection_hints = [] diff --git a/src/allmydata/util/rrefutil.py b/src/allmydata/util/rrefutil.py index 40e921507..abcd0e4d2 100644 --- a/src/allmydata/util/rrefutil.py +++ b/src/allmydata/util/rrefutil.py @@ -20,19 +20,6 @@ def add_version_to_remote_reference(rref, default): return d -def connection_hints_for_furl(furl): - hints = [] - for h in SturdyRef(furl).locationHints: - # Foolscap-0.2.5 and earlier used strings in .locationHints, 0.2.6 - # through 0.6.4 used tuples of ("ipv4",host,port), 0.6.5 through - # 0.8.0 used tuples of ("tcp",host,port), and >=0.9.0 uses strings - # again. Tolerate them all. - if isinstance(h, tuple): - hints.append(":".join([str(s) for s in h])) - else: - hints.append(h) - return hints - def stringify_remote_address(rref): remote = rref.getPeer() if isinstance(remote, address.IPv4Address): From 062740dc23ac43a2d34250e5b574609f06936753 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Apr 2021 10:23:37 -0400 Subject: [PATCH 080/463] Probably not worth unit testing a utility function used in one place, move it to place it's used. --- src/allmydata/introducer/server.py | 10 ++++++++++ src/allmydata/util/rrefutil.py | 9 --------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index dcc2fd2c0..6d73c4026 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -24,6 +24,7 @@ except ImportError: from zope.interface import implementer from twisted.application import service from twisted.internet import defer +from twisted.internet.address import IPv4Address from twisted.python.failure import Failure from foolscap.api import Referenceable import allmydata @@ -148,6 +149,15 @@ class _IntroducerNode(node.Node): ws = IntroducerWebishServer(self, webport, nodeurl_path, staticdir) ws.setServiceParent(self) + +def stringify_remote_address(rref): + remote = rref.getPeer() + if isinstance(remote, IPv4Address): + return "%s:%d" % (remote.host, remote.port) + # loopback is a non-IPv4Address + return str(remote) + + @implementer(RIIntroducerPublisherAndSubscriberService_v2) class IntroducerService(service.MultiService, Referenceable): name = "introducer" diff --git a/src/allmydata/util/rrefutil.py b/src/allmydata/util/rrefutil.py index abcd0e4d2..de0f2fbd5 100644 --- a/src/allmydata/util/rrefutil.py +++ b/src/allmydata/util/rrefutil.py @@ -1,5 +1,4 @@ -from twisted.internet import address from foolscap.api import Violation, RemoteException, SturdyRef @@ -18,11 +17,3 @@ def add_version_to_remote_reference(rref, default): return rref d.addCallbacks(_got_version, _no_get_version) return d - - -def stringify_remote_address(rref): - remote = rref.getPeer() - if isinstance(remote, address.IPv4Address): - return "%s:%d" % (remote.host, remote.port) - # loopback is a non-IPv4Address - return str(remote) From 6127fc8cc78c9c0c7da492bca48f9a37526dec44 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Apr 2021 10:49:59 -0400 Subject: [PATCH 081/463] Tests for rrefutil. --- src/allmydata/test/test_util.py | 39 +++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 4dc2793a4..7b0b52783 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -17,12 +17,15 @@ import yaml import json from twisted.trial import unittest +from twisted.internet import defer +from foolscap.api import Violation, RemoteException from allmydata.util import idlib, mathutil from allmydata.util import fileutil from allmydata.util import jsonbytes from allmydata.util import pollmixin from allmydata.util import yamlutil +from allmydata.util import rrefutil from allmydata.util.fileutil import EncryptedTemporaryFile from allmydata.test.common_util import ReallyEqualMixin @@ -526,3 +529,39 @@ class JSONBytes(unittest.TestCase): encoded = jsonbytes.dumps_bytes(x) self.assertIsInstance(encoded, bytes) self.assertEqual(json.loads(encoded, encoding="utf-8"), x) + + +class FakeRemoteRef(object): + """Emulate a RemoteRef.""" + + def __init__(self, result): + self.result = result + + def callRemote(self, method): + assert method == "get_version" + if isinstance(self.result, Exception): + return defer.fail(self.result) + return defer.succeed(self.result) + + +class RrefUtilTests(unittest.TestCase): + """Tests for rrefutil.""" + + def test_version_returned(self): + """If get_version() succeeded, it is set on the rref.""" + rref = FakeRemoteRef(12345) + result = self.successResultOf( + rrefutil.add_version_to_remote_reference(rref, "default") + ) + self.assertEqual(result.version, 12345) + self.assertIdentical(result, rref) + + def test_exceptions(self): + """If get_version() failed, default version is set on the rref.""" + for exception in (Violation(), RemoteException(ValueError())): + rref = FakeRemoteRef(exception) + result = self.successResultOf( + rrefutil.add_version_to_remote_reference(rref, "Default") + ) + self.assertEqual(result.version, "Default") + self.assertIdentical(result, rref) From e92b88195c6554257602442b83b5a721270811ba Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Apr 2021 10:51:13 -0400 Subject: [PATCH 082/463] Port to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/util/rrefutil.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index e8783aee7..56d9e6aec 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -136,6 +136,7 @@ PORTED_MODULES = [ "allmydata.util.observer", "allmydata.util.pipeline", "allmydata.util.pollmixin", + "allmydata.util.rrefutil", "allmydata.util.spans", "allmydata.util.statistics", "allmydata.util.time_format", diff --git a/src/allmydata/util/rrefutil.py b/src/allmydata/util/rrefutil.py index de0f2fbd5..5d57a2f45 100644 --- a/src/allmydata/util/rrefutil.py +++ b/src/allmydata/util/rrefutil.py @@ -1,3 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from foolscap.api import Violation, RemoteException, SturdyRef From 003e9c62a8106add64564b0b2ec47df1204cd182 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Apr 2021 10:58:27 -0400 Subject: [PATCH 083/463] Delete unused code paths. --- src/allmydata/util/dbutil.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/allmydata/util/dbutil.py b/src/allmydata/util/dbutil.py index 543dd2797..11491a4d0 100644 --- a/src/allmydata/util/dbutil.py +++ b/src/allmydata/util/dbutil.py @@ -2,8 +2,6 @@ import os, sys import sqlite3 -from sqlite3 import IntegrityError -[IntegrityError] class DBError(Exception): @@ -12,7 +10,7 @@ class DBError(Exception): def get_db(dbfile, stderr=sys.stderr, create_version=(None, None), updaters={}, just_create=False, dbname="db", - journal_mode=None, synchronous=None): + ): """Open or create the given db file. The parent directory must exist. create_version=(SCHEMA, VERNUM), and SCHEMA must have a 'version' table. Updaters is a {newver: commands} mapping, where e.g. updaters[2] is used @@ -32,12 +30,6 @@ def get_db(dbfile, stderr=sys.stderr, # The default is unspecified according to . c.execute("PRAGMA foreign_keys = ON;") - if journal_mode is not None: - c.execute("PRAGMA journal_mode = %s;" % (journal_mode,)) - - if synchronous is not None: - c.execute("PRAGMA synchronous = %s;" % (synchronous,)) - if must_create: c.executescript(schema) c.execute("INSERT INTO version (version) VALUES (?)", (target_version,)) From fd1860705ee710bcf03bd0196fd0bced9e06ef0f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Apr 2021 10:58:50 -0400 Subject: [PATCH 084/463] Port to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/util/dbutil.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 56d9e6aec..d5ea3ee31 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -116,6 +116,7 @@ PORTED_MODULES = [ "allmydata.util.configutil", "allmydata.util.connection_status", "allmydata.util.consumer", + "allmydata.util.dbutil", "allmydata.util.deferredutil", "allmydata.util.dictutil", "allmydata.util.eliotutil", diff --git a/src/allmydata/util/dbutil.py b/src/allmydata/util/dbutil.py index 11491a4d0..916382972 100644 --- a/src/allmydata/util/dbutil.py +++ b/src/allmydata/util/dbutil.py @@ -1,3 +1,19 @@ +""" +SQLite3 utilities. + +Test coverage currently provided by test_backupdb.py. + +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os, sys From c15fe70378da89e3b50fedeea7db3a5fcafb54c8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 1 Apr 2021 12:53:25 -0400 Subject: [PATCH 085/463] Replace plain "Tahoe" with "Tahoe-LAFS" as the name of the project/software/system/etc --- docs/proposed/http-storage-node-protocol.rst | 30 ++++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index a760ed7d9..83b6823e7 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -24,25 +24,25 @@ At its core it allows separate processes to refer each other's objects and metho This allows for extremely fine-grained access control in a system that remains highly securable without becoming overwhelmingly complicated. Supporting this is a flexible and extensible serialization system which allows data to be exchanged between processes in carefully controlled ways. -Tahoe avails itself of only a small portion of these features. -A Tahoe storage server typically only exposes one object with a fixed set of methods to clients. -A Tahoe introducer node does roughly the same. -Tahoe exchanges simple data structures that have many common, standard serialized representations. +Tahoe-LAFS avails itself of only a small portion of these features. +A Tahoe-LAFS storage server typically only exposes one object with a fixed set of methods to clients. +A Tahoe-LAFS introducer node does roughly the same. +Tahoe-LAFS exchanges simple data structures that have many common, standard serialized representations. In exchange for this slight use of Foolscap's sophisticated mechanisms, -Tahoe pays a substantial price: +Tahoe-LAFS pays a substantial price: * Foolscap is implemented only for Python. - Tahoe is thus limited to being implemented only in Python. + Tahoe-LAFS is thus limited to being implemented only in Python. * There is only one Python implementation of Foolscap. The implementation is therefore the de facto standard and understanding of the protocol often relies on understanding that implementation. * The Foolscap developer community is very small. - The implementation therefore advances very little and some non-trivial part of the maintenance cost falls on the Tahoe project. -* The extensible serialization system imposes substantial overhead for the simple data structures Tahoe exchanges. + The implementation therefore advances very little and some non-trivial part of the maintenance cost falls on the Tahoe-LAFS project. +* The extensible serialization system imposes substantial overhead for the simple data structures Tahoe-LAFS exchanges. * Foolscap encourages a "remote object" style of protocol design with involves many client-server interactions. However, Foolscap does not implement "promise pipelining". The result is that Foolscap encourages a protocol that requires many round-trips between client and server. -* The serialization overhead combined with the many round-trips result in Tahoe presenting a more sluggish experience to users and taxes servers more greatly than is necessary. +* The serialization overhead combined with the many round-trips result in Tahoe-LAFS presenting a more sluggish experience to users and taxes servers more greatly than is necessary. HTTP ~~~~ @@ -52,22 +52,22 @@ Combined with the principles of Representational State Transfer (REST) it is wid HTTP itself provides only modest functionality in comparison to Foolscap. However its simplicity and widespread use have led to a diverse and almost overwhelming ecosystem of libraries, frameworks, toolkits, and so on. -By adopting HTTP in place of Foolscap Tahoe can realize the following concrete benefits: +By adopting HTTP in place of Foolscap Tahoe-LAFS can realize the following concrete benefits: * Practically every language or runtime has an HTTP protocol implementation (or a dozen of them) available. - This change paves the way for new Tahoe implementations using tools better suited for certain situations + This change paves the way for new Tahoe-LAFS implementations using tools better suited for certain situations (mobile client implementations, high-performance server implementations, easily distributed desktop clients, etc). * The simplicity of and vast quantity of resources about HTTP make it a very easy protocol to learn and use. - This change reduces the barrier to entry for developers to contribute improvements to Tahoe's network interactions. + This change reduces the barrier to entry for developers to contribute improvements to Tahoe-LAFS's network interactions. * For any given language there is very likely an HTTP implementation with a large and active developer community. - Tahoe can therefore benefit from the large effort being put into making better libraries for using HTTP. + Tahoe-LAFS can therefore benefit from the large effort being put into making better libraries for using HTTP. * One of the core features of HTTP is the mundane transfer of bulk data and implementions are often capable of doing this with extreme efficiency. - The alignment of this core feature with a core activity of Tahoe of transferring bulk data means that a substantial barrier to improved Tahoe runtime performance will be eliminated. + The alignment of this core feature with a core activity of Tahoe-LAFS of transferring bulk data means that a substantial barrier to improved Tahoe-LAFS runtime performance will be eliminated. TLS ~~~ -The Foolscap-based protocol provides *some* of Tahoe's confidentiality, integrity, and authentication properties by leveraging TLS. +The Foolscap-based protocol provides *some* of Tahoe-LAFS's confidentiality, integrity, and authentication properties by leveraging TLS. An HTTP-based protocol can make use of TLS in largely the same way to provide the same properties. Provision of these properties *is* dependant on implementers following Great Black Swamp's rules for x509 certificate validation (rather than the standard "web" rules for validation). From 89e1865f78c236b5137c3504fc4f465149bafe3e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 2 Apr 2021 09:27:58 -0400 Subject: [PATCH 086/463] Forget about runtime performance --- docs/proposed/http-storage-node-protocol.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 83b6823e7..ad9dd30bc 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -38,11 +38,7 @@ Tahoe-LAFS pays a substantial price: The implementation is therefore the de facto standard and understanding of the protocol often relies on understanding that implementation. * The Foolscap developer community is very small. The implementation therefore advances very little and some non-trivial part of the maintenance cost falls on the Tahoe-LAFS project. -* The extensible serialization system imposes substantial overhead for the simple data structures Tahoe-LAFS exchanges. -* Foolscap encourages a "remote object" style of protocol design with involves many client-server interactions. - However, Foolscap does not implement "promise pipelining". - The result is that Foolscap encourages a protocol that requires many round-trips between client and server. -* The serialization overhead combined with the many round-trips result in Tahoe-LAFS presenting a more sluggish experience to users and taxes servers more greatly than is necessary. +* The extensible serialization system imposes substantial complexity compared to the simple data structures Tahoe-LAFS actually exchanges. HTTP ~~~~ From add2be1b44e847d6d9a2f754861b1c0f8c264163 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 2 Apr 2021 10:24:43 -0400 Subject: [PATCH 087/463] Pin decorator CI broke when decorator 5.0.1 (a dependency via pytest-twisted) with dropped Python 2.7 compatibility was released. --- newsfragments/3662.minor | 0 setup.py | 4 ++++ 2 files changed, 4 insertions(+) create mode 100644 newsfragments/3662.minor diff --git a/newsfragments/3662.minor b/newsfragments/3662.minor new file mode 100644 index 000000000..e69de29bb diff --git a/setup.py b/setup.py index f5ca91b2d..df770fadc 100644 --- a/setup.py +++ b/setup.py @@ -389,6 +389,10 @@ setup(name="tahoe-lafs", # also set in __init__.py "tox", "pytest", "pytest-twisted", + # XXX: decorator isn't a direct dependency, but pytest-twisted + # depends on decorator, and decorator 5.x isn't compatible with + # Python 2.7. + "decorator < 5", "hypothesis >= 3.6.1", "treq", "towncrier", From 939f1f840bc99947e5c545049a27ecfdafcf1cfc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 2 Apr 2021 12:48:50 -0400 Subject: [PATCH 088/463] Fix reference to new location. --- src/allmydata/introducer/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 6d73c4026..7f72d29f6 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -226,7 +226,7 @@ class IntroducerService(service.MultiService, Referenceable): # tubid will be None. Also, subscribers do not tell us which # pubkey they use; only publishers do that. tubid = rref.getRemoteTubID() or "?" - remote_address = rrefutil.stringify_remote_address(rref) + remote_address = stringify_remote_address(rref) # these three assume subscriber_info["version"]==0, but # should tolerate other versions nickname = subscriber_info.get("nickname", u"?") From dd6e0d546718a27d0ea4ff9fe03c59109113caf2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 2 Apr 2021 12:49:01 -0400 Subject: [PATCH 089/463] Switch to decode_furl(). --- src/allmydata/introducer/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/introducer/common.py b/src/allmydata/introducer/common.py index dd9bb527c..f6f70d861 100644 --- a/src/allmydata/introducer/common.py +++ b/src/allmydata/introducer/common.py @@ -12,7 +12,7 @@ if PY2: import re -from foolscap.api import SturdyRef +from foolscap.furl import decode_furl from allmydata.crypto.util import remove_prefix from allmydata.crypto import ed25519 from allmydata.util import base32, jsonbytes as json @@ -125,10 +125,10 @@ class AnnouncementDescriptor(object): self.service_name = ann_d["service-name"] self.version = ann_d.get("my-version", "") self.nickname = ann_d.get("nickname", u"") - (service_name, key_s) = index + (_, key_s) = index self.serverid = key_s furl = ann_d.get("anonymous-storage-FURL") if furl: - self.connection_hints = list(SturdyRef(furl).locationHints) + _, self.connection_hints, _ = decode_furl(furl) else: self.connection_hints = [] From c21288b3ddf40a338f53293bd7810813b3ad0f78 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 2 Apr 2021 12:49:58 -0400 Subject: [PATCH 090/463] Make it easier to read. --- src/allmydata/test/test_consumer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_consumer.py b/src/allmydata/test/test_consumer.py index 46b87e477..73c99d8a3 100644 --- a/src/allmydata/test/test_consumer.py +++ b/src/allmydata/test/test_consumer.py @@ -28,8 +28,8 @@ class Producer(object): If used as streaming, the test should call iterate() manually. """ - def __init__(self, consumer): - self.data = [b"abc", b"def", b"ghi"] + def __init__(self, consumer, data): + self.data = data self.consumer = consumer self.done = False @@ -58,7 +58,7 @@ class MemoryConsumerTests(TestCase): A MemoryConsumer accumulates all data sent by a streaming producer. """ consumer = MemoryConsumer() - producer = Producer(consumer) + producer = Producer(consumer, [b"abc", b"def", b"ghi"]) consumer.registerProducer(producer, True) self.assertEqual(consumer.chunks, [b"abc"]) producer.iterate() @@ -74,7 +74,7 @@ class MemoryConsumerTests(TestCase): A MemoryConsumer accumulates all data sent by a non-streaming producer. """ consumer = MemoryConsumer() - producer = Producer(consumer) + producer = Producer(consumer, [b"abc", b"def", b"ghi"]) consumer.registerProducer(producer, False) self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"]) self.assertEqual(consumer.done, True) From 9a96fec60401edaf8ab39154e7226797e92852a5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 2 Apr 2021 12:56:49 -0400 Subject: [PATCH 091/463] Use existing IRemoteReference implementation. --- src/allmydata/test/test_util.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 7b0b52783..2f77edf4d 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -28,7 +28,7 @@ from allmydata.util import yamlutil from allmydata.util import rrefutil from allmydata.util.fileutil import EncryptedTemporaryFile from allmydata.test.common_util import ReallyEqualMixin - +from .no_network import fireNow, LocalWrapper if six.PY3: long = int @@ -531,17 +531,16 @@ class JSONBytes(unittest.TestCase): self.assertEqual(json.loads(encoded, encoding="utf-8"), x) -class FakeRemoteRef(object): - """Emulate a RemoteRef.""" +class FakeGetVersion(object): + """Emulate an object with a get_version.""" def __init__(self, result): self.result = result - def callRemote(self, method): - assert method == "get_version" + def remote_get_version(self): if isinstance(self.result, Exception): - return defer.fail(self.result) - return defer.succeed(self.result) + raise self.result + return self.result class RrefUtilTests(unittest.TestCase): @@ -549,7 +548,7 @@ class RrefUtilTests(unittest.TestCase): def test_version_returned(self): """If get_version() succeeded, it is set on the rref.""" - rref = FakeRemoteRef(12345) + rref = LocalWrapper(FakeGetVersion(12345), fireNow) result = self.successResultOf( rrefutil.add_version_to_remote_reference(rref, "default") ) @@ -559,7 +558,7 @@ class RrefUtilTests(unittest.TestCase): def test_exceptions(self): """If get_version() failed, default version is set on the rref.""" for exception in (Violation(), RemoteException(ValueError())): - rref = FakeRemoteRef(exception) + rref = LocalWrapper(FakeGetVersion(exception), fireNow) result = self.successResultOf( rrefutil.add_version_to_remote_reference(rref, "Default") ) From d214fe3f16a8eeaf7fbd003ef2a1cdfe54692b88 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 2 Apr 2021 13:00:05 -0400 Subject: [PATCH 092/463] Delete unused imports. --- src/allmydata/introducer/server.py | 2 +- src/allmydata/test/test_consumer.py | 2 +- src/allmydata/test/test_util.py | 1 - src/allmydata/util/rrefutil.py | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 7f72d29f6..1e28f511b 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -29,7 +29,7 @@ from twisted.python.failure import Failure from foolscap.api import Referenceable import allmydata from allmydata import node -from allmydata.util import log, rrefutil, dictutil +from allmydata.util import log, dictutil from allmydata.util.i2p_provider import create as create_i2p_provider from allmydata.util.tor_provider import create as create_tor_provider from allmydata.introducer.interfaces import \ diff --git a/src/allmydata/test/test_consumer.py b/src/allmydata/test/test_consumer.py index 73c99d8a3..a689de462 100644 --- a/src/allmydata/test/test_consumer.py +++ b/src/allmydata/test/test_consumer.py @@ -17,7 +17,7 @@ from zope.interface import implementer from twisted.trial.unittest import TestCase from twisted.internet.interfaces import IPushProducer, IPullProducer -from allmydata.util.consumer import MemoryConsumer, download_to_data +from allmydata.util.consumer import MemoryConsumer @implementer(IPushProducer) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 2f77edf4d..9887897cf 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -17,7 +17,6 @@ import yaml import json from twisted.trial import unittest -from twisted.internet import defer from foolscap.api import Violation, RemoteException from allmydata.util import idlib, mathutil diff --git a/src/allmydata/util/rrefutil.py b/src/allmydata/util/rrefutil.py index 5d57a2f45..f39890ff1 100644 --- a/src/allmydata/util/rrefutil.py +++ b/src/allmydata/util/rrefutil.py @@ -10,7 +10,7 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from foolscap.api import Violation, RemoteException, SturdyRef +from foolscap.api import Violation, RemoteException def add_version_to_remote_reference(rref, default): From 1c3b1d0d276ce45788cb74c719aa8a03d18a29e8 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 2 Apr 2021 17:25:46 -0400 Subject: [PATCH 093/463] Add newsfragment --- newsfragments/3663.other | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3663.other diff --git a/newsfragments/3663.other b/newsfragments/3663.other new file mode 100644 index 000000000..62abf2666 --- /dev/null +++ b/newsfragments/3663.other @@ -0,0 +1 @@ +You can run `make livehtml` in docs directory to invoke sphinx-autobuild. From c13fb8d7effb40aba19d5068495323f15139152c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 2 Apr 2021 17:27:32 -0400 Subject: [PATCH 094/463] Add a make rule to invoke sphinx-autobuild when building docs --- docs/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/Makefile b/docs/Makefile index ed9e59186..3d7b51f7f 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -214,3 +214,7 @@ pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +.PHONY: livehtml +livehtml: + sphinx-autobuild -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html From 0d2f2825240a20eb9e5ec22aacb7e5df67db33f0 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 2 Apr 2021 18:16:20 -0400 Subject: [PATCH 095/463] Give documentation its own towncrier category --- newsfragments/3664.documentation | 1 + towncrier.pyproject.toml | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 newsfragments/3664.documentation diff --git a/newsfragments/3664.documentation b/newsfragments/3664.documentation new file mode 100644 index 000000000..ab5de8884 --- /dev/null +++ b/newsfragments/3664.documentation @@ -0,0 +1 @@ +Documentation now has its own towncrier category. diff --git a/towncrier.pyproject.toml b/towncrier.pyproject.toml index 26627c3f9..b8b561a98 100644 --- a/towncrier.pyproject.toml +++ b/towncrier.pyproject.toml @@ -37,6 +37,11 @@ name = "Configuration Changes" showcontent = true + [[tool.towncrier.type]] + directory = "documentation" + name = "Documentation Changes" + showcontent = true + [[tool.towncrier.type]] directory = "removed" name = "Removed Features" From 79fc2433b6f14944eebbc710386374971cd521cd Mon Sep 17 00:00:00 2001 From: May-Lee Sia Date: Sat, 3 Apr 2021 20:24:08 +0200 Subject: [PATCH 096/463] Add ticket triage documentation --- docs/ticket-triage.rst | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 docs/ticket-triage.rst diff --git a/docs/ticket-triage.rst b/docs/ticket-triage.rst new file mode 100644 index 000000000..28fa3e660 --- /dev/null +++ b/docs/ticket-triage.rst @@ -0,0 +1,27 @@ +============= +Ticket Triage +============= + +Ticket triage is a weekly, informal ritual that is meant to solve the problem of +tickets getting opened and then forgotten about. It is simple and keeps project +momentum going and prevents ticket cruft. + +It fosters conversation around project tasks and philosophies as they relate to +milestones. + +Process +------- +- The role of Ticket Triager rotates weekly +- The Triager needs admin status on ``Trac`` +- The Triager looks at all the tickets that have been created in the last week + - They can use a custom query or do this as the week progresses + - BONUS ROUND: Dig up a stale ticket from the past +- Assign each ticket to a milestone on the Roadmap +- The following situations merit discussion: + - A ticket doesn't have an appropriate milestone and we should create one + - A ticket should probably be deleted + - A ticket could be assigned to multiple milestones + - There is another question about a ticket +- These tickets will be brought to a weekly meeting (currently Tuesdays) for +discussion +- Ticket Triager role is also assigned at this meeting From 21198e30ffa634b194e736cbbc6b42dc0db6de0f Mon Sep 17 00:00:00 2001 From: May-Lee Sia Date: Sat, 3 Apr 2021 20:36:02 +0200 Subject: [PATCH 097/463] Add newsfragment --- newsfragments/3659.normal | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3659.normal diff --git a/newsfragments/3659.normal b/newsfragments/3659.normal new file mode 100644 index 000000000..e69de29bb From 043a8d8a4de9c7e611cdc25a47eeb30d57d86efa Mon Sep 17 00:00:00 2001 From: May-Lee Sia Date: Sat, 3 Apr 2021 20:54:03 +0200 Subject: [PATCH 098/463] Use towncrier type --- newsfragments/{3659.normal => 3659.minor} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{3659.normal => 3659.minor} (100%) diff --git a/newsfragments/3659.normal b/newsfragments/3659.minor similarity index 100% rename from newsfragments/3659.normal rename to newsfragments/3659.minor From d26b8673fa917bcb4de9645e95e827939a2ca028 Mon Sep 17 00:00:00 2001 From: May-Lee Sia Date: Sat, 3 Apr 2021 21:01:48 +0200 Subject: [PATCH 099/463] Change towncrier type to appropriate category --- newsfragments/{3659.minor => 3659.documentation} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{3659.minor => 3659.documentation} (100%) diff --git a/newsfragments/3659.minor b/newsfragments/3659.documentation similarity index 100% rename from newsfragments/3659.minor rename to newsfragments/3659.documentation From 9a793a93203944fec8de586450c34c7c76af6c41 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sun, 4 Apr 2021 08:44:55 -0400 Subject: [PATCH 100/463] Turn sphinx-build's warnings into errors --- newsfragments/3666.documentation | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3666.documentation diff --git a/newsfragments/3666.documentation b/newsfragments/3666.documentation new file mode 100644 index 000000000..3f9e34777 --- /dev/null +++ b/newsfragments/3666.documentation @@ -0,0 +1 @@ +`tox -e docs` will treat warnings about docs as errors. diff --git a/tox.ini b/tox.ini index 8908142f4..6537e1308 100644 --- a/tox.ini +++ b/tox.ini @@ -234,7 +234,7 @@ deps = # normal install is not needed for docs, and slows things down skip_install = True commands = - sphinx-build -b html -d {toxinidir}/docs/_build/doctrees {toxinidir}/docs {toxinidir}/docs/_build/html + sphinx-build -W -b html -d {toxinidir}/docs/_build/doctrees {toxinidir}/docs {toxinidir}/docs/_build/html [testenv:pyinstaller] # We override this to pass --no-use-pep517 because pyinstaller (3.4, at least) From cc2a7f3e83120311d932a49e51e2e4e8a7d024af Mon Sep 17 00:00:00 2001 From: May-Lee Sia Date: Sun, 4 Apr 2021 18:03:12 +0200 Subject: [PATCH 101/463] Add newline and toctree heading --- docs/index.rst | 1 + docs/ticket-triage.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 60a3aa5d4..6ad09d1bf 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -41,6 +41,7 @@ Contents: backupdb developer-guide + ticket-triage anonymity-configuration diff --git a/docs/ticket-triage.rst b/docs/ticket-triage.rst index 28fa3e660..18446d93e 100644 --- a/docs/ticket-triage.rst +++ b/docs/ticket-triage.rst @@ -24,4 +24,5 @@ Process - There is another question about a ticket - These tickets will be brought to a weekly meeting (currently Tuesdays) for discussion + - Ticket Triager role is also assigned at this meeting From db2e575ba7a8b1eaf9dc0d104a98899bc5fbc71b Mon Sep 17 00:00:00 2001 From: May-Lee Sia Date: Sun, 4 Apr 2021 18:23:41 +0200 Subject: [PATCH 102/463] Remove newline --- docs/ticket-triage.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/ticket-triage.rst b/docs/ticket-triage.rst index 18446d93e..4fedc2b36 100644 --- a/docs/ticket-triage.rst +++ b/docs/ticket-triage.rst @@ -22,7 +22,5 @@ Process - A ticket should probably be deleted - A ticket could be assigned to multiple milestones - There is another question about a ticket -- These tickets will be brought to a weekly meeting (currently Tuesdays) for -discussion - +- These tickets will be brought to a weekly meeting (currently Tuesdays) for discussion - Ticket Triager role is also assigned at this meeting From e836a6598ee59a016e116e02745057493b507893 Mon Sep 17 00:00:00 2001 From: May-Lee Sia Date: Mon, 5 Apr 2021 13:30:11 +0200 Subject: [PATCH 103/463] Avoid deletion of tickets --- docs/ticket-triage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ticket-triage.rst b/docs/ticket-triage.rst index 4fedc2b36..86096849b 100644 --- a/docs/ticket-triage.rst +++ b/docs/ticket-triage.rst @@ -19,7 +19,7 @@ Process - Assign each ticket to a milestone on the Roadmap - The following situations merit discussion: - A ticket doesn't have an appropriate milestone and we should create one - - A ticket should probably be deleted + - A ticket, in vanishingly rare circumstances, should be deleted - A ticket could be assigned to multiple milestones - There is another question about a ticket - These tickets will be brought to a weekly meeting (currently Tuesdays) for discussion From e074034e8f88db453bc0dc46e149b65e143664b9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 10:24:04 -0400 Subject: [PATCH 104/463] News file. --- newsfragments/3657.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3657.minor diff --git a/newsfragments/3657.minor b/newsfragments/3657.minor new file mode 100644 index 000000000..e69de29bb From b981e90de3286bc2d4244bf304a3bd777056d2aa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 10:51:44 -0400 Subject: [PATCH 105/463] Tahoe-LAFS now relies on a sufficiently new version of Twisted, which includes this bugfix. --- src/allmydata/test/_twisted_9607.py | 74 ----------------------------- src/allmydata/test/test_system.py | 3 +- 2 files changed, 1 insertion(+), 76 deletions(-) delete mode 100644 src/allmydata/test/_twisted_9607.py diff --git a/src/allmydata/test/_twisted_9607.py b/src/allmydata/test/_twisted_9607.py deleted file mode 100644 index c4e37ef38..000000000 --- a/src/allmydata/test/_twisted_9607.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -A copy of the implementation of Twisted's ``getProcessOutputAndValue`` -with the fix for Twisted #9607 (support for stdinBytes) patched in. -""" - -from __future__ import ( - division, - absolute_import, - print_function, - unicode_literals, -) - -from io import BytesIO - -from twisted.internet import protocol, defer - - -class _EverythingGetter(protocol.ProcessProtocol, object): - - def __init__(self, deferred, stdinBytes=None): - self.deferred = deferred - self.outBuf = BytesIO() - self.errBuf = BytesIO() - self.outReceived = self.outBuf.write - self.errReceived = self.errBuf.write - self.stdinBytes = stdinBytes - - def connectionMade(self): - if self.stdinBytes is not None: - self.transport.writeToChild(0, self.stdinBytes) - # The only compelling reason not to _always_ close stdin here is - # backwards compatibility. - self.transport.closeStdin() - - def processEnded(self, reason): - out = self.outBuf.getvalue() - err = self.errBuf.getvalue() - e = reason.value - code = e.exitCode - if e.signal: - self.deferred.errback((out, err, e.signal)) - else: - self.deferred.callback((out, err, code)) - - - -def _callProtocolWithDeferred(protocol, executable, args, env, path, - reactor=None, protoArgs=()): - if reactor is None: - from twisted.internet import reactor - - d = defer.Deferred() - p = protocol(d, *protoArgs) - reactor.spawnProcess(p, executable, (executable,)+tuple(args), env, path) - return d - - - -def getProcessOutputAndValue(executable, args=(), env={}, path=None, - reactor=None, stdinBytes=None): - """Spawn a process and returns a Deferred that will be called back with - its output (from stdout and stderr) and it's exit code as (out, err, code) - If a signal is raised, the Deferred will errback with the stdout and - stderr up to that point, along with the signal, as (out, err, signalNum) - """ - return _callProtocolWithDeferred( - _EverythingGetter, - executable, - args, - env, - path, - reactor, - protoArgs=(stdinBytes,), - ) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index ce575ce7a..040104b4c 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -50,8 +50,7 @@ from twisted.python.failure import Failure from twisted.python.filepath import ( FilePath, ) - -from ._twisted_9607 import ( +from twisted.internet.utils import ( getProcessOutputAndValue, ) From eedc8f23cffc89c320cc065a981dacaaee67344e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 11:00:12 -0400 Subject: [PATCH 106/463] Delete some unused code. --- src/allmydata/test/common_util.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index f62cd34cc..be00522cd 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -33,13 +33,6 @@ def skip_if_cannot_represent_filename(u): except UnicodeEncodeError: raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.") -def skip_if_cannot_represent_argv(u): - precondition(isinstance(u, unicode)) - try: - u.encode(get_io_encoding()) - except UnicodeEncodeError: - raise unittest.SkipTest("A non-ASCII argv could not be encoded on this platform.") - def _getvalue(io): """ @@ -395,27 +388,8 @@ class TimezoneMixin(object): return hasattr(time, 'tzset') -try: - import win32file - import win32con - def make_readonly(path): - win32file.SetFileAttributes(path, win32con.FILE_ATTRIBUTE_READONLY) - def make_accessible(path): - win32file.SetFileAttributes(path, win32con.FILE_ATTRIBUTE_NORMAL) -except ImportError: - import stat - def _make_readonly(path): - os.chmod(path, stat.S_IREAD) - os.chmod(os.path.dirname(path), stat.S_IREAD) - def _make_accessible(path): - os.chmod(os.path.dirname(path), stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD) - os.chmod(path, stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD) - make_readonly = _make_readonly - make_accessible = _make_accessible - - __all__ = [ - "make_readonly", "make_accessible", "TestMixin", "ShouldFailMixin", + "TestMixin", "ShouldFailMixin", "StallMixin", "skip_if_cannot_represent_argv", "run_cli", "parse_cli", "DevNullDictionary", "insecurerandstr", "flip_bit", "flip_one_bit", "SignalMixin", "skip_if_cannot_represent_filename", "ReallyEqualMixin" From 80385aea8ec9a583302f68244b68490817ac7d7b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 11:13:56 -0400 Subject: [PATCH 107/463] Port to Python 3. --- src/allmydata/test/cli_node_api.py | 15 +++++++++++++-- src/allmydata/util/_python3.py | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli_node_api.py b/src/allmydata/test/cli_node_api.py index 34d73a199..4e4173924 100644 --- a/src/allmydata/test/cli_node_api.py +++ b/src/allmydata/test/cli_node_api.py @@ -1,3 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 __all__ = [ "CLINodeAPI", @@ -81,7 +92,7 @@ class _ProcessProtocolAdapter(ProcessProtocol, object): self._fds = fds def connectionMade(self): - for proto in self._fds.values(): + for proto in list(self._fds.values()): proto.makeConnection(self.transport) def childDataReceived(self, childFD, data): @@ -94,7 +105,7 @@ class _ProcessProtocolAdapter(ProcessProtocol, object): def processEnded(self, reason): notified = set() - for proto in self._fds.values(): + for proto in list(self._fds.values()): if proto not in notified: proto.connectionLost(reason) notified.add(proto) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 2f6857cad..1d31725a6 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -98,6 +98,7 @@ PORTED_MODULES = [ "allmydata.storage.shares", "allmydata.test", "allmydata.test.cli", + "allmydata.test.cli_node_api", "allmydata.test.no_network", "allmydata.test.matchers", "allmydata.test.mutable", From 84e32882b425786b14590db9e4d5f8900bccd106 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 11:07:55 -0400 Subject: [PATCH 108/463] These don't belong in tests module. --- {src/allmydata/test => misc/checkers}/check_load.py | 0 {src/allmydata/test => misc/checkers}/check_memory.py | 0 {src/allmydata/test => misc/checkers}/check_speed.py | 0 src/allmydata/test/test_python2_regressions.py | 1 - 4 files changed, 1 deletion(-) rename {src/allmydata/test => misc/checkers}/check_load.py (100%) rename {src/allmydata/test => misc/checkers}/check_memory.py (100%) rename {src/allmydata/test => misc/checkers}/check_speed.py (100%) diff --git a/src/allmydata/test/check_load.py b/misc/checkers/check_load.py similarity index 100% rename from src/allmydata/test/check_load.py rename to misc/checkers/check_load.py diff --git a/src/allmydata/test/check_memory.py b/misc/checkers/check_memory.py similarity index 100% rename from src/allmydata/test/check_memory.py rename to misc/checkers/check_memory.py diff --git a/src/allmydata/test/check_speed.py b/misc/checkers/check_speed.py similarity index 100% rename from src/allmydata/test/check_speed.py rename to misc/checkers/check_speed.py diff --git a/src/allmydata/test/test_python2_regressions.py b/src/allmydata/test/test_python2_regressions.py index fc9ebe17a..59b16d011 100644 --- a/src/allmydata/test/test_python2_regressions.py +++ b/src/allmydata/test/test_python2_regressions.py @@ -15,7 +15,6 @@ from testtools.matchers import ( BLACKLIST = { "allmydata.scripts.types_", - "allmydata.test.check_load", "allmydata.test._win_subprocess", "allmydata.windows.registry", "allmydata.windows.fixups", From a37121f89ca638fa7621675653fc7401375d258e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 11:34:51 -0400 Subject: [PATCH 109/463] Already ported. --- src/allmydata/util/_python3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 1d31725a6..d747e581f 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -103,6 +103,7 @@ PORTED_MODULES = [ "allmydata.test.matchers", "allmydata.test.mutable", "allmydata.test.mutable.util", + "allmydata.test.python3_tests", "allmydata.test.web", "allmydata.testing", "allmydata.testing.web", From 625a0abb025617dc502a06b721478b5ed841f975 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 11:37:32 -0400 Subject: [PATCH 110/463] Port to Python 3. --- src/allmydata/test/common_web.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+) diff --git a/src/allmydata/test/common_web.py b/src/allmydata/test/common_web.py index ce1670341..bd55a9fe9 100644 --- a/src/allmydata/test/common_web.py +++ b/src/allmydata/test/common_web.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from six import ensure_str __all__ = [ diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index d747e581f..b6d4fc2ff 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -99,6 +99,7 @@ PORTED_MODULES = [ "allmydata.test", "allmydata.test.cli", "allmydata.test.cli_node_api", + "allmydata.test.common_web", "allmydata.test.no_network", "allmydata.test.matchers", "allmydata.test.mutable", From a367d333d943e73f1c4faeabbbb5a63a872a7225 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 12:04:09 -0400 Subject: [PATCH 111/463] Port to Python 3. --- src/allmydata/test/common.py | 29 +++++++++++++++++++---------- src/allmydata/util/_python3.py | 1 + 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index d7f00554d..2ce34bb1f 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -1,4 +1,15 @@ +""" +Ported to Python 3. +""" from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from past.builtins import chr as byteschr, str as native_str __all__ = [ "SyncTestCase", @@ -15,8 +26,6 @@ __all__ = [ "PIPE", ] -from past.builtins import chr as byteschr, unicode - import sys import os, random, struct import six @@ -106,7 +115,7 @@ from .eliotutil import ( ) from .common_util import ShouldFailMixin # noqa: F401 -if sys.platform == "win32": +if sys.platform == "win32" and PY2: # Python 2.7 doesn't have good options for launching a process with # non-ASCII in its command line. So use this alternative that does a # better job. However, only use it on Windows because it doesn't work @@ -253,7 +262,7 @@ class UseNode(object): plugin_config = attr.ib() storage_plugin = attr.ib() basedir = attr.ib(validator=attr.validators.instance_of(FilePath)) - introducer_furl = attr.ib(validator=attr.validators.instance_of(str), + introducer_furl = attr.ib(validator=attr.validators.instance_of(native_str), converter=six.ensure_str) node_config = attr.ib(default=attr.Factory(dict)) @@ -264,7 +273,7 @@ class UseNode(object): return "\n".join( " = ".join((key, value)) for (key, value) - in config.items() + in list(config.items()) ) if self.plugin_config is None: @@ -849,17 +858,17 @@ class WebErrorMixin(object): callable=None, *args, **kwargs): # returns a Deferred with the response body if isinstance(substring, bytes): - substring = unicode(substring, "ascii") - if isinstance(response_substring, unicode): + substring = str(substring, "ascii") + if isinstance(response_substring, str): response_substring = response_substring.encode("ascii") - assert substring is None or isinstance(substring, unicode) + assert substring is None or isinstance(substring, str) assert response_substring is None or isinstance(response_substring, bytes) assert callable def _validate(f): if code is not None: self.failUnlessEqual(f.value.status, b"%d" % code, which) if substring: - code_string = unicode(f) + code_string = str(f) self.failUnless(substring in code_string, "%s: substring '%s' not in '%s'" % (which, substring, code_string)) @@ -882,7 +891,7 @@ class WebErrorMixin(object): body = yield response.content() self.assertEquals(response.code, code) if response_substring is not None: - if isinstance(response_substring, unicode): + if isinstance(response_substring, str): response_substring = response_substring.encode("utf-8") self.assertIn(response_substring, body) returnValue(body) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index b6d4fc2ff..bef854054 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -99,6 +99,7 @@ PORTED_MODULES = [ "allmydata.test", "allmydata.test.cli", "allmydata.test.cli_node_api", + "allmydata.test.common", "allmydata.test.common_web", "allmydata.test.no_network", "allmydata.test.matchers", From a11b47785fc6b608d6d7d5d5b156a5d02e1930cb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 12:52:42 -0400 Subject: [PATCH 112/463] Port to Python 3. --- src/allmydata/test/storage_plugin.py | 11 ++++++++++- src/allmydata/util/_python3.py | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index 17ec89078..d3f1ec7c9 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -1,8 +1,17 @@ """ A storage server plugin the test suite can use to validate the functionality. -""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from future.utils import native_str, native_str_to_bytes from six import ensure_str diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index bef854054..97cfd64f8 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -106,6 +106,7 @@ PORTED_MODULES = [ "allmydata.test.mutable", "allmydata.test.mutable.util", "allmydata.test.python3_tests", + "allmydata.test.storage_plugin", "allmydata.test.web", "allmydata.testing", "allmydata.testing.web", From 74e9bdd476d3ca67fca89047562b81b2fe14e197 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 12:56:42 -0400 Subject: [PATCH 113/463] No point in having separate module. --- src/allmydata/test/cli/test_status.py | 18 +++++++++++++++++- src/allmydata/test/status.py | 15 --------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/allmydata/test/cli/test_status.py b/src/allmydata/test/cli/test_status.py index a04939429..4488299b2 100644 --- a/src/allmydata/test/cli/test_status.py +++ b/src/allmydata/test/cli/test_status.py @@ -37,10 +37,26 @@ from allmydata.util import jsonbytes as json from ..no_network import GridTestMixin from ..common_web import do_http -from ..status import FakeStatus from .common import CLITestMixin +class FakeStatus(object): + def __init__(self): + self.status = [] + + def setServiceParent(self, p): + pass + + def get_status(self): + return self.status + + def get_storage_index(self): + return None + + def get_size(self): + return None + + class ProgressBar(unittest.TestCase): def test_ascii0(self): diff --git a/src/allmydata/test/status.py b/src/allmydata/test/status.py index 44f2123f9..8b1378917 100644 --- a/src/allmydata/test/status.py +++ b/src/allmydata/test/status.py @@ -1,16 +1 @@ -class FakeStatus(object): - def __init__(self): - self.status = [] - - def setServiceParent(self, p): - pass - - def get_status(self): - return self.status - - def get_storage_index(self): - return None - - def get_size(self): - return None From 3bc9b0d5440f4b08506ba7723f3d7aff4da8a2dc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 13:00:27 -0400 Subject: [PATCH 114/463] Port to Python 3. --- src/allmydata/test/web/matchers.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+) diff --git a/src/allmydata/test/web/matchers.py b/src/allmydata/test/web/matchers.py index 99c91ef5c..f764da79d 100644 --- a/src/allmydata/test/web/matchers.py +++ b/src/allmydata/test/web/matchers.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import attr from testtools.matchers import Mismatch diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 97cfd64f8..03eafdea5 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -108,6 +108,7 @@ PORTED_MODULES = [ "allmydata.test.python3_tests", "allmydata.test.storage_plugin", "allmydata.test.web", + "allmydata.test.web.matchers", "allmydata.testing", "allmydata.testing.web", "allmydata.unknown", From 57aa798814bcce22a7651cbdeb8217800db6571a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 13:01:23 -0400 Subject: [PATCH 115/463] Delete another item that was deleted. --- src/allmydata/test/common_util.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index be00522cd..07c4a0ed7 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -389,8 +389,7 @@ class TimezoneMixin(object): __all__ = [ - "TestMixin", "ShouldFailMixin", - "StallMixin", "skip_if_cannot_represent_argv", "run_cli", "parse_cli", + "TestMixin", "ShouldFailMixin", "StallMixin", "run_cli", "parse_cli", "DevNullDictionary", "insecurerandstr", "flip_bit", "flip_one_bit", "SignalMixin", "skip_if_cannot_represent_filename", "ReallyEqualMixin" ] From 7b5cb13417cab0ff70d28314ad392fe437ca2eb5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 13:02:12 -0400 Subject: [PATCH 116/463] Flake fix. --- src/allmydata/test/common_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 07c4a0ed7..91f0b0f78 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -20,7 +20,7 @@ from twisted.trial import unittest from ..util.assertutil import precondition from ..scripts import runner -from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, get_io_encoding, argv_type, unicode_to_argv +from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, argv_type, unicode_to_argv def skip_if_cannot_represent_filename(u): From 65fd5a4912550b6da4e6d0e07cdc3aa131e7a7ef Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 5 Apr 2021 14:52:31 -0400 Subject: [PATCH 117/463] Add a note about adding Windows to GitHub Actions test matrix --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dcbab9461..e029104b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,9 @@ jobs: # Do not run coverage tests with Python 3.6 on Windows for # now. They will fail. Dealing with them separately would # be simpler. + # + # XXX: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3669 + # should track the effort to add Windows to the test matrix. - python-version: 3.6 os: windows-latest From 1cddae4133b67eac6e3955184c41cb251d1ea60c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 5 Apr 2021 18:34:33 -0400 Subject: [PATCH 118/463] Add newsfragment --- newsfragments/3669.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3669.minor diff --git a/newsfragments/3669.minor b/newsfragments/3669.minor new file mode 100644 index 000000000..e69de29bb From 5f7c6e4552c75f9989c87423803137c387733649 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 5 Apr 2021 18:34:48 -0400 Subject: [PATCH 119/463] Remove Windows exclusion --- .github/workflows/ci.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e029104b3..c45ceaa63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,15 +24,6 @@ jobs: python-version: - 2.7 - 3.6 - exclude: - # Do not run coverage tests with Python 3.6 on Windows for - # now. They will fail. Dealing with them separately would - # be simpler. - # - # XXX: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3669 - # should track the effort to add Windows to the test matrix. - - python-version: 3.6 - os: windows-latest steps: From d9446f9f06e8efdddce2774e979eab5b2b685905 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 5 Apr 2021 18:57:47 -0400 Subject: [PATCH 120/463] Remove deprecated `U` mode from open() call Under the right conditions (with newer Python 3.x versions), we will see this warning: setup.py:360: DeprecationWarning: 'U' mode is deprecated `U` is for `universal newline mode`. Docs for open() says this: 'U' mode is deprecated and will raise an exception in future versions of Python. It has no effect in Python 3. Use newline to control universal newlines mode. Off it goes. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index df770fadc..308ac0334 100644 --- a/setup.py +++ b/setup.py @@ -357,7 +357,7 @@ if version: setup(name="tahoe-lafs", # also set in __init__.py description='secure, decentralized, fault-tolerant file store', - long_description=open('README.rst', 'rU').read(), + long_description=open('README.rst', 'r').read(), author='the Tahoe-LAFS project', author_email='tahoe-dev@tahoe-lafs.org', url='https://tahoe-lafs.org/', From ae6e1e9e2fde6d8029e7c7e02d3973e345708254 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 5 Apr 2021 19:11:58 -0400 Subject: [PATCH 121/463] Use io.open() instead of builtin open() Windows does not like when we open README.rst using builtin open(): Traceback (most recent call last): File "setup.py", line 360, in long_description=open('README.rst', 'rU').read(), File "c:\hostedtoolcache\windows\python\3.6.8\x64\lib\encodings\cp1252.py", line 23, in decode return codecs.charmap_decode(input,self.errors,decoding_table)[0] UnicodeDecodeError: 'charmap' codec can't decode byte 0x9d in position 1720: character maps to --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 308ac0334..df917eb40 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,7 @@ import sys # See the docs/about.rst file for licensing information. import os, subprocess, re +from io import open basedir = os.path.dirname(os.path.abspath(__file__)) @@ -357,7 +358,7 @@ if version: setup(name="tahoe-lafs", # also set in __init__.py description='secure, decentralized, fault-tolerant file store', - long_description=open('README.rst', 'r').read(), + long_description=open('README.rst', 'r', encoding='utf-8').read(), author='the Tahoe-LAFS project', author_email='tahoe-dev@tahoe-lafs.org', url='https://tahoe-lafs.org/', From 8056b43df6ac7a247f9abefb6d6eed15e3af8c81 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 09:21:14 -0400 Subject: [PATCH 122/463] News file. --- newsfragments/3667.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3667.minor diff --git a/newsfragments/3667.minor b/newsfragments/3667.minor new file mode 100644 index 000000000..e69de29bb From 3841662ee6a4a796277b3481ea859b3d07255c90 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 09:23:31 -0400 Subject: [PATCH 123/463] Fix tests on Python 3. --- src/allmydata/test/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 2ce34bb1f..d874d07ae 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -6,10 +6,10 @@ from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals -from future.utils import PY2 +from future.utils import PY2, native_str if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from past.builtins import chr as byteschr, str as native_str +from past.builtins import chr as byteschr __all__ = [ "SyncTestCase", From 7ff5846b854280613fce152fe0ed0a704fb85159 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 09:41:33 -0400 Subject: [PATCH 124/463] New towncrier has different command-line option. --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 8908142f4..9c04ed56c 100644 --- a/tox.ini +++ b/tox.ini @@ -112,8 +112,8 @@ commands = # If towncrier.check fails, you forgot to add a towncrier news # fragment explaining the change in this branch. Create one at # `newsfragments/.` with some text for the news - # file. See pyproject.toml for legal values. - python -m towncrier.check --pyproject towncrier.pyproject.toml + # file. See towncrier.pyproject.toml for legal values. + python -m towncrier.check --config towncrier.pyproject.toml [testenv:typechecks] From bb3e80c1468ff9fc87e22cadfd8a2cd30e1a8897 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 09:42:01 -0400 Subject: [PATCH 125/463] News file. --- newsfragments/3670.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3670.minor diff --git a/newsfragments/3670.minor b/newsfragments/3670.minor new file mode 100644 index 000000000..e69de29bb From 7f4a99306bd08d0459e00fcf2169e0ac3b744199 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 10:04:18 -0400 Subject: [PATCH 126/463] No need to port to Python 3. --- src/allmydata/test/_win_subprocess.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/allmydata/test/_win_subprocess.py b/src/allmydata/test/_win_subprocess.py index fe6960c73..2c2cb60b4 100644 --- a/src/allmydata/test/_win_subprocess.py +++ b/src/allmydata/test/_win_subprocess.py @@ -1,3 +1,13 @@ +""" +This module is only necessary on Python 2. Once Python 2 code is dropped, it +can be deleted. +""" + +from future.utils import PY3 +if PY3: + raise RuntimeError("Just use subprocess.Popen") + + # -*- coding: utf-8 -*- ## Copyright (C) 2021 Valentin Lab From abe3fbc2e52dda4b6dfa31a298602195a09a2d64 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 10:22:30 -0400 Subject: [PATCH 127/463] Empty file. --- src/allmydata/test/status.py | 1 - 1 file changed, 1 deletion(-) delete mode 100644 src/allmydata/test/status.py diff --git a/src/allmydata/test/status.py b/src/allmydata/test/status.py deleted file mode 100644 index 8b1378917..000000000 --- a/src/allmydata/test/status.py +++ /dev/null @@ -1 +0,0 @@ - From ae7680759dbff6a86b25c76a22e5e7fd3658caea Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 10:22:47 -0400 Subject: [PATCH 128/463] Another checker script. --- {src/allmydata/test => misc/checkers}/check_grid.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {src/allmydata/test => misc/checkers}/check_grid.py (100%) diff --git a/src/allmydata/test/check_grid.py b/misc/checkers/check_grid.py similarity index 100% rename from src/allmydata/test/check_grid.py rename to misc/checkers/check_grid.py From 044c79c4db9c5b5885cdb675fb39c76aa4bfd650 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 10:29:29 -0400 Subject: [PATCH 129/463] Port to Python 3. --- src/allmydata/test/eliotutil.py | 9 +++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 10 insertions(+) diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py index 63c24f08a..c2359f132 100644 --- a/src/allmydata/test/eliotutil.py +++ b/src/allmydata/test/eliotutil.py @@ -1,12 +1,21 @@ """ Tools aimed at the interaction between tests and Eliot. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals # Python 2 compatibility # Can't use `builtins.str` because it's not JSON encodable: # `exceptions.TypeError: is not JSON-encodeable` from past.builtins import unicode as str from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 + from six import ensure_text __all__ = [ diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 03eafdea5..d17cbe6d8 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -101,6 +101,7 @@ PORTED_MODULES = [ "allmydata.test.cli_node_api", "allmydata.test.common", "allmydata.test.common_web", + "allmydata.test.eliotutil", "allmydata.test.no_network", "allmydata.test.matchers", "allmydata.test.mutable", From bb84442f4e939189cab1ea7142f5f3fa223f99ba Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 10:29:46 -0400 Subject: [PATCH 130/463] News file. --- newsfragments/3671.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3671.minor diff --git a/newsfragments/3671.minor b/newsfragments/3671.minor new file mode 100644 index 000000000..e69de29bb From 2257f89d39aa9eeff0bbbfdaebad7d31c7bedfe6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 10:45:44 -0400 Subject: [PATCH 131/463] More semantically robust test. --- src/allmydata/test/web/test_web.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 6b25305c6..e73cc12f8 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1394,8 +1394,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def _got(res_and_status_and_headers): (res, status, headers) = res_and_status_and_headers self.failUnlessReallyEqual(res, "") - self.failUnlessReallyEqual(headers.getRawHeaders("content-length")[0], - str(len(self.BAR_CONTENTS))) + self.failUnlessReallyEqual(int(headers.getRawHeaders("content-length")[0]), + len(self.BAR_CONTENTS)) self.failUnlessReallyEqual(headers.getRawHeaders("content-type"), ["text/plain"]) d.addCallback(_got) @@ -3015,8 +3015,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def _got_headers(res_and_status_and_headers): (res, status, headers) = res_and_status_and_headers self.failUnlessReallyEqual(res, "") - self.failUnlessReallyEqual(headers.getRawHeaders("content-length")[0], - str(len(NEW2_CONTENTS))) + self.failUnlessReallyEqual(int(headers.getRawHeaders("content-length")[0]), + len(NEW2_CONTENTS)) self.failUnlessReallyEqual(headers.getRawHeaders("content-type"), ["text/plain"]) d.addCallback(_got_headers) From 3429f8bf030b8f2713fcabb13cf49fed8e079279 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 10:45:58 -0400 Subject: [PATCH 132/463] Port to Python 3. --- src/allmydata/test/common_util.py | 21 ++++++++++++++------- src/allmydata/util/_python3.py | 1 + 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 91f0b0f78..f4cc4a074 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -1,8 +1,15 @@ +""" +Ported to Python 3. +""" from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals from future.utils import PY2, bchr, binary_type from future.builtins import str as future_str -from past.builtins import unicode +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 import os import time @@ -24,7 +31,7 @@ from allmydata.util.encodingutil import unicode_platform, get_filesystem_encodin def skip_if_cannot_represent_filename(u): - precondition(isinstance(u, unicode)) + precondition(isinstance(u, str)) enc = get_filesystem_encoding() if not unicode_platform(): @@ -44,7 +51,7 @@ def _getvalue(io): def maybe_unicode_to_argv(o): """Convert object to argv form if necessary.""" - if isinstance(o, unicode): + if isinstance(o, str): return unicode_to_argv(o) return o @@ -181,7 +188,7 @@ class DevNullDictionary(dict): return def insecurerandstr(n): - return b''.join(map(bchr, map(randrange, [0]*n, [256]*n))) + return b''.join(map(bchr, list(map(randrange, [0]*n, [256]*n)))) def flip_bit(good, which): """Flip the low-order bit of good[which].""" @@ -211,9 +218,9 @@ class ReallyEqualMixin(object): # type. They're equal, and _logically_ the same type, but have # different types in practice. if a.__class__ == future_str: - a = unicode(a) + a = str(a) if b.__class__ == future_str: - b = unicode(b) + b = str(b) self.assertEqual(type(a), type(b), "a :: %r (%s), b :: %r (%s), %r" % (a, type(a), b, type(b), msg)) @@ -297,7 +304,7 @@ class ShouldFailMixin(object): of the message wrapped by this Failure, or the test will fail. """ - assert substring is None or isinstance(substring, (bytes, unicode)) + assert substring is None or isinstance(substring, (bytes, str)) d = defer.maybeDeferred(callable, *args, **kwargs) def done(res): if isinstance(res, failure.Failure): diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index d17cbe6d8..4676f6955 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -100,6 +100,7 @@ PORTED_MODULES = [ "allmydata.test.cli", "allmydata.test.cli_node_api", "allmydata.test.common", + "allmydata.test.common_util", "allmydata.test.common_web", "allmydata.test.eliotutil", "allmydata.test.no_network", From 315bb672d11ca77de93edef9b07d82c4a6bc8405 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 10:49:00 -0400 Subject: [PATCH 133/463] Port to Python 3. --- src/allmydata/test/web/common.py | 11 +++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 12 insertions(+) diff --git a/src/allmydata/test/web/common.py b/src/allmydata/test/web/common.py index 00a40e3c5..43a13a902 100644 --- a/src/allmydata/test/web/common.py +++ b/src/allmydata/test/web/common.py @@ -1,3 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 4676f6955..5d70d1e73 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -110,6 +110,7 @@ PORTED_MODULES = [ "allmydata.test.python3_tests", "allmydata.test.storage_plugin", "allmydata.test.web", + "allmydata.test.web.common", "allmydata.test.web.matchers", "allmydata.testing", "allmydata.testing.web", From ddcca38f3148abb020e0f4d8139dac9bc3e9e966 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 10:51:14 -0400 Subject: [PATCH 134/463] Port to Python 3. --- src/allmydata/test/strategies.py | 10 ++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 11 insertions(+) diff --git a/src/allmydata/test/strategies.py b/src/allmydata/test/strategies.py index 553b2c226..c0f558ef6 100644 --- a/src/allmydata/test/strategies.py +++ b/src/allmydata/test/strategies.py @@ -1,6 +1,16 @@ """ Hypothesis strategies use for testing Tahoe-LAFS. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from hypothesis.strategies import ( one_of, diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 5d70d1e73..3f705fe09 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -109,6 +109,7 @@ PORTED_MODULES = [ "allmydata.test.mutable.util", "allmydata.test.python3_tests", "allmydata.test.storage_plugin", + "allmydata.test.strategies", "allmydata.test.web", "allmydata.test.web.common", "allmydata.test.web.matchers", From 47c8ce3590b4e3e2ed280a833bcaff67366c4986 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 7 Apr 2021 09:20:28 -0400 Subject: [PATCH 135/463] News file --- newsfragments/3674.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3674.minor diff --git a/newsfragments/3674.minor b/newsfragments/3674.minor new file mode 100644 index 000000000..e69de29bb From 803e00f6b1ad3cb92096b006a6f8f15a10bd5b2d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 7 Apr 2021 09:26:28 -0400 Subject: [PATCH 136/463] Tests pass on Python 3. --- src/allmydata/test/test_multi_introducers.py | 27 +++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/allmydata/test/test_multi_introducers.py b/src/allmydata/test/test_multi_introducers.py index bb22d551f..a016d4977 100644 --- a/src/allmydata/test/test_multi_introducers.py +++ b/src/allmydata/test/test_multi_introducers.py @@ -1,6 +1,9 @@ -#!/usr/bin/python +from past.builtins import unicode + import os +from six import ensure_binary + from twisted.python.filepath import FilePath from twisted.trial import unittest from twisted.internet import defer @@ -43,7 +46,7 @@ class MultiIntroTests(unittest.TestCase): u'intro2':{ 'furl': 'furl4' }, }, } - self.yaml_path.setContent(yamlutil.safe_dump(connections)) + self.yaml_path.setContent(ensure_binary(yamlutil.safe_dump(connections))) # get a client and count of introducer_clients myclient = yield create_client(self.basedir) ic_count = len(myclient.introducer_clients) @@ -73,7 +76,7 @@ class MultiIntroTests(unittest.TestCase): tahoe_cfg_furl = myclient.introducer_clients[0].introducer_furl # assertions - self.failUnlessEqual(fake_furl, tahoe_cfg_furl) + self.failUnlessEqual(fake_furl, unicode(tahoe_cfg_furl, "utf-8")) self.assertEqual( list( warning["message"] @@ -97,10 +100,10 @@ class MultiIntroTests(unittest.TestCase): u'default': { 'furl': 'furl1' }, }, } - self.yaml_path.setContent(yamlutil.safe_dump(connections)) + self.yaml_path.setContent(ensure_binary(yamlutil.safe_dump(connections))) FilePath(self.basedir).child("tahoe.cfg").setContent( - "[client]\n" - "introducer.furl = furl1\n" + b"[client]\n" + b"introducer.furl = furl1\n" ) with self.assertRaises(ValueError) as ctx: @@ -112,7 +115,7 @@ class MultiIntroTests(unittest.TestCase): "please fix impossible configuration.", ) -SIMPLE_YAML = """ +SIMPLE_YAML = b""" introducers: one: furl: furl1 @@ -121,7 +124,7 @@ introducers: # this format was recommended in docs/configuration.rst in 1.12.0, but it # isn't correct (the "furl = furl1" line is recorded as the string value of # the ["one"] key, instead of being parsed as a single-key dictionary). -EQUALS_YAML = """ +EQUALS_YAML = b""" introducers: one: furl = furl1 """ @@ -147,17 +150,17 @@ class NoDefault(unittest.TestCase): connections = {'introducers': { u'one': { 'furl': 'furl1' }, }} - self.yaml_path.setContent(yamlutil.safe_dump(connections)) + self.yaml_path.setContent(ensure_binary(yamlutil.safe_dump(connections))) myclient = yield create_client(self.basedir) tahoe_cfg_furl = myclient.introducer_clients[0].introducer_furl - self.assertEquals(tahoe_cfg_furl, 'furl1') + self.assertEquals(tahoe_cfg_furl, b'furl1') @defer.inlineCallbacks def test_real_yaml(self): self.yaml_path.setContent(SIMPLE_YAML) myclient = yield create_client(self.basedir) tahoe_cfg_furl = myclient.introducer_clients[0].introducer_furl - self.assertEquals(tahoe_cfg_furl, 'furl1') + self.assertEquals(tahoe_cfg_furl, b'furl1') @defer.inlineCallbacks def test_invalid_equals_yaml(self): @@ -172,6 +175,6 @@ class NoDefault(unittest.TestCase): @defer.inlineCallbacks def test_introducerless(self): connections = {'introducers': {} } - self.yaml_path.setContent(yamlutil.safe_dump(connections)) + self.yaml_path.setContent(ensure_binary(yamlutil.safe_dump(connections))) myclient = yield create_client(self.basedir) self.assertEquals(len(myclient.introducer_clients), 0) From 9a17c4a5d2ebdee7de1d1329baa4683715daabf6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 7 Apr 2021 09:28:04 -0400 Subject: [PATCH 137/463] Port to Python 3. --- src/allmydata/test/test_multi_introducers.py | 17 +++++++++++++---- src/allmydata/util/_python3.py | 1 + 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_multi_introducers.py b/src/allmydata/test/test_multi_introducers.py index a016d4977..a385abe54 100644 --- a/src/allmydata/test/test_multi_introducers.py +++ b/src/allmydata/test/test_multi_introducers.py @@ -1,9 +1,18 @@ -from past.builtins import unicode +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from six import ensure_binary import os -from six import ensure_binary - from twisted.python.filepath import FilePath from twisted.trial import unittest from twisted.internet import defer @@ -76,7 +85,7 @@ class MultiIntroTests(unittest.TestCase): tahoe_cfg_furl = myclient.introducer_clients[0].introducer_furl # assertions - self.failUnlessEqual(fake_furl, unicode(tahoe_cfg_furl, "utf-8")) + self.failUnlessEqual(fake_furl, str(tahoe_cfg_furl, "utf-8")) self.assertEqual( list( warning["message"] diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index d34e62d38..1feee8bdb 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -230,6 +230,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_json_metadata", "allmydata.test.test_log", "allmydata.test.test_monitor", + "allmydata.test.test_multi_introducers", "allmydata.test.test_netstring", "allmydata.test.test_no_network", "allmydata.test.test_node", From d6406d5edb75cb06f49bc579ff3ebe939d68d812 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 7 Apr 2021 10:44:48 -0400 Subject: [PATCH 138/463] Some progress towards passing backup tests on Python 3. --- src/allmydata/scripts/cli.py | 2 +- src/allmydata/scripts/common.py | 21 +++++++++++---------- src/allmydata/scripts/tahoe_backup.py | 10 +++++----- src/allmydata/test/cli/test_backup.py | 19 ++++++++++++++----- 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 6c5641b41..811ae7ef9 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -351,7 +351,7 @@ class BackupOptions(FileStoreOptions): line. The file is assumed to be in the argv encoding.""" abs_filepath = argv_to_abspath(filepath) try: - exclude_file = file(abs_filepath) + exclude_file = open(abs_filepath) except: raise BackupConfigurationError('Error opening exclude file %s.' % quote_local_unicode_path(abs_filepath)) try: diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 29342ec6b..0381b7db8 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -1,7 +1,7 @@ # coding: utf-8 from __future__ import print_function -from six import ensure_str +from six import ensure_binary import os, sys, textwrap import codecs @@ -22,6 +22,8 @@ from yaml import ( from future.utils import PY2 if PY2: from future.builtins import str # noqa: F401 +else: + from typing import Union from twisted.python import usage @@ -274,18 +276,17 @@ def get_alias(aliases, path_unicode, default): return uri.from_string_dirnode(aliases[alias]).to_string(), path[colon+1:] def escape_path(path): - # type: (str) -> str + # type: (Union[str,bytes]) -> bytes u""" Return path quoted to US-ASCII, valid URL characters. >>> path = u'/føö/bar/☃' >>> escaped = escape_path(path) - >>> str(escaped) - '/f%C3%B8%C3%B6/bar/%E2%98%83' - >>> escaped.encode('ascii').decode('ascii') == escaped - True + >>> escaped + b'/f%C3%B8%C3%B6/bar/%E2%98%83' """ - segments = path.split("/") - result = "/".join([urllib.parse.quote(unicode_to_url(s)) for s in segments]) - result = ensure_str(result, "ascii") - return result + if isinstance(path, str): + path = path.encode("utf-8") + segments = path.split(b"/") + return b"/".join([urllib.parse.quote(s).encode("ascii") for s in segments]) + diff --git a/src/allmydata/scripts/tahoe_backup.py b/src/allmydata/scripts/tahoe_backup.py index c63558eb1..99f5663ea 100644 --- a/src/allmydata/scripts/tahoe_backup.py +++ b/src/allmydata/scripts/tahoe_backup.py @@ -2,7 +2,7 @@ from __future__ import print_function import os.path import time -import urllib +from urllib.parse import quote as url_quote import json import datetime from allmydata.scripts.common import get_alias, escape_path, DEFAULT_ALIAS, \ @@ -52,7 +52,7 @@ def mkdir(contents, options): def put_child(dirurl, childname, childcap): assert dirurl[-1] != "/" - url = dirurl + "/" + urllib.quote(unicode_to_url(childname)) + "?t=uri" + url = dirurl + "/" + url_quote(unicode_to_url(childname)) + "?t=uri" resp = do_http("PUT", url, childcap) if resp.status not in (200, 201): raise HTTPError("Error during put_child", resp) @@ -97,7 +97,7 @@ class BackerUpper(object): except UnknownAliasError as e: e.display(stderr) return 1 - to_url = nodeurl + "uri/%s/" % urllib.quote(rootcap) + to_url = nodeurl + "uri/%s/" % url_quote(rootcap) if path: to_url += escape_path(path) if not to_url.endswith("/"): @@ -192,7 +192,7 @@ class BackerUpper(object): filecap = r.was_uploaded() self.verboseprint("checking %s" % quote_output(filecap)) nodeurl = self.options['node-url'] - checkurl = nodeurl + "uri/%s?t=check&output=JSON" % urllib.quote(filecap) + checkurl = nodeurl + "uri/%s?t=check&output=JSON" % url_quote(filecap) self._files_checked += 1 resp = do_http("POST", checkurl) if resp.status != 200: @@ -225,7 +225,7 @@ class BackerUpper(object): dircap = r.was_created() self.verboseprint("checking %s" % quote_output(dircap)) nodeurl = self.options['node-url'] - checkurl = nodeurl + "uri/%s?t=check&output=JSON" % urllib.quote(dircap) + checkurl = nodeurl + "uri/%s?t=check&output=JSON" % url_quote(dircap) self._directories_checked += 1 resp = do_http("POST", checkurl) if resp.status != 200: diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index ceecbd662..1bc8b281e 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -1,3 +1,9 @@ +from future.utils import PY2 +if PY2: + import __builtin__ as builtins +else: + import builtins + import os.path from six.moves import cStringIO as StringIO from datetime import timedelta @@ -6,7 +12,6 @@ import re from twisted.trial import unittest from twisted.python.monkey import MonkeyPatcher -import __builtin__ from allmydata.util import fileutil from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.util.encodingutil import get_io_encoding, unicode_to_argv @@ -407,12 +412,16 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): ns = Namespace() ns.called = False + original_open = open def call_file(name, *args): - ns.called = True - self.failUnlessEqual(name, abspath_expanduser_unicode(exclude_file)) - return StringIO() + if name.endswith("excludes.dummy"): + ns.called = True + self.failUnlessEqual(name, abspath_expanduser_unicode(exclude_file)) + return StringIO() + else: + return original_open(name, *args) - patcher = MonkeyPatcher((__builtin__, 'file', call_file)) + patcher = MonkeyPatcher((builtins, 'open', call_file)) patcher.runWithPatches(parse_options, basedir, "backup", ['--exclude-from', unicode_to_argv(exclude_file), 'from', 'to']) self.failUnless(ns.called) From 06c4ed13b7d91090ce9ecb0a253a7d84c4fb3eae Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Apr 2021 09:28:51 -0400 Subject: [PATCH 139/463] Some progress towards passing Python 3 tests. --- src/allmydata/scripts/common.py | 20 +++++++++++++++----- src/allmydata/scripts/tahoe_backup.py | 8 +++++--- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 0381b7db8..016eaacb3 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -1,7 +1,7 @@ # coding: utf-8 from __future__ import print_function -from six import ensure_binary +from six import ensure_str import os, sys, textwrap import codecs @@ -276,17 +276,27 @@ def get_alias(aliases, path_unicode, default): return uri.from_string_dirnode(aliases[alias]).to_string(), path[colon+1:] def escape_path(path): - # type: (Union[str,bytes]) -> bytes + # type: (Union[str,bytes]) -> str u""" Return path quoted to US-ASCII, valid URL characters. >>> path = u'/føö/bar/☃' >>> escaped = escape_path(path) >>> escaped - b'/f%C3%B8%C3%B6/bar/%E2%98%83' + u'/f%C3%B8%C3%B6/bar/%E2%98%83' """ if isinstance(path, str): path = path.encode("utf-8") segments = path.split(b"/") - return b"/".join([urllib.parse.quote(s).encode("ascii") for s in segments]) - + result = str( + b"/".join([ + urllib.parse.quote(s).encode("ascii") for s in segments + ]), + "ascii" + ) + # Eventually (i.e. as part of Python 3 port) we want this to always return + # Unicode strings. However, to reduce diff sizes in the short term it'll + # return native string (i.e. bytes) on Python 2. + if PY2: + result = result.encode("ascii").__native__() + return result diff --git a/src/allmydata/scripts/tahoe_backup.py b/src/allmydata/scripts/tahoe_backup.py index 99f5663ea..99e36d68f 100644 --- a/src/allmydata/scripts/tahoe_backup.py +++ b/src/allmydata/scripts/tahoe_backup.py @@ -1,5 +1,7 @@ from __future__ import print_function +from past.builtins import unicode + import os.path import time from urllib.parse import quote as url_quote @@ -345,7 +347,7 @@ class FileTarget(object): target = PermissionDeniedTarget(self._path, isdir=False) return target.backup(progress, upload_file, upload_directory) else: - assert isinstance(childcap, str) + assert isinstance(childcap, bytes) if created: return progress.created_file(self._path, childcap, metadata) return progress.reused_file(self._path, childcap, metadata) @@ -525,12 +527,12 @@ class BackupProgress(object): return self, { os.path.basename(create_path): create_value for (create_path, create_value) - in self._create_contents.iteritems() + in self._create_contents.items() if os.path.dirname(create_path) == dirpath }, { os.path.basename(compare_path): compare_value for (compare_path, compare_value) - in self._compare_contents.iteritems() + in self._compare_contents.items() if os.path.dirname(compare_path) == dirpath } From 069fcb91eb5cd52757689ec6c6ecc88daa3eb639 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Apr 2021 09:35:34 -0400 Subject: [PATCH 140/463] More progress towards passing Python 3 tests. --- src/allmydata/scripts/tahoe_backup.py | 6 +++--- src/allmydata/scripts/tahoe_ls.py | 10 +++++++--- src/allmydata/test/cli/test_backup.py | 4 ++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/allmydata/scripts/tahoe_backup.py b/src/allmydata/scripts/tahoe_backup.py index 99e36d68f..2f3bbeed1 100644 --- a/src/allmydata/scripts/tahoe_backup.py +++ b/src/allmydata/scripts/tahoe_backup.py @@ -5,12 +5,12 @@ from past.builtins import unicode import os.path import time from urllib.parse import quote as url_quote -import json import datetime + from allmydata.scripts.common import get_alias, escape_path, DEFAULT_ALIAS, \ UnknownAliasError from allmydata.scripts.common_http import do_http, HTTPError, format_http_error -from allmydata.util import time_format +from allmydata.util import time_format, jsonbytes as json from allmydata.scripts import backupdb from allmydata.util.encodingutil import listdir_unicode, quote_output, \ quote_local_unicode_path, to_bytes, FilenameEncodingError, unicode_to_url @@ -167,7 +167,7 @@ class BackerUpper(object): if must_create: self.verboseprint(" creating directory for %s" % quote_local_unicode_path(path)) newdircap = mkdir(create_contents, self.options) - assert isinstance(newdircap, str) + assert isinstance(newdircap, bytes) if r: r.did_create(newdircap) return True, newdircap diff --git a/src/allmydata/scripts/tahoe_ls.py b/src/allmydata/scripts/tahoe_ls.py index 2bfe16d27..650951dfe 100644 --- a/src/allmydata/scripts/tahoe_ls.py +++ b/src/allmydata/scripts/tahoe_ls.py @@ -1,6 +1,10 @@ from __future__ import print_function -import urllib, time +from past.builtins import unicode +from six import ensure_text + +import time +from urllib.parse import quote as url_quote import json from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError @@ -23,7 +27,7 @@ def list(options): except UnknownAliasError as e: e.display(stderr) return 1 - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + url = nodeurl + "uri/%s" % url_quote(rootcap) if path: # move where.endswith check here? url += "/" + escape_path(path) @@ -164,7 +168,7 @@ def list(options): while len(left_justifys) <= i: left_justifys.append(False) max_widths[i] = max(max_widths[i], len(cell)) - if cell.startswith("URI"): + if ensure_text(cell).startswith("URI"): left_justifys[i] = True if len(left_justifys) == 1: left_justifys[0] = True diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index 1bc8b281e..38c3a0d02 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -153,7 +153,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): lines = out.split("\n") children = dict([line.split() for line in lines if line]) latest_uri = children["Latest"] - self.failUnless(latest_uri.startswith("URI:DIR2-CHK:"), latest_uri) + self.failUnless(latest_uri.startswith(b"URI:DIR2-CHK:"), latest_uri) childnames = children.keys() self.failUnlessReallyEqual(sorted(childnames), ["Archives", "Latest"]) d.addCallback(_check1) @@ -387,7 +387,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): self._check_filtering(filtered, root_listdir, (u'_darcs', u'subdir'), (nice_doc, u'lib.a')) # read exclude patterns from file - exclusion_string = doc_pattern_arg + "\nlib.?" + exclusion_string = doc_pattern_arg + b"\nlib.?" excl_filepath = os.path.join(basedir, 'exclusion') fileutil.write(excl_filepath, exclusion_string) backup_options = parse(['--exclude-from', excl_filepath, 'from', 'to']) From 8512bdcd8d1d8cf2f12606d07cc0f31bf474ef53 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Apr 2021 09:51:53 -0400 Subject: [PATCH 141/463] Even more progress towards passing Python 3 tests. --- src/allmydata/scripts/tahoe_get.py | 4 ++-- src/allmydata/scripts/tahoe_ls.py | 6 +++--- src/allmydata/test/cli/test_backup.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/scripts/tahoe_get.py b/src/allmydata/scripts/tahoe_get.py index d90baf2c9..b934f54b8 100644 --- a/src/allmydata/scripts/tahoe_get.py +++ b/src/allmydata/scripts/tahoe_get.py @@ -1,6 +1,6 @@ from __future__ import print_function -import urllib +from urllib.parse import quote as url_quote from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError from allmydata.scripts.common_http import do_http, format_http_error @@ -20,7 +20,7 @@ def get(options): except UnknownAliasError as e: e.display(stderr) return 1 - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + url = nodeurl + "uri/%s" % url_quote(rootcap) if path: url += "/" + escape_path(path) diff --git a/src/allmydata/scripts/tahoe_ls.py b/src/allmydata/scripts/tahoe_ls.py index 650951dfe..91665e77b 100644 --- a/src/allmydata/scripts/tahoe_ls.py +++ b/src/allmydata/scripts/tahoe_ls.py @@ -1,7 +1,7 @@ from __future__ import print_function from past.builtins import unicode -from six import ensure_text +from six import ensure_text, ensure_str import time from urllib.parse import quote as url_quote @@ -153,9 +153,9 @@ def list(options): line.append(quote_output(name) + classify) if options["uri"]: - line.append(uri) + line.append(ensure_str(uri)) if options["readonly-uri"]: - line.append(quote_output(ro_uri or "-", quotemarks=False)) + line.append(quote_output(ensure_str(ro_uri) or "-", quotemarks=False)) rows.append((encoding_error, line)) diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index 38c3a0d02..18e26a860 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -153,7 +153,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): lines = out.split("\n") children = dict([line.split() for line in lines if line]) latest_uri = children["Latest"] - self.failUnless(latest_uri.startswith(b"URI:DIR2-CHK:"), latest_uri) + self.failUnless(latest_uri.startswith("URI:DIR2-CHK:"), latest_uri) childnames = children.keys() self.failUnlessReallyEqual(sorted(childnames), ["Archives", "Latest"]) d.addCallback(_check1) From e6532305bf25e44602fb4e6f20ac1a1e7479ef53 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Apr 2021 10:17:01 -0400 Subject: [PATCH 142/463] Match Python 3 behavior for stdout/stderr. --- src/allmydata/test/common_util.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 91f0b0f78..34d25e441 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -87,13 +87,18 @@ def run_cli_native(verb, *args, **kwargs): argv = nodeargs + [verb] + list(args) stdin = kwargs.get("stdin", "") if encoding is None: - # The original behavior, the Python 2 behavior, is to accept either - # bytes or unicode and try to automatically encode or decode as - # necessary. This works okay for ASCII and if LANG is set - # appropriately. These aren't great constraints so we should move - # away from this behavior. - stdout = StringIO() - stderr = StringIO() + if PY2: + # The original behavior, the Python 2 behavior, is to accept either + # bytes or unicode and try to automatically encode or decode as + # necessary. This works okay for ASCII and if LANG is set + # appropriately. These aren't great constraints so we should move + # away from this behavior. + stdout = StringIO() + stderr = StringIO() + else: + # Default on Python 3 is accepting text. + stdout = TextIOWrapper(BytesIO(), "utf-8") + stderr = TextIOWrapper(BytesIO(), "utf-8") else: # The new behavior, the Python 3 behavior, is to accept unicode and # encode it using a specific encoding. For older versions of Python From 2554112045b850764a728124b7cba966eca6aacc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Apr 2021 10:17:12 -0400 Subject: [PATCH 143/463] Handle Python 3 case where stdout by default is Unicode. --- src/allmydata/scripts/tahoe_get.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/scripts/tahoe_get.py b/src/allmydata/scripts/tahoe_get.py index b934f54b8..769a366a5 100644 --- a/src/allmydata/scripts/tahoe_get.py +++ b/src/allmydata/scripts/tahoe_get.py @@ -30,6 +30,10 @@ def get(options): outf = open(to_file, "wb") else: outf = stdout + # Make sure we can write bytes; on Python 3 stdout is Unicode by + # default. + if getattr(outf, "encoding", None) is not None: + outf = outf.buffer while True: data = resp.read(4096) if not data: From a0302c50dac6587393d9f4c0896e0e66e100dd65 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Apr 2021 10:25:15 -0400 Subject: [PATCH 144/463] Fix a BytesWarning. --- src/allmydata/scripts/create_node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index e03e9b918..4959ed391 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -449,12 +449,13 @@ def create_node(config): v = remote_config.get(k, None) if v is not None: # we're faking usually argv-supplied options :/ + v_orig = v if isinstance(v, str): v = v.encode(get_io_encoding()) config[k] = v if k not in sensitive_keys: if k not in ['shares-happy', 'shares-total', 'shares-needed']: - print(" {}: {}".format(k, v), file=out) + print(" {}: {}".format(k, v_orig), file=out) else: print(" {}: [sensitive data; see tahoe.cfg]".format(k), file=out) From c3d2a26ee5a9663736f8c5387d5d421c8ae008df Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Apr 2021 10:27:00 -0400 Subject: [PATCH 145/463] Remove unused imports. --- src/allmydata/scripts/common.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 016eaacb3..bc969de93 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -1,7 +1,6 @@ # coding: utf-8 from __future__ import print_function -from six import ensure_str import os, sys, textwrap import codecs @@ -28,7 +27,7 @@ else: from twisted.python import usage from allmydata.util.assertutil import precondition -from allmydata.util.encodingutil import unicode_to_url, quote_output, \ +from allmydata.util.encodingutil import quote_output, \ quote_local_unicode_path, argv_to_abspath from allmydata.scripts.default_nodedir import _default_nodedir From 43620a3c19060058c73856eb17c4e5ccb46e27e9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Apr 2021 12:53:17 -0400 Subject: [PATCH 146/463] Port to Python 3. --- src/allmydata/test/cli/test_backup.py | 49 ++++++++++++++++----------- src/allmydata/util/_python3.py | 1 + 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index 18e26a860..36874fd05 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -1,5 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + from future.utils import PY2 if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import __builtin__ as builtins else: import builtins @@ -91,7 +100,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): d.addCallback(lambda res: do_backup(True)) def _check0(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0) self.failUnlessReallyEqual(rc, 0) ( files_uploaded, @@ -148,40 +157,40 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("ls", "--uri", "tahoe:backups")) def _check1(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0) self.failUnlessReallyEqual(rc, 0) lines = out.split("\n") children = dict([line.split() for line in lines if line]) latest_uri = children["Latest"] self.failUnless(latest_uri.startswith("URI:DIR2-CHK:"), latest_uri) - childnames = children.keys() + childnames = list(children.keys()) self.failUnlessReallyEqual(sorted(childnames), ["Archives", "Latest"]) d.addCallback(_check1) d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Latest")) def _check2(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0) self.failUnlessReallyEqual(rc, 0) self.failUnlessReallyEqual(sorted(out.split()), ["empty", "parent"]) d.addCallback(_check2) d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Latest/empty")) def _check2a(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0) self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(out.strip(), "") + self.assertFalse(out.strip()) d.addCallback(_check2a) d.addCallback(lambda res: self.do_cli("get", "tahoe:backups/Latest/parent/subdir/foo.txt")) def _check3(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertFalse(err) self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(out, "foo") + self.assertEqual(out, "foo") d.addCallback(_check3) d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Archives")) def _check4(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertFalse(err) self.failUnlessReallyEqual(rc, 0) self.old_archives = out.split() self.failUnlessReallyEqual(len(self.old_archives), 1) @@ -194,7 +203,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): # second backup should reuse everything, if the backupdb is # available (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertFalse(err) self.failUnlessReallyEqual(rc, 0) fu, fr, fs, dc, dr, ds = self.count_output(out) # foo.txt, bar.txt, blah.txt @@ -226,7 +235,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): # the directories should have been changed, so we should # re-use all of them too. (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertFalse(err) self.failUnlessReallyEqual(rc, 0) fu, fr, fs, dc, dr, ds = self.count_output(out) fchecked, dchecked = self.count_output2(out) @@ -243,7 +252,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Archives")) def _check5(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertFalse(err) self.failUnlessReallyEqual(rc, 0) self.new_archives = out.split() self.failUnlessReallyEqual(len(self.new_archives), 3, out) @@ -270,7 +279,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): # second backup should reuse bar.txt (if backupdb is available), # and upload the rest. None of the directories can be reused. (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertFalse(err) self.failUnlessReallyEqual(rc, 0) fu, fr, fs, dc, dr, ds = self.count_output(out) # new foo.txt, surprise file, subfile, empty @@ -286,7 +295,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Archives")) def _check6(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertFalse(err) self.failUnlessReallyEqual(rc, 0) self.new_archives = out.split() self.failUnlessReallyEqual(len(self.new_archives), 4) @@ -296,17 +305,17 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("get", "tahoe:backups/Latest/parent/subdir/foo.txt")) def _check7(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertFalse(err) self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(out, "FOOF!") + self.assertEqual(out, "FOOF!") # the old snapshot should not be modified return self.do_cli("get", "tahoe:backups/Archives/%s/parent/subdir/foo.txt" % self.old_archives[0]) d.addCallback(_check7) def _check8(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertFalse(err) self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(out, "foo") + self.assertEqual(out, "foo") d.addCallback(_check8) return d @@ -593,7 +602,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0) d.addCallback(_check) return d @@ -609,6 +618,6 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0) d.addCallback(_check) return d diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 1feee8bdb..f2db424db 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -170,6 +170,7 @@ PORTED_MODULES = [ PORTED_TEST_MODULES = [ "allmydata.test.cli.test_alias", + "allmydata.test.cli.test_backup", "allmydata.test.cli.test_backupdb", "allmydata.test.cli.test_create", "allmydata.test.cli.test_invite", From b0e6c860859fa4bf986f878948b785d90afcb37a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 13 Apr 2021 09:21:26 -0400 Subject: [PATCH 147/463] Better error messages. --- src/allmydata/test/cli/test_backup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index 36874fd05..4bd4b8bfa 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -100,7 +100,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): d.addCallback(lambda res: do_backup(True)) def _check0(args): (rc, out, err) = args - self.assertEqual(len(err), 0) + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) ( files_uploaded, @@ -157,7 +157,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("ls", "--uri", "tahoe:backups")) def _check1(args): (rc, out, err) = args - self.assertEqual(len(err), 0) + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.split("\n") children = dict([line.split() for line in lines if line]) @@ -169,14 +169,14 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Latest")) def _check2(args): (rc, out, err) = args - self.assertEqual(len(err), 0) + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) self.failUnlessReallyEqual(sorted(out.split()), ["empty", "parent"]) d.addCallback(_check2) d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Latest/empty")) def _check2a(args): (rc, out, err) = args - self.assertEqual(len(err), 0) + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) self.assertFalse(out.strip()) d.addCallback(_check2a) From 6035a4a2aee7f6533afba811746cca94e9af4bb3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 13 Apr 2021 09:22:33 -0400 Subject: [PATCH 148/463] News file. --- newsfragments/3675.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3675.minor diff --git a/newsfragments/3675.minor b/newsfragments/3675.minor new file mode 100644 index 000000000..e69de29bb From cbd816fbd5909883c4af12b3803c66e47af1344a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 13 Apr 2021 09:34:55 -0400 Subject: [PATCH 149/463] Ensure warnings get turned into exceptions. Getting sneaking suspicion it's passing in filenames sometimes, not just modules. --- src/allmydata/test/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py index c75f8d003..e9c47bd69 100644 --- a/src/allmydata/test/__init__.py +++ b/src/allmydata/test/__init__.py @@ -36,7 +36,7 @@ from foolscap.logging.incident import IncidentQualifier if PY3: # Error on BytesWarnings, to catch things like str(b""), but only for # allmydata code. - warnings.filterwarnings("error", category=BytesWarning, module="allmydata.*") + warnings.filterwarnings("error", category=BytesWarning, module=".*allmydata.*") class NonQualifier(IncidentQualifier, object): From 2299c2dcc827e02c6649913ac2d210bb920d60dc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 13 Apr 2021 09:40:24 -0400 Subject: [PATCH 150/463] Fix implicit str(bytesobj). --- src/allmydata/test/test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index f6a7c2ee1..c80b5dc9c 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -515,7 +515,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): 0, "Expected error message from '{}', got something else: {}".format( description, - p.get_buffered_output(), + str(p.get_buffered_output(), "utf-8"), ), ) From 953c06a18d7d360b84ae1c45216a140efbd6a8ef Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 13 Apr 2021 09:53:08 -0400 Subject: [PATCH 151/463] Fix some plain-str()-of-bytes bugs. --- src/allmydata/test/test_download.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_download.py b/src/allmydata/test/test_download.py index 3a42b0819..d61942839 100644 --- a/src/allmydata/test/test_download.py +++ b/src/allmydata/test/test_download.py @@ -1304,7 +1304,7 @@ class MyShare(object): self._dyhb_rtt = rtt def __repr__(self): - return "sh%d-on-%s" % (self._shnum, self._server.get_name()) + return "sh%d-on-%s" % (self._shnum, str(self._server.get_name(), "ascii")) class MySegmentFetcher(SegmentFetcher): def __init__(self, *args, **kwargs): @@ -1383,7 +1383,7 @@ class Selection(unittest.TestCase): self.failUnless(node.failed) self.failUnless(node.failed.check(NotEnoughSharesError)) sname = serverA.get_name() - self.failUnlessIn("complete= pending=sh0-on-%s overdue= unused=" % sname, + self.failUnlessIn("complete= pending=sh0-on-%s overdue= unused=" % str(sname, "ascii"), str(node.failed)) d.addCallback(_check2) return d @@ -1605,7 +1605,7 @@ class Selection(unittest.TestCase): self.failUnless(node.failed) self.failUnless(node.failed.check(NotEnoughSharesError)) sname = servers[b"peer-2"].get_name() - self.failUnlessIn("complete=sh0 pending= overdue=sh2-on-%s unused=" % sname, + self.failUnlessIn("complete=sh0 pending= overdue=sh2-on-%s unused=" % str(sname, "ascii"), str(node.failed)) d.addCallback(_check4) return d From e386a1e705c50b5c84d009a366613e2bfa4ad0d0 Mon Sep 17 00:00:00 2001 From: May-Lee Sia Date: Tue, 13 Apr 2021 16:01:50 +0200 Subject: [PATCH 152/463] Add newsfragment description and edit details --- docs/ticket-triage.rst | 11 ++++++----- newsfragments/3651.minor | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/ticket-triage.rst b/docs/ticket-triage.rst index 86096849b..b92232507 100644 --- a/docs/ticket-triage.rst +++ b/docs/ticket-triage.rst @@ -11,16 +11,17 @@ milestones. Process ------- -- The role of Ticket Triager rotates weekly -- The Triager needs admin status on ``Trac`` -- The Triager looks at all the tickets that have been created in the last week +- The role of Ticket Triager rotates regularly-ish, and is assigned ad hoc +- The Triager needs a ``Trac`` account +- The Triager looks at all the tickets that have been created in the last week (or month, etc.) - They can use a custom query or do this as the week progresses - BONUS ROUND: Dig up a stale ticket from the past - Assign each ticket to a milestone on the Roadmap - The following situations merit discussion: - A ticket doesn't have an appropriate milestone and we should create one - A ticket, in vanishingly rare circumstances, should be deleted + - The ticket is spam + - The ticket contains sensitive information and harm will come to one or more people if it continues to be distributed - A ticket could be assigned to multiple milestones - There is another question about a ticket -- These tickets will be brought to a weekly meeting (currently Tuesdays) for discussion -- Ticket Triager role is also assigned at this meeting +- These tickets will be brought as necessary to one of our meetings (currently Tuesdays) for discussion diff --git a/newsfragments/3651.minor b/newsfragments/3651.minor index e69de29bb..9a2f5a0ed 100644 --- a/newsfragments/3651.minor +++ b/newsfragments/3651.minor @@ -0,0 +1 @@ +We added documentation detailing the project's ticket triage process From e242bf50c77d155970fcb488d690f9f4ae44302c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Apr 2021 09:55:21 -0400 Subject: [PATCH 153/463] Handle bytes in log messages. --- newsfragments/3626.minor | 0 src/allmydata/test/web/test_logs.py | 10 +++++++--- src/allmydata/web/logs.py | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 newsfragments/3626.minor diff --git a/newsfragments/3626.minor b/newsfragments/3626.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py index 5d697f910..d4fa5e944 100644 --- a/src/allmydata/test/web/test_logs.py +++ b/src/allmydata/test/web/test_logs.py @@ -92,7 +92,7 @@ class TestStreamingLogs(unittest.TestCase): @inlineCallbacks def test_one_log(self): """ - write a single Eliot log and see it streamed via websocket + Write a single Eliot log actin and see it streamed via websocket. """ proto = yield self.agent.open( @@ -106,14 +106,18 @@ class TestStreamingLogs(unittest.TestCase): proto.on("message", got_message) @log_call(action_type=u"test:cli:some-exciting-action") - def do_a_thing(): + def do_a_thing(arguments): pass - do_a_thing() + do_a_thing(arguments=[u"hello", b"good-day", 123, {"a": 35}, [None]]) proto.transport.loseConnection() yield proto.is_closed self.assertEqual(len(messages), 2) + self.assertEqual(messages[0]["action_type"], "test:cli:some-exciting-action") + self.assertEqual(messages[0]["arguments"], + ["hello", "good-day", 123, {"a": 35}, [None]]) + self.assertEqual(messages[1]["action_type"], "test:cli:some-exciting-action") self.assertEqual("started", messages[0]["action_status"]) self.assertEqual("succeeded", messages[1]["action_status"]) diff --git a/src/allmydata/web/logs.py b/src/allmydata/web/logs.py index a78e9cd12..9bd59ae53 100644 --- a/src/allmydata/web/logs.py +++ b/src/allmydata/web/logs.py @@ -8,8 +8,6 @@ from __future__ import ( division, ) -import json - from autobahn.twisted.resource import WebSocketResource from autobahn.twisted.websocket import ( WebSocketServerFactory, @@ -21,6 +19,8 @@ from twisted.web.resource import ( Resource, ) +from allmydata.util import jsonbytes as json + class TokenAuthenticatedWebSocketServerProtocol(WebSocketServerProtocol): """ From 1abf944dd21b8c5db052215b4497696ce2eeb592 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Apr 2021 10:38:57 -0400 Subject: [PATCH 154/463] News file. --- newsfragments/3672.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3672.minor diff --git a/newsfragments/3672.minor b/newsfragments/3672.minor new file mode 100644 index 000000000..e69de29bb From 32607b5ada8065f181c0e83937eaf247d9409256 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Apr 2021 10:42:01 -0400 Subject: [PATCH 155/463] For logging, using a new JSON bytes encoder that works on any bytes string, not just UTF-8-encoded strings. --- src/allmydata/test/__init__.py | 4 +- src/allmydata/test/eliotutil.py | 7 +-- src/allmydata/test/test_eliotutil.py | 4 +- src/allmydata/test/test_util.py | 37 +++++++++++-- src/allmydata/util/eliotutil.py | 6 +-- src/allmydata/util/jsonbytes.py | 78 ++++++++++++++++++++-------- src/allmydata/web/logs.py | 5 +- 7 files changed, 99 insertions(+), 42 deletions(-) diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py index c75f8d003..26b17e997 100644 --- a/src/allmydata/test/__init__.py +++ b/src/allmydata/test/__init__.py @@ -131,5 +131,5 @@ if sys.platform == "win32": initialize() from eliot import to_file -from allmydata.util.jsonbytes import BytesJSONEncoder -to_file(open("eliot.log", "wb"), encoder=BytesJSONEncoder) +from allmydata.util.jsonbytes import AnyBytesJSONEncoder +to_file(open("eliot.log", "wb"), encoder=AnyBytesJSONEncoder) diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py index c2359f132..35dfb09eb 100644 --- a/src/allmydata/test/eliotutil.py +++ b/src/allmydata/test/eliotutil.py @@ -54,7 +54,7 @@ from twisted.python.monkey import ( MonkeyPatcher, ) -from ..util.jsonbytes import BytesJSONEncoder +from ..util.jsonbytes import AnyBytesJSONEncoder _NAME = Field.for_types( @@ -73,10 +73,7 @@ RUN_TEST = ActionType( # On Python 3, we want to use our custom JSON encoder when validating messages # can be encoded to JSON: -if PY2: - _memory_logger = MemoryLogger -else: - _memory_logger = lambda: MemoryLogger(encoder=BytesJSONEncoder) +_memory_logger = lambda: MemoryLogger(encoder=AnyBytesJSONEncoder) @attr.s diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index aca677323..3f915ecd2 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -69,7 +69,7 @@ from ..util.eliotutil import ( _parse_destination_description, _EliotLogging, ) -from ..util.jsonbytes import BytesJSONEncoder +from ..util.jsonbytes import AnyBytesJSONEncoder from .common import ( SyncTestCase, @@ -109,7 +109,7 @@ class ParseDestinationDescriptionTests(SyncTestCase): reactor = object() self.assertThat( _parse_destination_description("file:-")(reactor), - Equals(FileDestination(stdout, encoder=BytesJSONEncoder)), + Equals(FileDestination(stdout, encoder=AnyBytesJSONEncoder)), ) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 9887897cf..8f3a39670 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -495,10 +495,10 @@ class YAML(unittest.TestCase): class JSONBytes(unittest.TestCase): - """Tests for BytesJSONEncoder.""" + """Tests for jsonbytes module.""" def test_encode_bytes(self): - """BytesJSONEncoder can encode bytes. + """jsonbytes.dumps() encodes bytes. Bytes are presumed to be UTF-8 encoded. """ @@ -515,7 +515,7 @@ class JSONBytes(unittest.TestCase): self.assertEqual(jsonbytes.loads(encoded), expected) def test_encode_unicode(self): - """BytesJSONEncoder encodes Unicode string as usual.""" + """jsonbytes.dumps() encodes Unicode string as usual.""" expected = { u"hello": [1, u"cd"], } @@ -529,6 +529,37 @@ class JSONBytes(unittest.TestCase): self.assertIsInstance(encoded, bytes) self.assertEqual(json.loads(encoded, encoding="utf-8"), x) + def test_any_bytes_unsupported_by_default(self): + """By default non-UTF-8 bytes raise error.""" + bytestring = b"abc\xff\x00" + with self.assertRaises(UnicodeDecodeError): + jsonbytes.dumps(bytestring) + with self.assertRaises(UnicodeDecodeError): + jsonbytes.dumps_bytes(bytestring) + with self.assertRaises(UnicodeDecodeError): + json.dumps(bytestring, cls=jsonbytes.UTF8BytesJSONEncoder) + + def test_any_bytes(self): + """If any_bytes is True, non-UTF-8 bytes don't break encoding.""" + bytestring = b"abc\xff" + o = {bytestring: bytestring} + expected = {"abc\\xff": "abc\\xff"} + self.assertEqual( + json.loads(jsonbytes.dumps(o, any_bytes=True)), + expected, + ) + self.assertEqual( + json.loads(json.dumps( + o, cls=jsonbytes.AnyBytesJSONEncoder)), + expected, + ) + self.assertEqual( + json.loads(jsonbytes.dumps(o, any_bytes=True), + encoding="utf-8"), + expected, + ) + + class FakeGetVersion(object): """Emulate an object with a get_version.""" diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index 5d144eb1d..4e48fbb9f 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -87,7 +87,7 @@ from twisted.internet.defer import ( ) from twisted.application.service import Service -from .jsonbytes import BytesJSONEncoder +from .jsonbytes import AnyBytesJSONEncoder def validateInstanceOf(t): @@ -306,7 +306,7 @@ class _DestinationParser(object): rotateLength=rotate_length, maxRotatedFiles=max_rotated_files, ) - return lambda reactor: FileDestination(get_file(), BytesJSONEncoder) + return lambda reactor: FileDestination(get_file(), AnyBytesJSONEncoder) _parse_destination_description = _DestinationParser().parse @@ -333,4 +333,4 @@ def log_call_deferred(action_type): if PY2: capture_logging = eliot_capture_logging else: - capture_logging = partial(eliot_capture_logging, encoder_=BytesJSONEncoder) + capture_logging = partial(eliot_capture_logging, encoder_=AnyBytesJSONEncoder) diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index c46a932d0..849fd6f0a 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -16,43 +16,75 @@ if PY2: import json -def _bytes_to_unicode(obj): - """Convert any bytes objects to unicode, recursively.""" - if isinstance(obj, bytes): - return obj.decode("utf-8") - if isinstance(obj, dict): - new_obj = {} - for k, v in obj.items(): - if isinstance(k, bytes): - k = k.decode("utf-8") - v = _bytes_to_unicode(v) - new_obj[k] = v - return new_obj - if isinstance(obj, (list, set, tuple)): - return [_bytes_to_unicode(i) for i in obj] - return obj +def _make_bytes_to_unicode(any_bytes): + """Create a function that recursively converts bytes to unicode. - -class BytesJSONEncoder(json.JSONEncoder): + :param any_bytes: If True, also support non-UTF-8-encoded bytes. """ - A JSON encoder than can also encode bytes. + errors = "backslashreplace" if any_bytes else "strict" - The bytes are assumed to be UTF-8 encoded Unicode strings. + def _bytes_to_unicode(obj): + """Convert any bytes objects to unicode, recursively.""" + if isinstance(obj, bytes): + return obj.decode("utf-8", errors=errors) + if isinstance(obj, dict): + new_obj = {} + for k, v in obj.items(): + if isinstance(k, bytes): + k = k.decode("utf-8", errors=errors) + v = _bytes_to_unicode(v) + new_obj[k] = v + return new_obj + if isinstance(obj, (list, set, tuple)): + return [_bytes_to_unicode(i) for i in obj] + return obj + + return _bytes_to_unicode + + +class UTF8BytesJSONEncoder(json.JSONEncoder): + """ + A JSON encoder than can also encode UTF-8 encoded strings. """ def iterencode(self, o, **kwargs): - return json.JSONEncoder.iterencode(self, _bytes_to_unicode(o), **kwargs) + return json.JSONEncoder.iterencode( + self, _make_bytes_to_unicode(False)(o), **kwargs) + + +class AnyBytesJSONEncoder(json.JSONEncoder): + """ + A JSON encoder than can also encode bytes of any sort. + + Bytes are decoded to strings using UTF-8, if that fails to decode then the + bytes are quoted. + """ + def iterencode(self, o, **kwargs): + return json.JSONEncoder.iterencode( + self, _make_bytes_to_unicode(True)(o), **kwargs) def dumps(obj, *args, **kwargs): """Encode to JSON, supporting bytes as keys or values. - The bytes are assumed to be UTF-8 encoded Unicode strings. + :param bool any_bytes: If False (the default) the bytes are assumed to be + UTF-8 encoded Unicode strings. If True, non-UTF-8 bytes are quoted for + human consumption. """ - return json.dumps(obj, cls=BytesJSONEncoder, *args, **kwargs) + any_bytes = kwargs.pop("any_bytes", False) + if any_bytes: + cls = AnyBytesJSONEncoder + else: + cls = UTF8BytesJSONEncoder + return json.dumps(obj, cls=cls, *args, **kwargs) def dumps_bytes(obj, *args, **kwargs): - """Encode to JSON, then encode as bytes.""" + """Encode to JSON, then encode as bytes. + + :param bool all_bytes: If False (the default) the bytes are assumed to be + UTF-8 encoded Unicode strings. If True, non-UTF-8 bytes are quoted for + human consumption. + """ result = dumps(obj, *args, **kwargs) if PY3: result = result.encode("utf-8") diff --git a/src/allmydata/web/logs.py b/src/allmydata/web/logs.py index 9bd59ae53..a79440eb9 100644 --- a/src/allmydata/web/logs.py +++ b/src/allmydata/web/logs.py @@ -47,10 +47,7 @@ class TokenAuthenticatedWebSocketServerProtocol(WebSocketServerProtocol): """ # probably want a try/except around here? what do we do if # transmission fails or anything else bad happens? - encoded = json.dumps(message) - if isinstance(encoded, str): - # On Python 3 dumps() returns Unicode... - encoded = encoded.encode("utf-8") + encoded = json.dumps_bytes(message, any_bytes=True) self.sendMessage(encoded) def onOpen(self): From d60bc2841abe03ff459f9c9c4687da5da2c2ff9d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Apr 2021 11:19:04 -0400 Subject: [PATCH 156/463] Oh right, Python 2 Eliot doesn't support custom JSON encoders. --- src/allmydata/test/eliotutil.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py index 35dfb09eb..1685744fd 100644 --- a/src/allmydata/test/eliotutil.py +++ b/src/allmydata/test/eliotutil.py @@ -73,7 +73,10 @@ RUN_TEST = ActionType( # On Python 3, we want to use our custom JSON encoder when validating messages # can be encoded to JSON: -_memory_logger = lambda: MemoryLogger(encoder=AnyBytesJSONEncoder) +if PY2: + _memory_logger = MemoryLogger +else: + _memory_logger = lambda: MemoryLogger(encoder=AnyBytesJSONEncoder) @attr.s From 51ebbae15a0a97c0784d7b4863d25d660bc04e89 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 16 Apr 2021 11:21:47 -0400 Subject: [PATCH 157/463] Fix typo. --- src/allmydata/test/web/test_logs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py index d4fa5e944..579e51dbc 100644 --- a/src/allmydata/test/web/test_logs.py +++ b/src/allmydata/test/web/test_logs.py @@ -92,7 +92,7 @@ class TestStreamingLogs(unittest.TestCase): @inlineCallbacks def test_one_log(self): """ - Write a single Eliot log actin and see it streamed via websocket. + Write a single Eliot log action and see it streamed via websocket. """ proto = yield self.agent.open( From bc9e4ac72859c7e79ccce3c18158f6721ee5a269 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 16 Apr 2021 11:36:53 -0400 Subject: [PATCH 158/463] Support quoting any-old-bytes correctly on Python 2. --- src/allmydata/test/test_util.py | 4 ++-- src/allmydata/util/jsonbytes.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 8f3a39670..4c2e98683 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -541,9 +541,9 @@ class JSONBytes(unittest.TestCase): def test_any_bytes(self): """If any_bytes is True, non-UTF-8 bytes don't break encoding.""" - bytestring = b"abc\xff" + bytestring = b"abc\xff\xff123" o = {bytestring: bytestring} - expected = {"abc\\xff": "abc\\xff"} + expected = {"abc\\xff\\xff123": "abc\\xff\\xff123"} self.assertEqual( json.loads(jsonbytes.dumps(o, any_bytes=True)), expected, diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index 849fd6f0a..995165ee6 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -14,7 +14,18 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import json +import codecs +if PY2: + def backslashreplace_py2(ex): + """ + On Python 2 'backslashreplace' error handler doesn't work, so write our + own. + """ + return ''.join('\\x{:02x}'.format(ord(c)) + for c in ex.object[ex.start:ex.end]), ex.end + + codecs.register_error("backslashreplace_tahoe_py2", backslashreplace_py2) def _make_bytes_to_unicode(any_bytes): """Create a function that recursively converts bytes to unicode. @@ -22,6 +33,8 @@ def _make_bytes_to_unicode(any_bytes): :param any_bytes: If True, also support non-UTF-8-encoded bytes. """ errors = "backslashreplace" if any_bytes else "strict" + if PY2 and errors == "backslashreplace": + errors = "backslashreplace_tahoe_py2" def _bytes_to_unicode(obj): """Convert any bytes objects to unicode, recursively.""" From 61506f87bb1f46f46eac30577be5df9413526883 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 16 Apr 2021 11:55:20 -0400 Subject: [PATCH 159/463] Make BytesWarning->exception global, to ease use in integration tests. --- src/allmydata/__init__.py | 17 ++++++++++++++++- src/allmydata/test/__init__.py | 6 ------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index b29868c05..333394fc5 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -8,7 +8,7 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future.utils import PY2 +from future.utils import PY2, PY3 if PY2: # Don't import future str() so we don't break Foolscap serialization on Python 2. from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 @@ -62,3 +62,18 @@ standard_library.install_aliases() from ._monkeypatch import patch patch() del patch + + +# On Python 3, turn BytesWarnings into exceptions. This can have potential +# production impact... if BytesWarnings are actually present in the codebase. +# Given that this has been enabled before Python 3 Tahoe-LAFS was publicly +# released, no such code should exist, and this will ensure it doesn't get +# added either. +# +# Also note that BytesWarnings only happen if Python is run with -b option, so +# in practice this should only affect tests. +if PY3: + import warnings + # Error on BytesWarnings, to catch things like str(b""), but only for + # allmydata code. + warnings.filterwarnings("error", category=BytesWarning, module=".*allmydata.*") diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py index e9c47bd69..45536a6c6 100644 --- a/src/allmydata/test/__init__.py +++ b/src/allmydata/test/__init__.py @@ -24,7 +24,6 @@ from future.utils import PY2, PY3 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -import warnings from traceback import extract_stack, format_list from foolscap.pb import Listener @@ -33,11 +32,6 @@ from twisted.application import service from foolscap.logging.incident import IncidentQualifier -if PY3: - # Error on BytesWarnings, to catch things like str(b""), but only for - # allmydata code. - warnings.filterwarnings("error", category=BytesWarning, module=".*allmydata.*") - class NonQualifier(IncidentQualifier, object): def check_event(self, ev): From fa46efdb3aae72df3f0ecc2987d42b70c92927c7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 16 Apr 2021 11:58:37 -0400 Subject: [PATCH 160/463] Enable BytesWarnings in integration tests. --- integration/test_servers_of_happiness.py | 2 +- integration/test_tor.py | 6 +++--- integration/util.py | 4 ++-- src/allmydata/test/cli_node_api.py | 1 + src/allmydata/test/test_runner.py | 2 +- src/allmydata/test/test_system.py | 2 +- 6 files changed, 9 insertions(+), 8 deletions(-) diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index 97392bf00..1f350eb8e 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -30,7 +30,7 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto proto, sys.executable, [ - sys.executable, '-m', 'allmydata.scripts.runner', + sys.executable, '-b', '-m', 'allmydata.scripts.runner', '-d', node_dir, 'put', __file__, ] diff --git a/integration/test_tor.py b/integration/test_tor.py index dcbfb1151..3b374f669 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -46,7 +46,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne proto, sys.executable, ( - sys.executable, '-m', 'allmydata.scripts.runner', + sys.executable, '-b', '-m', 'allmydata.scripts.runner', '-d', join(temp_dir, 'carol'), 'put', gold_path, ) @@ -60,7 +60,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne proto, sys.executable, ( - sys.executable, '-m', 'allmydata.scripts.runner', + sys.executable, '-b', '-m', 'allmydata.scripts.runner', '-d', join(temp_dir, 'dave'), 'get', cap, ) @@ -84,7 +84,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ proto, sys.executable, ( - sys.executable, '-m', 'allmydata.scripts.runner', + sys.executable, '-b', '-m', 'allmydata.scripts.runner', 'create-node', '--nickname', name, '--introducer', introducer_furl, diff --git a/integration/util.py b/integration/util.py index 256fd68c1..b72e11c72 100644 --- a/integration/util.py +++ b/integration/util.py @@ -152,9 +152,9 @@ def _tahoe_runner_optional_coverage(proto, reactor, request, other_args): `--coverage` option if the `request` indicates we should. """ if request.config.getoption('coverage'): - args = [sys.executable, '-m', 'coverage', 'run', '-m', 'allmydata.scripts.runner', '--coverage'] + args = [sys.executable, '-b', '-m', 'coverage', 'run', '-m', 'allmydata.scripts.runner', '--coverage'] else: - args = [sys.executable, '-m', 'allmydata.scripts.runner'] + args = [sys.executable, '-b', '-m', 'allmydata.scripts.runner'] args += other_args return reactor.spawnProcess( proto, diff --git a/src/allmydata/test/cli_node_api.py b/src/allmydata/test/cli_node_api.py index 4e4173924..be0381e11 100644 --- a/src/allmydata/test/cli_node_api.py +++ b/src/allmydata/test/cli_node_api.py @@ -154,6 +154,7 @@ class CLINodeAPI(object): exe = sys.executable argv = [ exe, + "-b", u"-m", u"allmydata.scripts.runner", ] + argv diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index c80b5dc9c..7cc89c287 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -88,7 +88,7 @@ def run_bintahoe(extra_argv, python_options=None): argv = [executable] if python_options is not None: argv.extend(python_options) - argv.extend([u"-m", u"allmydata.scripts.runner"]) + argv.extend([u"-b", u"-m", u"allmydata.scripts.runner"]) argv.extend(extra_argv) argv = list(unicode_to_argv(arg) for arg in argv) p = Popen(argv, stdout=PIPE, stderr=PIPE) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 040104b4c..0ff1e06e9 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -76,7 +76,7 @@ class RunBinTahoeMixin(object): # support env yet and is also synchronous. If we could get rid of # this in favor of that, though, it would probably be an improvement. command = sys.executable - argv = python_options + ["-m", "allmydata.scripts.runner"] + args + argv = python_options + ["-b", "-m", "allmydata.scripts.runner"] + args if env is None: env = os.environ From 08772c5a86b3f3b360e17891e3d3bf887bb68462 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 16 Apr 2021 11:58:55 -0400 Subject: [PATCH 161/463] News file. --- newsfragments/3619.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3619.minor diff --git a/newsfragments/3619.minor b/newsfragments/3619.minor new file mode 100644 index 000000000..e69de29bb From abb247b3cc120568a9765c89d7f0ea61faea3072 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 16 Apr 2021 12:01:07 -0400 Subject: [PATCH 162/463] Fix flake. --- src/allmydata/test/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py index 45536a6c6..abf23a301 100644 --- a/src/allmydata/test/__init__.py +++ b/src/allmydata/test/__init__.py @@ -20,7 +20,7 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future.utils import PY2, PY3 +from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 From 1b46f981c6386c1d092d8752699f40eea96aa32d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 9 Apr 2021 13:43:43 -0400 Subject: [PATCH 163/463] Remove AccountURLChecker, the code that relies on it, associated tests, and docs. --- docs/frontends/FTP-and-SFTP.rst | 27 ---------------- newsfragments/3652.removed | 1 + src/allmydata/client.py | 4 +-- src/allmydata/frontends/auth.py | 55 -------------------------------- src/allmydata/frontends/sftpd.py | 11 +++---- 5 files changed, 6 insertions(+), 92 deletions(-) create mode 100644 newsfragments/3652.removed diff --git a/docs/frontends/FTP-and-SFTP.rst b/docs/frontends/FTP-and-SFTP.rst index ee6371812..4c87b0bc4 100644 --- a/docs/frontends/FTP-and-SFTP.rst +++ b/docs/frontends/FTP-and-SFTP.rst @@ -78,33 +78,6 @@ start with "ssh-". Now add an ``accounts.file`` directive to your ``tahoe.cfg`` file, as described in the next sections. -Running An Account Server (accounts.url) -======================================== - -The accounts.url directive allows access requests to be controlled by an -HTTP-based login service, useful for centralized deployments. This was used -by AllMyData to provide web-based file access, where the service used a -simple PHP script and database lookups to map an account email address and -password to a Tahoe-LAFS directory cap. The service will receive a -multipart/form-data POST, just like one created with a
and -fields, with three parameters: - -• action: "authenticate" (this is a static string) -• email: USERNAME (Tahoe-LAFS has no notion of email addresses, but the - authentication service uses them as account names, so the interface - presents this argument as "email" rather than "username"). -• passwd: PASSWORD - -It should return a single string that either contains a Tahoe-LAFS directory -cap (URI:DIR2:...), or "0" to indicate a login failure. - -Tahoe-LAFS recommends the service be secure, preferably localhost-only. This -makes it harder for attackers to brute force the password or use DNS -poisoning to cause the Tahoe-LAFS gateway to talk with the wrong server, -thereby revealing the usernames and passwords. - -Public key authentication is not supported when an account server is used. - Configuring SFTP Access ======================= diff --git a/newsfragments/3652.removed b/newsfragments/3652.removed new file mode 100644 index 000000000..a3e964702 --- /dev/null +++ b/newsfragments/3652.removed @@ -0,0 +1 @@ +Removed support for the Account Server frontend authentication type. diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 3bf976fe5..a6c45643f 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -116,7 +116,6 @@ _client_config = configutil.ValidConfiguration( ), "sftpd": ( "accounts.file", - "accounts.url", "enabled", "host_privkey_file", "host_pubkey_file", @@ -1042,13 +1041,12 @@ class _Client(node.Node, pollmixin.PollMixin): accountfile = self.config.get_config("sftpd", "accounts.file", None) if accountfile: accountfile = self.config.get_config_path(accountfile) - accounturl = self.config.get_config("sftpd", "accounts.url", None) sftp_portstr = self.config.get_config("sftpd", "port", "tcp:8022") pubkey_file = self.config.get_config("sftpd", "host_pubkey_file") privkey_file = self.config.get_config("sftpd", "host_privkey_file") from allmydata.frontends import sftpd - s = sftpd.SFTPServer(self, accountfile, accounturl, + s = sftpd.SFTPServer(self, accountfile, sftp_portstr, pubkey_file, privkey_file) s.setServiceParent(self) diff --git a/src/allmydata/frontends/auth.py b/src/allmydata/frontends/auth.py index 7f81572fe..f2ac99b8f 100644 --- a/src/allmydata/frontends/auth.py +++ b/src/allmydata/frontends/auth.py @@ -1,14 +1,10 @@ -import os - from zope.interface import implementer -from twisted.web.client import getPage from twisted.internet import defer from twisted.cred import error, checkers, credentials from twisted.conch.ssh import keys from twisted.conch.checkers import SSHPublicKeyChecker, InMemorySSHKeyDB from allmydata.util.dictutil import BytesKeyDict -from allmydata.util import base32 from allmydata.util.fileutil import abspath_expanduser_unicode @@ -86,54 +82,3 @@ class AccountFileChecker(object): d = defer.maybeDeferred(creds.checkPassword, correct) d.addCallback(self._cbPasswordMatch, str(creds.username)) return d - - -@implementer(checkers.ICredentialsChecker) -class AccountURLChecker(object): - credentialInterfaces = (credentials.IUsernamePassword,) - - def __init__(self, client, auth_url): - self.client = client - self.auth_url = auth_url - - def _cbPasswordMatch(self, rootcap, username): - return FTPAvatarID(username, rootcap) - - def post_form(self, username, password): - sepbase = base32.b2a(os.urandom(4)) - sep = "--" + sepbase - form = [] - form.append(sep) - fields = {"action": "authenticate", - "email": username, - "passwd": password, - } - for name, value in fields.iteritems(): - form.append('Content-Disposition: form-data; name="%s"' % name) - form.append('') - assert isinstance(value, str) - form.append(value) - form.append(sep) - form[-1] += "--" - body = "\r\n".join(form) + "\r\n" - headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase, - } - return getPage(self.auth_url, method="POST", - postdata=body, headers=headers, - followRedirect=True, timeout=30) - - def _parse_response(self, res): - rootcap = res.strip() - if rootcap == "0": - raise error.UnauthorizedLogin - return rootcap - - def requestAvatarId(self, credentials): - # construct a POST to the login form. While this could theoretically - # be done with something like the stdlib 'email' package, I can't - # figure out how, so we just slam together a form manually. - d = self.post_form(credentials.username, credentials.password) - d.addCallback(self._parse_response) - d.addCallback(self._cbPasswordMatch, str(credentials.username)) - return d - diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py index bc7196de6..17eca993e 100644 --- a/src/allmydata/frontends/sftpd.py +++ b/src/allmydata/frontends/sftpd.py @@ -1983,7 +1983,7 @@ class ShellSession(PrefixingLogMixin): components.registerAdapter(ShellSession, SFTPUserHandler, ISession) -from allmydata.frontends.auth import AccountURLChecker, AccountFileChecker, NeedRootcapLookupScheme +from allmydata.frontends.auth import AccountFileChecker, NeedRootcapLookupScheme @implementer(portal.IRealm) class Dispatcher(object): @@ -2000,7 +2000,7 @@ class Dispatcher(object): class SFTPServer(service.MultiService): name = "frontend:sftp" - def __init__(self, client, accountfile, accounturl, + def __init__(self, client, accountfile, sftp_portstr, pubkey_file, privkey_file): precondition(isinstance(accountfile, (str, type(None))), accountfile) precondition(isinstance(pubkey_file, str), pubkey_file) @@ -2013,12 +2013,9 @@ class SFTPServer(service.MultiService): if accountfile: c = AccountFileChecker(self, accountfile) p.registerChecker(c) - if accounturl: - c = AccountURLChecker(self, accounturl) - p.registerChecker(c) - if not accountfile and not accounturl: + if not accountfile: # we could leave this anonymous, with just the /uri/CAP form - raise NeedRootcapLookupScheme("must provide an account file or URL") + raise NeedRootcapLookupScheme("must provide an account file") pubkey = keys.Key.fromFile(pubkey_file.encode(get_filesystem_encoding())) privkey = keys.Key.fromFile(privkey_file.encode(get_filesystem_encoding())) From 72a370992fbc41837a1d88a11f3fae8c5d2ab6d1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 17 Apr 2021 18:24:22 -0400 Subject: [PATCH 164/463] Remove ToC entry for account server. --- docs/frontends/FTP-and-SFTP.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/frontends/FTP-and-SFTP.rst b/docs/frontends/FTP-and-SFTP.rst index 4c87b0bc4..9d4f1dcec 100644 --- a/docs/frontends/FTP-and-SFTP.rst +++ b/docs/frontends/FTP-and-SFTP.rst @@ -7,11 +7,10 @@ Tahoe-LAFS SFTP Frontend 1. `SFTP Background`_ 2. `Tahoe-LAFS Support`_ 3. `Creating an Account File`_ -4. `Running An Account Server (accounts.url)`_ -5. `Configuring SFTP Access`_ -6. `Dependencies`_ -7. `Immutable and Mutable Files`_ -8. `Known Issues`_ +4. `Configuring SFTP Access`_ +5. `Dependencies`_ +6. `Immutable and Mutable Files`_ +7. `Known Issues`_ SFTP Background From 6e8dde3b14b80f6bdccd6c6ba263d315d9eed7bc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Apr 2021 10:09:03 -0400 Subject: [PATCH 165/463] Simplify. --- src/allmydata/util/jsonbytes.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index 995165ee6..2d3cb7504 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -27,16 +27,18 @@ if PY2: codecs.register_error("backslashreplace_tahoe_py2", backslashreplace_py2) -def _make_bytes_to_unicode(any_bytes): + +def _bytes_to_unicode(any_bytes, obj): """Create a function that recursively converts bytes to unicode. :param any_bytes: If True, also support non-UTF-8-encoded bytes. + :param obj: Object to de-byte-ify. """ errors = "backslashreplace" if any_bytes else "strict" if PY2 and errors == "backslashreplace": errors = "backslashreplace_tahoe_py2" - def _bytes_to_unicode(obj): + def doit(obj): """Convert any bytes objects to unicode, recursively.""" if isinstance(obj, bytes): return obj.decode("utf-8", errors=errors) @@ -45,14 +47,14 @@ def _make_bytes_to_unicode(any_bytes): for k, v in obj.items(): if isinstance(k, bytes): k = k.decode("utf-8", errors=errors) - v = _bytes_to_unicode(v) + v = doit(v) new_obj[k] = v return new_obj if isinstance(obj, (list, set, tuple)): - return [_bytes_to_unicode(i) for i in obj] + return [doit(i) for i in obj] return obj - return _bytes_to_unicode + return doit(obj) class UTF8BytesJSONEncoder(json.JSONEncoder): @@ -61,7 +63,7 @@ class UTF8BytesJSONEncoder(json.JSONEncoder): """ def iterencode(self, o, **kwargs): return json.JSONEncoder.iterencode( - self, _make_bytes_to_unicode(False)(o), **kwargs) + self, _bytes_to_unicode(False, o), **kwargs) class AnyBytesJSONEncoder(json.JSONEncoder): @@ -73,7 +75,7 @@ class AnyBytesJSONEncoder(json.JSONEncoder): """ def iterencode(self, o, **kwargs): return json.JSONEncoder.iterencode( - self, _make_bytes_to_unicode(True)(o), **kwargs) + self, _bytes_to_unicode(True, o), **kwargs) def dumps(obj, *args, **kwargs): From 08cb514eeea1226fca1c642207e70b19d3adf39a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Apr 2021 10:09:26 -0400 Subject: [PATCH 166/463] Correct parameter name. --- src/allmydata/util/jsonbytes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index 2d3cb7504..152b79861 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -96,7 +96,7 @@ def dumps(obj, *args, **kwargs): def dumps_bytes(obj, *args, **kwargs): """Encode to JSON, then encode as bytes. - :param bool all_bytes: If False (the default) the bytes are assumed to be + :param bool any_bytes: If False (the default) the bytes are assumed to be UTF-8 encoded Unicode strings. If True, non-UTF-8 bytes are quoted for human consumption. """ From e090891935a13a4953ced575b60a5189c05808dc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Apr 2021 10:39:27 -0400 Subject: [PATCH 167/463] In PyPy encode() doesn't call iterencode(). --- src/allmydata/util/jsonbytes.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index 152b79861..f6143f4d1 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -61,6 +61,10 @@ class UTF8BytesJSONEncoder(json.JSONEncoder): """ A JSON encoder than can also encode UTF-8 encoded strings. """ + def encode(self, o, **kwargs): + return json.JSONEncoder.encode( + self, _bytes_to_unicode(False, o), **kwargs) + def iterencode(self, o, **kwargs): return json.JSONEncoder.iterencode( self, _bytes_to_unicode(False, o), **kwargs) @@ -73,6 +77,10 @@ class AnyBytesJSONEncoder(json.JSONEncoder): Bytes are decoded to strings using UTF-8, if that fails to decode then the bytes are quoted. """ + def encode(self, o, **kwargs): + return json.JSONEncoder.encode( + self, _bytes_to_unicode(True, o), **kwargs) + def iterencode(self, o, **kwargs): return json.JSONEncoder.iterencode( self, _bytes_to_unicode(True, o), **kwargs) From 83e16d40a452ac5766d07094243c497f37a3a624 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Apr 2021 11:18:36 -0400 Subject: [PATCH 168/463] Some tests passing on Python 3. --- src/allmydata/scripts/cli.py | 6 +++--- src/allmydata/scripts/tahoe_check.py | 8 ++++---- src/allmydata/test/cli/test_check.py | 16 ++++++++-------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 811ae7ef9..826c36e1f 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -224,7 +224,7 @@ class CpOptions(FileStoreOptions): def parseArgs(self, *args): if len(args) < 2: raise usage.UsageError("cp requires at least two arguments") - self.sources = map(argv_to_unicode, args[:-1]) + self.sources = list(map(argv_to_unicode, args[:-1])) self.destination = argv_to_unicode(args[-1]) synopsis = "[options] FROM.. TO" @@ -435,7 +435,7 @@ class CheckOptions(FileStoreOptions): ("add-lease", None, "Add/renew lease on all shares."), ] def parseArgs(self, *locations): - self.locations = map(argv_to_unicode, locations) + self.locations = list(map(argv_to_unicode, locations)) synopsis = "[options] [ALIAS:PATH]" description = """ @@ -452,7 +452,7 @@ class DeepCheckOptions(FileStoreOptions): ("verbose", "v", "Be noisy about what is happening."), ] def parseArgs(self, *locations): - self.locations = map(argv_to_unicode, locations) + self.locations = list(map(argv_to_unicode, locations)) synopsis = "[options] [ALIAS:PATH]" description = """ diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index cef9e32be..1700e2b77 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -1,6 +1,6 @@ from __future__ import print_function -import urllib +from urllib.parse import quote as url_quote import json # Python 2 compatibility @@ -36,7 +36,7 @@ def check_location(options, where): return 1 if path == '/': path = '' - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + url = nodeurl + "uri/%s" % url_quote(rootcap) if path: url += "/" + escape_path(path) # todo: should it end with a slash? @@ -139,7 +139,7 @@ class DeepCheckOutput(LineOnlyReceiver, object): if self.in_error: print(quote_output(line, quotemarks=False), file=self.stderr) return - if line.startswith("ERROR:"): + if line.startswith(b"ERROR:"): self.in_error = True self.streamer.rc = 1 print(quote_output(line, quotemarks=False), file=self.stderr) @@ -297,7 +297,7 @@ class DeepCheckStreamer(LineOnlyReceiver, object): return 1 if path == '/': path = '' - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + url = nodeurl + "uri/%s" % url_quote(rootcap) if path: url += "/" + escape_path(path) # todo: should it end with a slash? diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index 8cf963da6..eef8d3108 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -18,7 +18,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.basedir = "cli/Check/check" self.set_up_grid() c0 = self.g.clients[0] - DATA = "data" * 100 + DATA = b"data" * 100 DATA_uploadable = MutableData(DATA) d = c0.create_mutable_file(DATA_uploadable) def _stash_uri(n): @@ -45,7 +45,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(data["results"]["healthy"], True) d.addCallback(_check2) - d.addCallback(lambda ign: c0.upload(upload.Data("literal", convergence=""))) + d.addCallback(lambda ign: c0.upload(upload.Data(b"literal", convergence=b""))) def _stash_lit_uri(n): self.lit_uri = n.get_uri() d.addCallback(_stash_lit_uri) @@ -156,14 +156,14 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): c0 = self.g.clients[0] self.uris = {} self.fileurls = {} - DATA = "data" * 100 + DATA = b"data" * 100 quoted_good = quote_output(u"g\u00F6\u00F6d") d = c0.create_dirnode() def _stash_root_and_create_file(n): self.rootnode = n self.rooturi = n.get_uri() - return n.add_file(u"g\u00F6\u00F6d", upload.Data(DATA, convergence="")) + return n.add_file(u"g\u00F6\u00F6d", upload.Data(DATA, convergence=b"")) d.addCallback(_stash_root_and_create_file) def _stash_uri(fn, which): self.uris[which] = fn.get_uri() @@ -171,11 +171,11 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(_stash_uri, u"g\u00F6\u00F6d") d.addCallback(lambda ign: self.rootnode.add_file(u"small", - upload.Data("literal", - convergence=""))) + upload.Data(b"literal", + convergence=b""))) d.addCallback(_stash_uri, "small") d.addCallback(lambda ign: - c0.create_mutable_file(MutableData(DATA+"1"))) + c0.create_mutable_file(MutableData(DATA+b"1"))) d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn)) d.addCallback(_stash_uri, "mutable") @@ -322,7 +322,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.rootnode.create_subdirectory(u"subdir")) d.addCallback(_stash_uri, "subdir") d.addCallback(lambda fn: - fn.add_file(u"subfile", upload.Data(DATA+"2", ""))) + fn.add_file(u"subfile", upload.Data(DATA+b"2", b""))) d.addCallback(lambda ign: self.delete_shares_numbered(self.uris["subdir"], range(10))) From 5e59b9d8d6a059fd7f14129d7bf8785aa39a4fed Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Apr 2021 11:30:37 -0400 Subject: [PATCH 169/463] A little closer to passing tests on Python 3. --- src/allmydata/scripts/slow_operation.py | 4 ++-- src/allmydata/scripts/tahoe_check.py | 10 +++++++--- src/allmydata/test/cli/test_check.py | 15 ++++++++------- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py index ce25e9667..cce7d91c1 100644 --- a/src/allmydata/scripts/slow_operation.py +++ b/src/allmydata/scripts/slow_operation.py @@ -6,7 +6,7 @@ from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ from allmydata.scripts.common_http import do_http, format_http_error from allmydata.util import base32 from allmydata.util.encodingutil import quote_output, is_printable_ascii -import urllib +from urllib.parse import quote as url_quote import json class SlowOperationRunner(object): @@ -27,7 +27,7 @@ class SlowOperationRunner(object): return 1 if path == '/': path = '' - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + url = nodeurl + "uri/%s" % url_quote(rootcap) if path: url += "/" + escape_path(path) # todo: should it end with a slash? diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index 1700e2b77..08569ec5d 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -4,7 +4,7 @@ from urllib.parse import quote as url_quote import json # Python 2 compatibility -from future.utils import PY2 +from future.utils import PY2, PY3 if PY2: from future.builtins import str # noqa: F401 @@ -54,8 +54,12 @@ def check_location(options, where): return 1 jdata = resp.read() if options.get("raw"): - stdout.write(jdata) - stdout.write("\n") + if PY3: + stdoutb = stdout.buffer + else: + stdoutb = stdout + stdoutb.write(jdata) + stdoutb.write(b"\n") return 0 data = json.loads(jdata) diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index eef8d3108..e1a45faa2 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -1,3 +1,4 @@ +from past.builtins import unicode import os.path import json from twisted.trial import unittest @@ -41,7 +42,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) data = json.loads(out) - self.failUnlessReallyEqual(to_bytes(data["summary"]), "Healthy") + self.failUnlessReallyEqual(to_bytes(data["summary"]), b"Healthy") self.failUnlessReallyEqual(data["results"]["healthy"], True) d.addCallback(_check2) @@ -68,7 +69,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(data["results"]["healthy"], True) d.addCallback(_check_lit_raw) - d.addCallback(lambda ign: c0.create_immutable_dirnode({}, convergence="")) + d.addCallback(lambda ign: c0.create_immutable_dirnode({}, convergence=b"")) def _stash_lit_dir_uri(n): self.lit_dir_uri = n.get_uri() d.addCallback(_stash_lit_dir_uri) @@ -89,9 +90,9 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): cso.parseOptions([shares[1][2]]) storage_index = uri.from_string(self.uri).get_storage_index() self._corrupt_share_line = " server %s, SI %s, shnum %d" % \ - (base32.b2a(shares[1][1]), - base32.b2a(storage_index), - shares[1][0]) + (unicode(base32.b2a(shares[1][1]), "ascii"), + unicode(base32.b2a(storage_index), "ascii"), + shares[1][0]) debug.corrupt_share(cso) d.addCallback(_clobber_shares) @@ -418,8 +419,8 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 0) self.failUnlessReallyEqual(err, "") #Ensure healthy appears for each uri - self.failUnlessIn("Healthy", out[:len(out)/2]) - self.failUnlessIn("Healthy", out[len(out)/2:]) + self.failUnlessIn("Healthy", out[:len(out)//2]) + self.failUnlessIn("Healthy", out[len(out)//2:]) d.addCallback(_check) d.addCallback(lambda ign: self.do_cli("check", self.uriList[0], "nonexistent:")) From f6e0611b07c0e939fa96a27fa3d08a110b856a69 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Apr 2021 11:42:05 -0400 Subject: [PATCH 170/463] All tests pass on Python 3. --- src/allmydata/scripts/slow_operation.py | 3 ++- src/allmydata/scripts/tahoe_check.py | 9 +++++++-- src/allmydata/scripts/tahoe_manifest.py | 7 ++++--- src/allmydata/test/cli/test_check.py | 4 ++-- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py index cce7d91c1..f7366fcc0 100644 --- a/src/allmydata/scripts/slow_operation.py +++ b/src/allmydata/scripts/slow_operation.py @@ -1,4 +1,5 @@ from __future__ import print_function +from six import ensure_str import os, time from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ @@ -14,7 +15,7 @@ class SlowOperationRunner(object): def run(self, options): stderr = options.stderr self.options = options - self.ophandle = ophandle = base32.b2a(os.urandom(16)) + self.ophandle = ophandle = ensure_str(base32.b2a(os.urandom(16))) nodeurl = options['node-url'] if not nodeurl.endswith("/"): nodeurl += "/" diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index 08569ec5d..3859f0061 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -206,7 +206,7 @@ class DeepCheckAndRepairOutput(LineOnlyReceiver, object): if self.in_error: print(quote_output(line, quotemarks=False), file=self.stderr) return - if line.startswith("ERROR:"): + if line.startswith(b"ERROR:"): self.in_error = True self.streamer.rc = 1 print(quote_output(line, quotemarks=False), file=self.stderr) @@ -321,12 +321,17 @@ class DeepCheckStreamer(LineOnlyReceiver, object): return 1 # use Twisted to split this into lines + if PY3: + stdoutb = stdout.buffer + else: + stdoutb = stdout + while True: chunk = resp.read(100) if not chunk: break if self.options["raw"]: - stdout.write(chunk) + stdoutb.write(chunk) else: output.dataReceived(chunk) if not self.options["raw"]: diff --git a/src/allmydata/scripts/tahoe_manifest.py b/src/allmydata/scripts/tahoe_manifest.py index 386cdd1ad..6166564d3 100644 --- a/src/allmydata/scripts/tahoe_manifest.py +++ b/src/allmydata/scripts/tahoe_manifest.py @@ -1,6 +1,7 @@ from __future__ import print_function -import urllib, json +from urllib.parse import quote as url_quote +import json from twisted.protocols.basic import LineOnlyReceiver from allmydata.util.abbreviate import abbreviate_space_both from allmydata.scripts.slow_operation import SlowOperationRunner @@ -35,7 +36,7 @@ class ManifestStreamer(LineOnlyReceiver, object): return 1 if path == '/': path = '' - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + url = nodeurl + "uri/%s" % url_quote(rootcap) if path: url += "/" + escape_path(path) # todo: should it end with a slash? @@ -63,7 +64,7 @@ class ManifestStreamer(LineOnlyReceiver, object): if self.in_error: print(quote_output(line, quotemarks=False), file=stderr) return - if line.startswith("ERROR:"): + if line.startswith(b"ERROR:"): self.in_error = True self.rc = 1 print(quote_output(line, quotemarks=False), file=stderr) diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index e1a45faa2..756ed4f17 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -237,8 +237,8 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): cso.parseOptions([shares[1][2]]) storage_index = uri.from_string(self.uris["mutable"]).get_storage_index() self._corrupt_share_line = " corrupt: server %s, SI %s, shnum %d" % \ - (base32.b2a(shares[1][1]), - base32.b2a(storage_index), + (unicode(base32.b2a(shares[1][1]), "ascii"), + unicode(base32.b2a(storage_index), "ascii"), shares[1][0]) debug.corrupt_share(cso) d.addCallback(_clobber_shares) From 87f1620ab0a6f7bce4da9fe9c6bb8bd02274df53 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Apr 2021 11:42:23 -0400 Subject: [PATCH 171/463] News file. --- newsfragments/3678.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3678.minor diff --git a/newsfragments/3678.minor b/newsfragments/3678.minor new file mode 100644 index 000000000..e69de29bb From 5ebb385c10f801fd72dc2248e748c3dcb1a69c65 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Apr 2021 11:58:48 -0400 Subject: [PATCH 172/463] Port to Python 3. --- src/allmydata/test/cli/test_check.py | 57 ++++++++++++++++------------ src/allmydata/util/_python3.py | 1 + 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index 756ed4f17..fda8b4352 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -1,4 +1,12 @@ -from past.builtins import unicode +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os.path import json from twisted.trial import unittest @@ -13,6 +21,7 @@ from allmydata.scripts import debug from ..no_network import GridTestMixin from .common import CLITestMixin + class Check(GridTestMixin, CLITestMixin, unittest.TestCase): def test_check(self): @@ -29,7 +38,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", self.uri)) def _check1(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("Summary: Healthy" in lines, out) @@ -39,7 +48,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", "--raw", self.uri)) def _check2(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) data = json.loads(out) self.failUnlessReallyEqual(to_bytes(data["summary"]), b"Healthy") @@ -54,7 +63,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", self.lit_uri)) def _check_lit(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("Summary: Healthy (LIT)" in lines, out) @@ -63,7 +72,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", "--raw", self.lit_uri)) def _check_lit_raw(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) data = json.loads(out) self.failUnlessReallyEqual(data["results"]["healthy"], True) @@ -90,8 +99,8 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): cso.parseOptions([shares[1][2]]) storage_index = uri.from_string(self.uri).get_storage_index() self._corrupt_share_line = " server %s, SI %s, shnum %d" % \ - (unicode(base32.b2a(shares[1][1]), "ascii"), - unicode(base32.b2a(storage_index), "ascii"), + (str(base32.b2a(shares[1][1]), "ascii"), + str(base32.b2a(storage_index), "ascii"), shares[1][0]) debug.corrupt_share(cso) d.addCallback(_clobber_shares) @@ -99,7 +108,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", "--verify", self.uri)) def _check3(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() summary = [l for l in lines if l.startswith("Summary")][0] @@ -113,7 +122,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", "--verify", "--raw", self.uri)) def _check3_raw(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) data = json.loads(out) self.failUnlessReallyEqual(data["results"]["healthy"], False) @@ -127,7 +136,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.do_cli("check", "--verify", "--repair", self.uri)) def _check4(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("Summary: not healthy" in lines, out) @@ -141,7 +150,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.do_cli("check", "--verify", "--repair", self.uri)) def _check5(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("Summary: healthy" in lines, out) @@ -183,7 +192,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("deep-check", self.rooturi)) def _check1(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("done: 4 objects checked, 4 healthy, 0 unhealthy" @@ -199,7 +208,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.rooturi)) def _check2(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("'': Healthy" in lines, out) @@ -213,7 +222,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("stats", self.rooturi)) def _check_stats(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnlessIn(" count-immutable-files: 1", lines) @@ -237,8 +246,8 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): cso.parseOptions([shares[1][2]]) storage_index = uri.from_string(self.uris["mutable"]).get_storage_index() self._corrupt_share_line = " corrupt: server %s, SI %s, shnum %d" % \ - (unicode(base32.b2a(shares[1][1]), "ascii"), - unicode(base32.b2a(storage_index), "ascii"), + (str(base32.b2a(shares[1][1]), "ascii"), + str(base32.b2a(storage_index), "ascii"), shares[1][0]) debug.corrupt_share(cso) d.addCallback(_clobber_shares) @@ -252,7 +261,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.do_cli("deep-check", "--verbose", self.rooturi)) def _check3(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("'': Healthy" in lines, out) @@ -269,7 +278,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.rooturi)) def _check4(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("'': Healthy" in lines, out) @@ -288,7 +297,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.rooturi)) def _check5(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() units = [json.loads(line) for line in lines] @@ -302,7 +311,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.rooturi)) def _check6(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("'': healthy" in lines, out) @@ -326,7 +335,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): fn.add_file(u"subfile", upload.Data(DATA+b"2", b""))) d.addCallback(lambda ign: self.delete_shares_numbered(self.uris["subdir"], - range(10))) + list(range(10)))) # root # rootg\u00F6\u00F6d/ @@ -380,7 +389,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) d.addCallback(lambda ign: self.do_cli("deep-check")) d.addCallback(_check) @@ -397,7 +406,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) return d @@ -417,7 +426,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): def _check(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) #Ensure healthy appears for each uri self.failUnlessIn("Healthy", out[:len(out)//2]) self.failUnlessIn("Healthy", out[len(out)//2:]) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 3003d2909..26f6b8e2a 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -176,6 +176,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.cli.test_alias", "allmydata.test.cli.test_backup", "allmydata.test.cli.test_backupdb", + "allmydata.test.cli.test_check", "allmydata.test.cli.test_create", "allmydata.test.cli.test_invite", "allmydata.test.cli.test_status", From 567c0f019e0a20f3aeb16de46b15ac820c0d3afe Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 22 Apr 2021 09:39:50 -0400 Subject: [PATCH 173/463] Test random bytes. --- src/allmydata/test/web/test_logs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py index 579e51dbc..89ec7ba42 100644 --- a/src/allmydata/test/web/test_logs.py +++ b/src/allmydata/test/web/test_logs.py @@ -109,7 +109,7 @@ class TestStreamingLogs(unittest.TestCase): def do_a_thing(arguments): pass - do_a_thing(arguments=[u"hello", b"good-day", 123, {"a": 35}, [None]]) + do_a_thing(arguments=[u"hello", b"good-\xff-day", 123, {"a": 35}, [None]]) proto.transport.loseConnection() yield proto.is_closed @@ -117,7 +117,7 @@ class TestStreamingLogs(unittest.TestCase): self.assertEqual(len(messages), 2) self.assertEqual(messages[0]["action_type"], "test:cli:some-exciting-action") self.assertEqual(messages[0]["arguments"], - ["hello", "good-day", 123, {"a": 35}, [None]]) + ["hello", "good-\\xff-day", 123, {"a": 35}, [None]]) self.assertEqual(messages[1]["action_type"], "test:cli:some-exciting-action") self.assertEqual("started", messages[0]["action_status"]) self.assertEqual("succeeded", messages[1]["action_status"]) From 86fe350bef7a340d28f5947b289de6a2a927b2fc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 22 Apr 2021 10:15:43 -0400 Subject: [PATCH 174/463] Tests pass on Python 2. --- src/allmydata/test/cli/test_check.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index fda8b4352..320106be9 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -6,6 +6,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from six import ensure_text import os.path import json @@ -167,7 +168,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.uris = {} self.fileurls = {} DATA = b"data" * 100 - quoted_good = quote_output(u"g\u00F6\u00F6d") + quoted_good = u"'g\u00F6\u00F6d'" d = c0.create_dirnode() def _stash_root_and_create_file(n): @@ -210,6 +211,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) + out = ensure_text(out) lines = out.splitlines() self.failUnless("'': Healthy" in lines, out) self.failUnless("'small': Healthy (LIT)" in lines, out) @@ -263,6 +265,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) + out = ensure_text(out) lines = out.splitlines() self.failUnless("'': Healthy" in lines, out) self.failUnless("'small': Healthy (LIT)" in lines, out) @@ -280,6 +283,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) + out = ensure_text(out) lines = out.splitlines() self.failUnless("'': Healthy" in lines, out) self.failUnless("'small': Healthy (LIT)" in lines, out) @@ -313,6 +317,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) + out = ensure_text(out) lines = out.splitlines() self.failUnless("'': healthy" in lines, out) self.failUnless("'small': healthy" in lines, out) @@ -350,7 +355,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.failIfEqual(rc, 0) self.failUnlessIn("ERROR: UnrecoverableFileError", err) # the fatal directory should still show up, as the last line - self.failUnlessIn(" subdir\n", out) + self.failUnlessIn(" subdir\n", ensure_text(out)) d.addCallback(_manifest_failed) d.addCallback(lambda ign: self.do_cli("deep-check", self.rooturi)) From 416813578a99ab5dc4aa1cb43e00e6cf455f8ce8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 22 Apr 2021 10:18:59 -0400 Subject: [PATCH 175/463] Some progress towards passing tests on Python 3. --- src/allmydata/scripts/tahoe_cp.py | 32 ++++++++++++++++--------------- src/allmydata/test/cli/test_cp.py | 2 ++ 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/allmydata/scripts/tahoe_cp.py b/src/allmydata/scripts/tahoe_cp.py index f7879f35c..58492a7a1 100644 --- a/src/allmydata/scripts/tahoe_cp.py +++ b/src/allmydata/scripts/tahoe_cp.py @@ -1,7 +1,9 @@ from __future__ import print_function +from past.builtins import unicode + import os.path -import urllib +from urllib.parse import quote as url_quote import json from collections import defaultdict from six.moves import cStringIO as StringIO @@ -61,8 +63,8 @@ def mkdir(targeturl): def make_tahoe_subdirectory(nodeurl, parent_writecap, name): url = nodeurl + "/".join(["uri", - urllib.quote(parent_writecap), - urllib.quote(unicode_to_url(name)), + url_quote(parent_writecap), + url_quote(unicode_to_url(name)), ]) + "?t=mkdir" resp = do_http("POST", url) if resp.status in (200, 201): @@ -199,7 +201,7 @@ class TahoeFileSource(object): def open(self, caps_only): if caps_only: return StringIO(self.readcap) - url = self.nodeurl + "uri/" + urllib.quote(self.readcap) + url = self.nodeurl + "uri/" + url_quote(self.readcap) return GET_to_file(url) def bestcap(self): @@ -239,7 +241,7 @@ class TahoeDirectorySource(object): self.writecap = writecap self.readcap = readcap bestcap = writecap or readcap - url = self.nodeurl + "uri/%s" % urllib.quote(bestcap) + url = self.nodeurl + "uri/%s" % url_quote(bestcap) resp = do_http("GET", url + "?t=json") if resp.status != 200: raise HTTPError("Error examining source directory", resp) @@ -249,7 +251,7 @@ class TahoeDirectorySource(object): self.mutable = d.get("mutable", False) # older nodes don't provide it self.children_d = dict( [(unicode(name),value) for (name,value) - in d["children"].iteritems()] ) + in d["children"].items()] ) self.children = None def init_from_parsed(self, parsed): @@ -259,7 +261,7 @@ class TahoeDirectorySource(object): self.mutable = d.get("mutable", False) # older nodes don't provide it self.children_d = dict( [(unicode(name),value) for (name,value) - in d["children"].iteritems()] ) + in d["children"].items()] ) self.children = None def populate(self, recurse): @@ -329,14 +331,14 @@ class TahoeDirectoryTarget(object): self.mutable = d.get("mutable", False) # older nodes don't provide it self.children_d = dict( [(unicode(name),value) for (name,value) - in d["children"].iteritems()] ) + in d["children"].items()] ) self.children = None def init_from_grid(self, writecap, readcap): self.writecap = writecap self.readcap = readcap bestcap = writecap or readcap - url = self.nodeurl + "uri/%s" % urllib.quote(bestcap) + url = self.nodeurl + "uri/%s" % url_quote(bestcap) resp = do_http("GET", url + "?t=json") if resp.status != 200: raise HTTPError("Error examining target directory", resp) @@ -346,7 +348,7 @@ class TahoeDirectoryTarget(object): self.mutable = d.get("mutable", False) # older nodes don't provide it self.children_d = dict( [(unicode(name),value) for (name,value) - in d["children"].iteritems()] ) + in d["children"].items()] ) self.children = None def just_created(self, writecap): @@ -370,8 +372,8 @@ class TahoeDirectoryTarget(object): url = None if self.writecap: url = self.nodeurl + "/".join(["uri", - urllib.quote(self.writecap), - urllib.quote(unicode_to_url(name))]) + url_quote(self.writecap), + url_quote(unicode_to_url(name))]) self.children[name] = TahoeFileTarget(self.nodeurl, mutable, writecap, readcap, url) elif data[0] == "dirnode": @@ -439,7 +441,7 @@ class TahoeDirectoryTarget(object): def set_children(self): if not self.new_children: return - url = (self.nodeurl + "uri/" + urllib.quote(self.writecap) + url = (self.nodeurl + "uri/" + url_quote(self.writecap) + "?t=set_children") set_data = {} for (name, filecap) in self.new_children.items(): @@ -603,7 +605,7 @@ class Copier(object): t = LocalFileTarget(pathname) # non-empty else: # this is a tahoe object - url = self.nodeurl + "uri/%s" % urllib.quote(rootcap) + url = self.nodeurl + "uri/%s" % url_quote(rootcap) if path: url += "/" + escape_path(path) @@ -656,7 +658,7 @@ class Copier(object): t = LocalFileSource(pathname, name) # non-empty else: # this is a tahoe object - url = self.nodeurl + "uri/%s" % urllib.quote(rootcap) + url = self.nodeurl + "uri/%s" % url_quote(rootcap) name = None if path: if path.endswith("/"): diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py index 6cebec4a5..643a52bc6 100644 --- a/src/allmydata/test/cli/test_cp.py +++ b/src/allmydata/test/cli/test_cp.py @@ -1,5 +1,7 @@ from __future__ import print_function +from past.builtins import unicode + import os.path, json from twisted.trial import unittest from twisted.python import usage From 2aac69e0df288b4c1a55a6ef5ab62478075aa297 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 22 Apr 2021 10:20:26 -0400 Subject: [PATCH 176/463] More passing tests on Python 3. --- src/allmydata/scripts/tahoe_cp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/tahoe_cp.py b/src/allmydata/scripts/tahoe_cp.py index 58492a7a1..0a6c67942 100644 --- a/src/allmydata/scripts/tahoe_cp.py +++ b/src/allmydata/scripts/tahoe_cp.py @@ -4,7 +4,6 @@ from past.builtins import unicode import os.path from urllib.parse import quote as url_quote -import json from collections import defaultdict from six.moves import cStringIO as StringIO from twisted.python.failure import Failure @@ -17,6 +16,7 @@ from allmydata.util.fileutil import abspath_expanduser_unicode, precondition_abs from allmydata.util.encodingutil import unicode_to_url, listdir_unicode, quote_output, \ quote_local_unicode_path, to_bytes from allmydata.util.assertutil import precondition, _assert +from allmydata.util import jsonbytes as json class MissingSourceError(TahoeError): @@ -452,7 +452,7 @@ class TahoeDirectoryTarget(object): # TODO: think about how this affects forward-compatibility for # unknown caps set_data[name] = ["filenode", {"rw_uri": filecap}] - body = json.dumps(set_data) + body = json.dumps_bytes(set_data) POST(url, body) FileSources = (LocalFileSource, TahoeFileSource) From b675ca23800fa52f7cb8ad2192952a0565587a2d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 22 Apr 2021 10:23:58 -0400 Subject: [PATCH 177/463] Lint fix. --- src/allmydata/test/cli/test_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index 320106be9..e01dcc4cb 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -15,7 +15,7 @@ from six.moves import cStringIO as StringIO from allmydata import uri from allmydata.util import base32 -from allmydata.util.encodingutil import quote_output, to_bytes +from allmydata.util.encodingutil import to_bytes from allmydata.mutable.publish import MutableData from allmydata.immutable import upload from allmydata.scripts import debug From a393b54315e6f46bc82b2b3781125487b26215a2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 22 Apr 2021 10:27:59 -0400 Subject: [PATCH 178/463] Fix BytesWarning errors. --- src/allmydata/scripts/slow_operation.py | 3 +++ src/allmydata/scripts/tahoe_check.py | 2 ++ src/allmydata/scripts/tahoe_manifest.py | 3 +++ 3 files changed, 8 insertions(+) diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py index f7366fcc0..5d8c77538 100644 --- a/src/allmydata/scripts/slow_operation.py +++ b/src/allmydata/scripts/slow_operation.py @@ -1,4 +1,6 @@ from __future__ import print_function + +from past.builtins import unicode from six import ensure_str import os, time @@ -26,6 +28,7 @@ class SlowOperationRunner(object): except UnknownAliasError as e: e.display(stderr) return 1 + path = unicode(path, "utf-8") if path == '/': path = '' url = nodeurl + "uri/%s" % url_quote(rootcap) diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index 3859f0061..d8b7c9bce 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -34,6 +34,7 @@ def check_location(options, where): except UnknownAliasError as e: e.display(stderr) return 1 + path = str(path, "utf-8") if path == '/': path = '' url = nodeurl + "uri/%s" % url_quote(rootcap) @@ -299,6 +300,7 @@ class DeepCheckStreamer(LineOnlyReceiver, object): except UnknownAliasError as e: e.display(stderr) return 1 + path = str(path, "utf-8") if path == '/': path = '' url = nodeurl + "uri/%s" % url_quote(rootcap) diff --git a/src/allmydata/scripts/tahoe_manifest.py b/src/allmydata/scripts/tahoe_manifest.py index 6166564d3..b837e648a 100644 --- a/src/allmydata/scripts/tahoe_manifest.py +++ b/src/allmydata/scripts/tahoe_manifest.py @@ -1,5 +1,7 @@ from __future__ import print_function +from past.builtins import unicode + from urllib.parse import quote as url_quote import json from twisted.protocols.basic import LineOnlyReceiver @@ -34,6 +36,7 @@ class ManifestStreamer(LineOnlyReceiver, object): except UnknownAliasError as e: e.display(stderr) return 1 + path = unicode(path, "utf-8") if path == '/': path = '' url = nodeurl + "uri/%s" % url_quote(rootcap) From 56e4385103828bf8033077c5309192a704042bf8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 22 Apr 2021 10:36:10 -0400 Subject: [PATCH 179/463] More progress towards Python 3 tests passing. --- src/allmydata/scripts/tahoe_mkdir.py | 13 ++++++++----- src/allmydata/test/cli/test_cp.py | 19 +++++-------------- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/src/allmydata/scripts/tahoe_mkdir.py b/src/allmydata/scripts/tahoe_mkdir.py index a76adc8fc..54e8ebe46 100644 --- a/src/allmydata/scripts/tahoe_mkdir.py +++ b/src/allmydata/scripts/tahoe_mkdir.py @@ -1,6 +1,8 @@ from __future__ import print_function -import urllib +from past.builtins import unicode + +from urllib.parse import quote as url_quote from allmydata.scripts.common_http import do_http, check_http_error from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, UnknownAliasError from allmydata.util.encodingutil import quote_output @@ -24,7 +26,7 @@ def mkdir(options): # create a new unlinked directory url = nodeurl + "uri?t=mkdir" if options["format"]: - url += "&format=%s" % urllib.quote(options['format']) + url += "&format=%s" % url_quote(options['format']) resp = do_http("POST", url) rc = check_http_error(resp, stderr) if rc: @@ -35,13 +37,14 @@ def mkdir(options): return 0 # create a new directory at the given location + path = unicode(path, "utf-8") if path.endswith("/"): path = path[:-1] # path must be "/".join([s.encode("utf-8") for s in segments]) - url = nodeurl + "uri/%s/%s?t=mkdir" % (urllib.quote(rootcap), - urllib.quote(path)) + url = nodeurl + "uri/%s/%s?t=mkdir" % (url_quote(rootcap), + url_quote(path)) if options['format']: - url += "&format=%s" % urllib.quote(options['format']) + url += "&format=%s" % url_quote(options['format']) resp = do_http("POST", url) check_http_error(resp, stderr) diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py index 643a52bc6..fcd8c3bbf 100644 --- a/src/allmydata/test/cli/test_cp.py +++ b/src/allmydata/test/cli/test_cp.py @@ -26,12 +26,8 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): def test_unicode_filename(self): self.basedir = "cli/Cp/unicode_filename" - fn1 = os.path.join(unicode(self.basedir), u"\u00C4rtonwall") - try: - fn1_arg = fn1.encode(get_io_encoding()) - artonwall_arg = u"\u00C4rtonwall".encode(get_io_encoding()) - except UnicodeEncodeError: - raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.") + fn1 = os.path.join(self.basedir, u"\u00C4rtonwall") + artonwall_arg = u"\u00C4rtonwall" skip_if_cannot_represent_filename(fn1) @@ -46,7 +42,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): d = self.do_cli("create-alias", "tahoe") - d.addCallback(lambda res: self.do_cli("cp", fn1_arg, "tahoe:")) + d.addCallback(lambda res: self.do_cli("cp", fn1, "tahoe:")) d.addCallback(lambda res: self.do_cli("get", "tahoe:" + artonwall_arg)) d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA1)) @@ -202,13 +198,8 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): def test_unicode_dirnames(self): self.basedir = "cli/Cp/unicode_dirnames" - fn1 = os.path.join(unicode(self.basedir), u"\u00C4rtonwall") - try: - fn1_arg = fn1.encode(get_io_encoding()) - del fn1_arg # hush pyflakes - artonwall_arg = u"\u00C4rtonwall".encode(get_io_encoding()) - except UnicodeEncodeError: - raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.") + fn1 = os.path.join(self.basedir, u"\u00C4rtonwall") + artonwall_arg = u"\u00C4rtonwall" skip_if_cannot_represent_filename(fn1) From b85d735b8bbba53b12f198ea6fd99af60af9c263 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 22 Apr 2021 10:43:55 -0400 Subject: [PATCH 180/463] Even more progress towards Python 3 tests passing. --- src/allmydata/scripts/tahoe_put.py | 9 ++++++--- src/allmydata/test/cli/test_cp.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/allmydata/scripts/tahoe_put.py b/src/allmydata/scripts/tahoe_put.py index 8d87408dc..b64283a81 100644 --- a/src/allmydata/scripts/tahoe_put.py +++ b/src/allmydata/scripts/tahoe_put.py @@ -1,7 +1,9 @@ from __future__ import print_function +from past.builtins import unicode + from six.moves import cStringIO as StringIO -import urllib +from urllib.parse import quote as url_quote from allmydata.scripts.common_http import do_http, format_http_success, format_http_error from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ @@ -46,19 +48,20 @@ def put(options): # FIXME: don't hardcode cap format. if to_file.startswith("URI:MDMF:") or to_file.startswith("URI:SSK:"): - url = nodeurl + "uri/%s" % urllib.quote(to_file) + url = nodeurl + "uri/%s" % url_quote(to_file) else: try: rootcap, path = get_alias(aliases, to_file, DEFAULT_ALIAS) except UnknownAliasError as e: e.display(stderr) return 1 + path = unicode(path, "utf-8") if path.startswith("/"): suggestion = to_file.replace(u"/", u"", 1) print("Error: The remote filename must not start with a slash", file=stderr) print("Please try again, perhaps with %s" % quote_output(suggestion), file=stderr) return 1 - url = nodeurl + "uri/%s/" % urllib.quote(rootcap) + url = nodeurl + "uri/%s/" % url_quote(rootcap) if path: url += escape_path(path) else: diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py index fcd8c3bbf..e7c9d2323 100644 --- a/src/allmydata/test/cli/test_cp.py +++ b/src/allmydata/test/cli/test_cp.py @@ -1,5 +1,6 @@ from __future__ import print_function +from future.utils import PY2 from past.builtins import unicode import os.path, json @@ -64,7 +65,9 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessIn("files whose names could not be converted", err) else: self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(out.decode(get_io_encoding()), u"Metallica\n\u00C4rtonwall\n") + if PY2: + out = out.decode(get_io_encoding()) + self.failUnlessReallyEqual(out, u"Metallica\n\u00C4rtonwall\n") self.failUnlessReallyEqual(err, "") d.addCallback(_check) @@ -220,7 +223,9 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessIn("files whose names could not be converted", err) else: self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(out.decode(get_io_encoding()), u"\u00C4rtonwall\n") + if PY2: + out = out.decode(get_io_encoding()) + self.failUnlessReallyEqual(out, u"\u00C4rtonwall\n") self.failUnlessReallyEqual(err, "") d.addCallback(_check) @@ -259,6 +264,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): def _get_test_txt_uris(args): (rc, out, err) = args self.failUnlessEqual(rc, 0) + import pdb; pdb.set_trace() filetype, data = json.loads(out) self.failUnlessEqual(filetype, "filenode") From eb5211672c8752c07a66e6f3450b0f23b8918122 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 26 Apr 2021 09:46:21 -0400 Subject: [PATCH 181/463] Tests pass on Python 3. --- src/allmydata/scripts/tahoe_ls.py | 2 +- src/allmydata/scripts/tahoe_mv.py | 6 +++--- src/allmydata/test/cli/test_cp.py | 9 ++++----- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/allmydata/scripts/tahoe_ls.py b/src/allmydata/scripts/tahoe_ls.py index 91665e77b..71db3f5cb 100644 --- a/src/allmydata/scripts/tahoe_ls.py +++ b/src/allmydata/scripts/tahoe_ls.py @@ -45,10 +45,10 @@ def list(options): return resp.status data = resp.read() - if options['json']: # The webapi server should always output printable ASCII. if is_printable_ascii(data): + data = unicode(data, "ascii") print(data, file=stdout) return 0 else: diff --git a/src/allmydata/scripts/tahoe_mv.py b/src/allmydata/scripts/tahoe_mv.py index 7d13ea72a..bdaf134ff 100644 --- a/src/allmydata/scripts/tahoe_mv.py +++ b/src/allmydata/scripts/tahoe_mv.py @@ -1,7 +1,7 @@ from __future__ import print_function import re -import urllib +from urllib.parse import quote as url_quote import json from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError @@ -25,7 +25,7 @@ def mv(options, mode="move"): except UnknownAliasError as e: e.display(stderr) return 1 - from_url = nodeurl + "uri/%s" % urllib.quote(rootcap) + from_url = nodeurl + "uri/%s" % url_quote(rootcap) if from_path: from_url += "/" + escape_path(from_path) # figure out the source cap @@ -43,7 +43,7 @@ def mv(options, mode="move"): except UnknownAliasError as e: e.display(stderr) return 1 - to_url = nodeurl + "uri/%s" % urllib.quote(rootcap) + to_url = nodeurl + "uri/%s" % url_quote(rootcap) if path: to_url += "/" + escape_path(path) diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py index e7c9d2323..a63718cae 100644 --- a/src/allmydata/test/cli/test_cp.py +++ b/src/allmydata/test/cli/test_cp.py @@ -99,7 +99,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): fn1 = os.path.join(self.basedir, "Metallica") fn2 = os.path.join(outdir, "Not Metallica") fn3 = os.path.join(outdir, "test2") - DATA1 = "puppies" * 10000 + DATA1 = b"puppies" * 10000 fileutil.write(fn1, DATA1) d = self.do_cli("create-alias", "tahoe") @@ -264,7 +264,6 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): def _get_test_txt_uris(args): (rc, out, err) = args self.failUnlessEqual(rc, 0) - import pdb; pdb.set_trace() filetype, data = json.loads(out) self.failUnlessEqual(filetype, "filenode") @@ -817,9 +816,9 @@ cp -r $DIRCAP5 $DIRCAP6 to : E9-COLLIDING-TARGETS """ class CopyOut(GridTestMixin, CLITestMixin, unittest.TestCase): - FILE_CONTENTS = "file text" - FILE_CONTENTS_5 = "5" - FILE_CONTENTS_6 = "6" + FILE_CONTENTS = b"file text" + FILE_CONTENTS_5 = b"5" + FILE_CONTENTS_6 = b"6" def do_setup(self): # first we build a tahoe filesystem that contains: From 04a09558b60978d1a94fe5a6a9277cd1d9ee1dbf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 26 Apr 2021 09:54:35 -0400 Subject: [PATCH 182/463] Port to Python 3. --- src/allmydata/test/cli/test_cp.py | 22 +++++++++++++++------- src/allmydata/util/_python3.py | 1 + 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py index a63718cae..c46ba1f84 100644 --- a/src/allmydata/test/cli/test_cp.py +++ b/src/allmydata/test/cli/test_cp.py @@ -1,7 +1,15 @@ +""" +Ported to Python 3. +""" from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals from future.utils import PY2 -from past.builtins import unicode +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from six import ensure_text import os.path, json from twisted.trial import unittest @@ -46,12 +54,12 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("cp", fn1, "tahoe:")) d.addCallback(lambda res: self.do_cli("get", "tahoe:" + artonwall_arg)) - d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA1)) + d.addCallback(lambda rc_out_err: self.assertEqual(rc_out_err[1], DATA1)) d.addCallback(lambda res: self.do_cli("cp", fn2, "tahoe:")) d.addCallback(lambda res: self.do_cli("get", "tahoe:Metallica")) - d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA2)) + d.addCallback(lambda rc_out_err: self.assertEqual(rc_out_err[1], DATA2)) d.addCallback(lambda res: self.do_cli("ls", "tahoe:")) def _check(args): @@ -68,7 +76,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): if PY2: out = out.decode(get_io_encoding()) self.failUnlessReallyEqual(out, u"Metallica\n\u00C4rtonwall\n") - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) d.addCallback(_check) return d @@ -129,7 +137,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("when copying into a directory, all source files must have names, but", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_resp) # Create a directory, linked at tahoe:test . @@ -218,7 +226,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): unicode_to_output(u"\u00C4rtonwall") except UnicodeEncodeError: self.failUnlessReallyEqual(rc, 1) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) self.failUnlessIn(quote_output(u"\u00C4rtonwall"), err) self.failUnlessIn("files whose names could not be converted", err) else: @@ -226,7 +234,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): if PY2: out = out.decode(get_io_encoding()) self.failUnlessReallyEqual(out, u"\u00C4rtonwall\n") - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) d.addCallback(_check) return d diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 26f6b8e2a..471c7e913 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -177,6 +177,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.cli.test_backup", "allmydata.test.cli.test_backupdb", "allmydata.test.cli.test_check", + "allmydata.test.cli.test_cp", "allmydata.test.cli.test_create", "allmydata.test.cli.test_invite", "allmydata.test.cli.test_status", From f424e906ad3d7438ee540bd9ffc8e25c86bb5ae4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 26 Apr 2021 09:54:58 -0400 Subject: [PATCH 183/463] News file. --- newsfragments/3679.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3679.minor diff --git a/newsfragments/3679.minor b/newsfragments/3679.minor new file mode 100644 index 000000000..e69de29bb From 2ad8a4745519eac032cee4d2f46189cc3fa117b3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 26 Apr 2021 09:59:18 -0400 Subject: [PATCH 184/463] Fix flake. --- src/allmydata/test/cli/test_cp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py index c46ba1f84..d198a832c 100644 --- a/src/allmydata/test/cli/test_cp.py +++ b/src/allmydata/test/cli/test_cp.py @@ -9,7 +9,6 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from six import ensure_text import os.path, json from twisted.trial import unittest From b707a6ca7b33eafd6644eff7ef8c3824a4a1b87f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 26 Apr 2021 11:36:51 -0400 Subject: [PATCH 185/463] GitHub Actions: do not install vcpython27 Microsoft seems to have pulled the compiler download. --- .github/workflows/ci.yml | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c45ceaa63..6bec373f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,15 +27,6 @@ jobs: steps: - # Get vcpython27 on Windows + Python 2.7, to build netifaces - # extension. See https://chocolatey.org/packages/vcpython27 and - # https://github.com/crazy-max/ghaction-chocolatey - - name: Install MSVC 9.0 for Python 2.7 [Windows] - if: matrix.os == 'windows-latest' && matrix.python-version == '2.7' - uses: crazy-max/ghaction-chocolatey@v1 - with: - args: install vcpython27 - # See https://github.com/actions/checkout. A fetch-depth of 0 # fetches all tags and branches. - name: Check out Tahoe-LAFS sources @@ -164,15 +155,6 @@ jobs: steps: - # Get vcpython27 for Windows + Python 2.7, to build netifaces - # extension. See https://chocolatey.org/packages/vcpython27 and - # https://github.com/crazy-max/ghaction-chocolatey - - name: Install MSVC 9.0 for Python 2.7 [Windows] - if: matrix.os == 'windows-latest' && matrix.python-version == '2.7' - uses: crazy-max/ghaction-chocolatey@v1 - with: - args: install vcpython27 - - name: Install Tor [Ubuntu] if: matrix.os == 'ubuntu-latest' run: sudo apt install tor @@ -242,15 +224,6 @@ jobs: steps: - # Get vcpython27 for Windows + Python 2.7, to build netifaces - # extension. See https://chocolatey.org/packages/vcpython27 and - # https://github.com/crazy-max/ghaction-chocolatey - - name: Install MSVC 9.0 for Python 2.7 [Windows] - if: matrix.os == 'windows-latest' && matrix.python-version == '2.7' - uses: crazy-max/ghaction-chocolatey@v1 - with: - args: install vcpython27 - - name: Check out Tahoe-LAFS sources uses: actions/checkout@v2 with: From 196ce5103dd203fd7cc59310fa55d3bd851a14d5 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 26 Apr 2021 12:00:11 -0400 Subject: [PATCH 186/463] GitHub Actions: test with 32-bit Python 2.7 on Windows --- .github/workflows/ci.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6bec373f8..9f0515f86 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,10 +35,22 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} + if: matrix.os != 'windows-latest' && matrix.python-version != '2.7' uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + # We use netifaces, which does not ship a 64-bit wheel for + # Windows, but it ships a 32-bit wheel. Since MS has pulled + # vcpython27 compiler, building a netifaces wheel is not an + # option anymore, so let us test on 32-bit Windows. + - name: Set up Python ${{ matrix.python-version }} [Windows x86] + if: matrix.os == 'windows-latest' && matrix.python-version == '2.7' + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + # To use pip caching with GitHub Actions in an OS-independent # manner, we need `pip cache dir` command, which became # available since pip v20.1+. At the time of writing this, @@ -175,10 +187,19 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} + if: matrix.os != 'windows-latest' && matrix.python-version != '2.7' uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + # See this step under coverage job. + - name: Set up Python ${{ matrix.python-version }} [Windows x86] + if: matrix.os == 'windows-latest' && matrix.python-version == '2.7' + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + - name: Get pip cache directory id: pip-cache run: | @@ -230,10 +251,19 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} + if: matrix.os != 'windows-latest' && matrix.python-version != '2.7' uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + # See this step under coverage job. + - name: Set up Python ${{ matrix.python-version }} [Windows x86] + if: matrix.os == 'windows-latest' && matrix.python-version == '2.7' + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + - name: Get pip cache directory id: pip-cache run: | From f4b8780ba7c2ef136f60635c0c9fa56f41197f7c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 26 Apr 2021 12:00:37 -0400 Subject: [PATCH 187/463] Add newsfragment --- newsfragments/3681.installations | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 newsfragments/3681.installations diff --git a/newsfragments/3681.installations b/newsfragments/3681.installations new file mode 100644 index 000000000..a697e6c60 --- /dev/null +++ b/newsfragments/3681.installations @@ -0,0 +1,3 @@ +Tahoe-LAFS CI now runs tests only on 32-bit Windows. Microsoft has +removed vcpython27 compiler downloads from their site, and Tahoe-LAFS +needs vcpython27 to build and install netifaces on 64-bit Windows. From 106976e8cc8d119025b1dc307bb9a722e26da87d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 26 Apr 2021 12:17:25 -0400 Subject: [PATCH 188/463] GitHub Actions: use expression syntax Per GitHub documentation: When you use expressions in an if conditional, you may omit the expression syntax (${{ }}) because GitHub automatically evaluates the if conditional as an expression, unless the expression contains any operators. If the expression contains any operators, the expression must be contained within ${{ }} to explicitly mark it for evaluation. https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f0515f86..61e40fcd0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - if: matrix.os != 'windows-latest' && matrix.python-version != '2.7' + if: ${{ matrix.os != 'windows-latest' && matrix.python-version != '2.7' }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} @@ -45,7 +45,7 @@ jobs: # vcpython27 compiler, building a netifaces wheel is not an # option anymore, so let us test on 32-bit Windows. - name: Set up Python ${{ matrix.python-version }} [Windows x86] - if: matrix.os == 'windows-latest' && matrix.python-version == '2.7' + if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '2.7' }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} @@ -187,14 +187,14 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - if: matrix.os != 'windows-latest' && matrix.python-version != '2.7' + if: ${{ matrix.os != 'windows-latest' && matrix.python-version != '2.7' }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} # See this step under coverage job. - name: Set up Python ${{ matrix.python-version }} [Windows x86] - if: matrix.os == 'windows-latest' && matrix.python-version == '2.7' + if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '2.7' }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} @@ -251,14 +251,14 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - if: matrix.os != 'windows-latest' && matrix.python-version != '2.7' + if: ${{ matrix.os != 'windows-latest' && matrix.python-version != '2.7' }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} # See this step under coverage job. - name: Set up Python ${{ matrix.python-version }} [Windows x86] - if: matrix.os == 'windows-latest' && matrix.python-version == '2.7' + if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '2.7' }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} From ed82119f32af602c21201e572a4b2c705d892038 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 26 Apr 2021 13:14:28 -0400 Subject: [PATCH 189/463] GitHub Actions: when in doubt, throw in more parens --- .github/workflows/ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61e40fcd0..b82f3cbda 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - if: ${{ matrix.os != 'windows-latest' && matrix.python-version != '2.7' }} + if: ${{ ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' ) }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} @@ -45,7 +45,7 @@ jobs: # vcpython27 compiler, building a netifaces wheel is not an # option anymore, so let us test on 32-bit Windows. - name: Set up Python ${{ matrix.python-version }} [Windows x86] - if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '2.7' }} + if: ${{ ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' ) }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} @@ -187,14 +187,14 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - if: ${{ matrix.os != 'windows-latest' && matrix.python-version != '2.7' }} + if: ${{ ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' ) }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - # See this step under coverage job. + # See this step under coverage job. - name: Set up Python ${{ matrix.python-version }} [Windows x86] - if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '2.7' }} + if: ${{ ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' ) }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} @@ -251,14 +251,14 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - if: ${{ matrix.os != 'windows-latest' && matrix.python-version != '2.7' }} + if: ${{ ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' ) }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} # See this step under coverage job. - name: Set up Python ${{ matrix.python-version }} [Windows x86] - if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '2.7' }} + if: ${{ ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' ) }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} From dec97b3aa119e6f242459d2d188d4c44598778cb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 26 Apr 2021 13:35:37 -0400 Subject: [PATCH 190/463] GitHub Actions: when in doubt, drop some curly braces --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b82f3cbda..2fe6bf9b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - if: ${{ ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' ) }} + if: ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' ) uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} @@ -45,7 +45,7 @@ jobs: # vcpython27 compiler, building a netifaces wheel is not an # option anymore, so let us test on 32-bit Windows. - name: Set up Python ${{ matrix.python-version }} [Windows x86] - if: ${{ ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' ) }} + if: ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' ) uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} @@ -187,14 +187,14 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - if: ${{ ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' ) }} + if: ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' ) uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} # See this step under coverage job. - name: Set up Python ${{ matrix.python-version }} [Windows x86] - if: ${{ ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' ) }} + if: ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' ) uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} @@ -251,14 +251,14 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - if: ${{ ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' ) }} + if: ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' ) uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} # See this step under coverage job. - name: Set up Python ${{ matrix.python-version }} [Windows x86] - if: ${{ ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' ) }} + if: ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' ) uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} From 68603fc015898a06d23052735b2059fb880323f6 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 26 Apr 2021 13:41:17 -0400 Subject: [PATCH 191/463] GitHub Actions: just use x86 Python to test on Windows I can't figure out the correct GitHub Actions magic incantation^w^w expression syntax that is needed to isolate (Windows && Python 2.7), so let's just run also run x86 Python 3.6 on Windows. --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fe6bf9b0..ab0ba3f08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - if: ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' ) + if: ${{ matrix.os != 'windows-latest' }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} @@ -45,7 +45,7 @@ jobs: # vcpython27 compiler, building a netifaces wheel is not an # option anymore, so let us test on 32-bit Windows. - name: Set up Python ${{ matrix.python-version }} [Windows x86] - if: ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' ) + if: ${{ matrix.os == 'windows-latest' }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} @@ -187,14 +187,14 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - if: ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' ) + if: ${{ matrix.os != 'windows-latest' }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} # See this step under coverage job. - name: Set up Python ${{ matrix.python-version }} [Windows x86] - if: ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' ) + if: ${{ matrix.os == 'windows-latest' }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} @@ -251,14 +251,14 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - if: ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' ) + if: ${{ matrix.os != 'windows-latest' }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} # See this step under coverage job. - name: Set up Python ${{ matrix.python-version }} [Windows x86] - if: ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' ) + if: ${{ matrix.os == 'windows-latest' }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} From 1a05fb21de203c9b3c31e8a6143da12a6cd48157 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 26 Apr 2021 14:02:26 -0400 Subject: [PATCH 192/463] Name newsfragment correctly --- newsfragments/{3681.installations => 3681.installation} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{3681.installations => 3681.installation} (100%) diff --git a/newsfragments/3681.installations b/newsfragments/3681.installation similarity index 100% rename from newsfragments/3681.installations rename to newsfragments/3681.installation From e46e4409c2649dc4549ab256d3dee618fd99dbf7 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 26 Apr 2021 16:39:23 -0400 Subject: [PATCH 193/463] GitHub Actions: really use x86 Python on Windows In a prior commit, I mistakenly used "architecture: x64" instead of "architecture: x86", and tests actually passed. That was surprising, because netifaces do not have amd64_win will on PyPI. But mystery was solved when itamarst pointed out that netifaces wheel (that we previously built) is present on pip cache. But pip cache might be purged one day, and tests will fail again that day. We can't have that, so we will try to stick with x86 for now. --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab0ba3f08..bfa705d28 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - architecture: x64 + architecture: x86 # To use pip caching with GitHub Actions in an OS-independent # manner, we need `pip cache dir` command, which became @@ -198,7 +198,7 @@ jobs: uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - architecture: x64 + architecture: x86 - name: Get pip cache directory id: pip-cache @@ -262,7 +262,7 @@ jobs: uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - architecture: x64 + architecture: x86 - name: Get pip cache directory id: pip-cache From 1531bea63fda3851ed988ac2b97574c413741cbb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 26 Apr 2021 16:58:24 -0400 Subject: [PATCH 194/463] GitHub Actions: update note about Windows [skip ci] --- .github/workflows/ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bfa705d28..2adb22bc8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,10 +40,11 @@ jobs: with: python-version: ${{ matrix.python-version }} - # We use netifaces, which does not ship a 64-bit wheel for - # Windows, but it ships a 32-bit wheel. Since MS has pulled - # vcpython27 compiler, building a netifaces wheel is not an - # option anymore, so let us test on 32-bit Windows. + # We use netifaces, which does not ship a 64-bit wheel for the + # Python 2.7 + Windows combination, but it ships a 32-bit wheel. + # Since MS has removed vcpython27 compiler downloads from their + # usual site, building a netifaces wheel is not an option + # anymore. So let us just test with 32-bit Python on Windows. - name: Set up Python ${{ matrix.python-version }} [Windows x86] if: ${{ matrix.os == 'windows-latest' }} uses: actions/setup-python@v1 From 65398a2d63bea5e70ec65d767a7d9969d204ea82 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 26 Apr 2021 17:06:34 -0400 Subject: [PATCH 195/463] GitHub Actions: update note about Windows again --- .github/workflows/ci.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2adb22bc8..eeb6bbe5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,8 +43,14 @@ jobs: # We use netifaces, which does not ship a 64-bit wheel for the # Python 2.7 + Windows combination, but it ships a 32-bit wheel. # Since MS has removed vcpython27 compiler downloads from their - # usual site, building a netifaces wheel is not an option - # anymore. So let us just test with 32-bit Python on Windows. + # usual download site, building a netifaces wheel locally is not + # an option anymore. So let us just test with 32-bit Python on + # Windows. + # + # It seems that GitHub Actions' expression syntax is not + # expressive enough to select just the (32-bit Python 2.7, + # Windows) combination, so we'd also be switching to (32-bit + # Python 3.6, Windows). - name: Set up Python ${{ matrix.python-version }} [Windows x86] if: ${{ matrix.os == 'windows-latest' }} uses: actions/setup-python@v1 From 73d3295a6ae75de170f629821d315510208c6231 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 27 Apr 2021 15:49:25 -0400 Subject: [PATCH 196/463] GitHub Actions: use 64-bit Python 3.6 to test on Windows --- .github/workflows/ci.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eeb6bbe5b..aa128d4ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,19 +40,25 @@ jobs: with: python-version: ${{ matrix.python-version }} + # See note below about need for using 32-bit Python 2.7 on + # Windows. The extra handling here for Python 3.6 on Windows is + # because I could not figure out the right GitHub Actions + # expression to do this in a better way. + - name: Set up Python ${{ matrix.python-version }} [Windows x64] + if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '3.6' }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + # We use netifaces, which does not ship a 64-bit wheel for the # Python 2.7 + Windows combination, but it ships a 32-bit wheel. # Since MS has removed vcpython27 compiler downloads from their # usual download site, building a netifaces wheel locally is not # an option anymore. So let us just test with 32-bit Python on # Windows. - # - # It seems that GitHub Actions' expression syntax is not - # expressive enough to select just the (32-bit Python 2.7, - # Windows) combination, so we'd also be switching to (32-bit - # Python 3.6, Windows). - name: Set up Python ${{ matrix.python-version }} [Windows x86] - if: ${{ matrix.os == 'windows-latest' }} + if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '2.7' }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} From 3722b8f628472a4c509a78267ae259438046c749 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 27 Apr 2021 15:52:28 -0400 Subject: [PATCH 197/463] GitHub Actions: quote architecture Docs for actions/setup-python seem to do that, although inconsistently. --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa128d4ef..73294b730 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - architecture: x64 + architecture: 'x64' # We use netifaces, which does not ship a 64-bit wheel for the # Python 2.7 + Windows combination, but it ships a 32-bit wheel. @@ -62,7 +62,7 @@ jobs: uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - architecture: x86 + architecture: 'x86' # To use pip caching with GitHub Actions in an OS-independent # manner, we need `pip cache dir` command, which became @@ -211,7 +211,7 @@ jobs: uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - architecture: x86 + architecture: 'x86' - name: Get pip cache directory id: pip-cache @@ -275,7 +275,7 @@ jobs: uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - architecture: x86 + architecture: 'x86' - name: Get pip cache directory id: pip-cache From 6d1b95b965f16f7a15a238f8431fa60cb68f9cdf Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 27 Apr 2021 16:21:22 -0400 Subject: [PATCH 198/463] GitHub Actions: add more parenthesis https://github.community/t/and-operator-in-if-condition/154825 suggests that adding more parens might work. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73294b730..f70432267 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: # because I could not figure out the right GitHub Actions # expression to do this in a better way. - name: Set up Python ${{ matrix.python-version }} [Windows x64] - if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '3.6' }} + if: ${{ ( matrix.os == 'windows-latest' ) && ( matrix.python-version == '3.6' ) }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} @@ -58,7 +58,7 @@ jobs: # an option anymore. So let us just test with 32-bit Python on # Windows. - name: Set up Python ${{ matrix.python-version }} [Windows x86] - if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '2.7' }} + if: ${{ ( matrix.os == 'windows-latest' ) && ( matrix.python-version == '2.7' ) }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} From ec449970eef7baacb80fd67fd82318f3b4f491f6 Mon Sep 17 00:00:00 2001 From: May-Lee Sia Date: Thu, 29 Apr 2021 07:56:27 +0200 Subject: [PATCH 199/463] Add CONTRIBUTORS.rst --- contributors.rst | 43 ++++++++++++++++++++++++++++++++ newsfragments/3682.documentation | 1 + 2 files changed, 44 insertions(+) create mode 100644 contributors.rst create mode 100644 newsfragments/3682.documentation diff --git a/contributors.rst b/contributors.rst new file mode 100644 index 000000000..bde4b014d --- /dev/null +++ b/contributors.rst @@ -0,0 +1,43 @@ +Contributor Checklist +===================== + + +* Create a ``Trac`` ticket, fill it out and assign it to yourself (contact exarkun if you don't have an account): + + ``https://tahoe-lafs.org/trac/tahoe-lafs/newticket`` + +* Use the ticket number to name your branch (example): + + ``3003.contributor-guide`` + +* Good idea to add tests at the same time you write your code. + +* Add a file to the ``/newsfragments`` folder, named with the ticket number and the type of patch (example): + + ``newsfragments/3651.minor`` + +* ``towncrier`` recognizes the following types: + + ``incompat``, ``feature``, ``bugfix``, ``installation``, ``configuration``, ``documentation``, ``removed``, ``other``, ``minor`` +* Add one sentence to ``newsfragments/.`` describing the change (example): + + ``The integration test suite has been updated to use pytest-twisted instead of deprecated pytest APIs.`` + +* Run the test suite with ``tox``, ``tox -e codechecks`` and ``tox -e typechecks`` + +* Push your branch to Github with your ticket number in the merge commit message (example): + + ``Fixes ticket:3003`` + + This makes the ``Trac`` ticket close when your PR gets approved. + +* Request appropriate review. + + +References +---------- + +This checklist is a summary of `this page on contributing Patches `__ + +Before authoring or reviewing a patch, please familiarize yourself with the `Coding Standard `__ +and the `Contributor Code of Conduct `__. diff --git a/newsfragments/3682.documentation b/newsfragments/3682.documentation new file mode 100644 index 000000000..74c4cf246 --- /dev/null +++ b/newsfragments/3682.documentation @@ -0,0 +1 @@ +A cheatsheet-style document for contributors was created at CONTRIBUTORS.md From db65b3d69b6f79fef257fc4b67b3861e3d473eb8 Mon Sep 17 00:00:00 2001 From: May-Lee Sia Date: Thu, 29 Apr 2021 08:00:13 +0200 Subject: [PATCH 200/463] Clean up some files --- contributors.rst => CONTRIBUTORS.rst | 0 newsfragments/3682.documentation | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename contributors.rst => CONTRIBUTORS.rst (100%) diff --git a/contributors.rst b/CONTRIBUTORS.rst similarity index 100% rename from contributors.rst rename to CONTRIBUTORS.rst diff --git a/newsfragments/3682.documentation b/newsfragments/3682.documentation index 74c4cf246..5cf78bd90 100644 --- a/newsfragments/3682.documentation +++ b/newsfragments/3682.documentation @@ -1 +1 @@ -A cheatsheet-style document for contributors was created at CONTRIBUTORS.md +A cheatsheet-style document for contributors was created at CONTRIBUTORS.rst \ No newline at end of file From 9137da5483c9dddc62f3e7ab639151c3d4472c44 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Apr 2021 10:16:41 -0400 Subject: [PATCH 201/463] Stick to Unicode when possible. --- src/allmydata/scripts/tahoe_check.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index d8b7c9bce..494c1dba3 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -53,14 +53,11 @@ def check_location(options, where): if resp.status != 200: print(format_http_error("ERROR", resp), file=stderr) return 1 - jdata = resp.read() + jdata = resp.read().decode() + if options.get("raw"): - if PY3: - stdoutb = stdout.buffer - else: - stdoutb = stdout - stdoutb.write(jdata) - stdoutb.write(b"\n") + stdout.write(jdata) + stdout.write("\n") return 0 data = json.loads(jdata) @@ -323,17 +320,12 @@ class DeepCheckStreamer(LineOnlyReceiver, object): return 1 # use Twisted to split this into lines - if PY3: - stdoutb = stdout.buffer - else: - stdoutb = stdout - while True: chunk = resp.read(100) if not chunk: break if self.options["raw"]: - stdoutb.write(chunk) + stdout.write(chunk.decode()) else: output.dataReceived(chunk) if not self.options["raw"]: From 72a85ba62422b3e398459bde3ea6ad8cbd2fafef Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Apr 2021 10:19:59 -0400 Subject: [PATCH 202/463] Fix lint. --- src/allmydata/scripts/tahoe_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index 494c1dba3..82885d073 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -4,7 +4,7 @@ from urllib.parse import quote as url_quote import json # Python 2 compatibility -from future.utils import PY2, PY3 +from future.utils import PY2 if PY2: from future.builtins import str # noqa: F401 From 2f6535e26e09f0bb7e5a38d2e895d5966a0bdfd3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Apr 2021 10:33:59 -0400 Subject: [PATCH 203/463] First test passes on Python 3. --- src/allmydata/scripts/tahoe_webopen.py | 8 +++++-- src/allmydata/test/cli/test_create_alias.py | 26 +++++++++++---------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/allmydata/scripts/tahoe_webopen.py b/src/allmydata/scripts/tahoe_webopen.py index a7b7ca7e1..0292e0d40 100644 --- a/src/allmydata/scripts/tahoe_webopen.py +++ b/src/allmydata/scripts/tahoe_webopen.py @@ -1,7 +1,10 @@ +from past.builtins import unicode + +from urllib.parse import quote as url_quote from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError -import urllib + def webopen(options, opener=None): nodeurl = options['node-url'] @@ -15,9 +18,10 @@ def webopen(options, opener=None): except UnknownAliasError as e: e.display(stderr) return 1 + path = unicode(path, "utf-8") if path == '/': path = '' - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + url = nodeurl + "uri/%s" % url_quote(rootcap) if path: url += "/" + escape_path(path) else: diff --git a/src/allmydata/test/cli/test_create_alias.py b/src/allmydata/test/cli/test_create_alias.py index ea3200e2e..2718a706f 100644 --- a/src/allmydata/test/cli/test_create_alias.py +++ b/src/allmydata/test/cli/test_create_alias.py @@ -1,7 +1,9 @@ +from future.builtins import str from six.moves import StringIO import os.path from twisted.trial import unittest -import urllib +from urllib.parse import quote as url_quote + from allmydata.util import fileutil from allmydata.scripts.common import get_aliases from allmydata.scripts import cli, runner @@ -22,7 +24,7 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): rc = cli.webopen(o.subOptions, urls.append) self.failUnlessReallyEqual(rc, 0) self.failUnlessReallyEqual(len(urls), 1) - self.failUnlessReallyEqual(urls[0], expected_url) + self.assertEqual(urls[0], expected_url) def test_create(self): self.basedir = "cli/CreateAlias/create" @@ -36,19 +38,19 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): self.assertIn("Alias 'tahoe' created", stdout) aliases = get_aliases(self.get_clientdir()) self.failUnless("tahoe" in aliases) - self.failUnless(aliases["tahoe"].startswith("URI:DIR2:")) + self.failUnless(aliases["tahoe"].startswith(b"URI:DIR2:")) d.addCallback(_done) d.addCallback(lambda res: self.do_cli("create-alias", "two:")) def _stash_urls(res): aliases = get_aliases(self.get_clientdir()) node_url_file = os.path.join(self.get_clientdir(), "node.url") - nodeurl = fileutil.read(node_url_file).strip() + nodeurl = fileutil.read(node_url_file, mode="r").strip() self.welcome_url = nodeurl uribase = nodeurl + "uri/" - self.tahoe_url = uribase + urllib.quote(aliases["tahoe"]) + self.tahoe_url = uribase + url_quote(aliases["tahoe"]) self.tahoe_subdir_url = self.tahoe_url + "/subdir" - self.two_url = uribase + urllib.quote(aliases["two"]) + self.two_url = uribase + url_quote(aliases["two"]) self.two_uri = aliases["two"] d.addCallback(_stash_urls) @@ -128,13 +130,13 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): # like a valid dircap, so get_aliases() will raise an exception. aliases = get_aliases(self.get_clientdir()) self.failUnless("added" in aliases) - self.failUnless(aliases["added"].startswith("URI:DIR2:")) + self.failUnless(aliases["added"].startswith(b"URI:DIR2:")) # to be safe, let's confirm that we don't see "NAME2:" in CAP1. # No chance of a false-negative, because the hyphen in # "un-corrupted1" is not a valid base32 character. - self.failIfIn("un-corrupted1:", aliases["added"]) + self.failIfIn(b"un-corrupted1:", aliases["added"]) self.failUnless("un-corrupted1" in aliases) - self.failUnless(aliases["un-corrupted1"].startswith("URI:DIR2:")) + self.failUnless(aliases["un-corrupted1"].startswith(b"URI:DIR2:")) d.addCallback(_check_not_corrupted1) def _remove_trailing_newline_and_add_alias(ign): @@ -149,10 +151,10 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): self.failIf(stderr) aliases = get_aliases(self.get_clientdir()) self.failUnless("un-corrupted1" in aliases) - self.failUnless(aliases["un-corrupted1"].startswith("URI:DIR2:")) - self.failIfIn("un-corrupted2:", aliases["un-corrupted1"]) + self.failUnless(aliases["un-corrupted1"].startswith(b"URI:DIR2:")) + self.failIfIn(b"un-corrupted2:", aliases["un-corrupted1"]) self.failUnless("un-corrupted2" in aliases) - self.failUnless(aliases["un-corrupted2"].startswith("URI:DIR2:")) + self.failUnless(aliases["un-corrupted2"].startswith(b"URI:DIR2:")) d.addCallback(_check_not_corrupted) return d From 46c03f6b75ffe6799af751ba8eedf92c7d35cf99 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Apr 2021 11:05:56 -0400 Subject: [PATCH 204/463] Remove duplication, and add support for testing cases where stdin/stdout/stderr have to be bytes. --- src/allmydata/test/common_util.py | 76 ++++++++++++++++++------------- 1 file changed, 44 insertions(+), 32 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 16f945239..95065710c 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -6,7 +6,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals -from future.utils import PY2, bchr, binary_type +from future.utils import PY2, PY3, bchr, binary_type from future.builtins import str as future_str if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 @@ -15,7 +15,8 @@ import os import time import signal from random import randrange -from six.moves import StringIO +if PY2: + from StringIO import StringIO from io import ( TextIOWrapper, BytesIO, @@ -64,23 +65,28 @@ def run_cli_native(verb, *args, **kwargs): Most code should prefer ``run_cli_unicode`` which deals with all the necessary encoding considerations. - :param native_str verb: The command to run. For example, ``"create-node"``. + :param native_str verb: The command to run. For example, + ``"create-node"``. - :param [native_str] args: The arguments to pass to the command. For example, - ``("--hostname=localhost",)``. + :param [native_str] args: The arguments to pass to the command. For + example, ``("--hostname=localhost",)``. - :param [native_str] nodeargs: Extra arguments to pass to the Tahoe executable - before ``verb``. + :param [native_str] nodeargs: Extra arguments to pass to the Tahoe + executable before ``verb``. - :param native_str stdin: Text to pass to the command via stdin. + :param bytes|unicode stdin: Text or bytes to pass to the command via stdin. :param NoneType|str encoding: The name of an encoding which stdout and - stderr will be configured to use. ``None`` means stdout and stderr - will accept bytes and unicode and use the default system encoding for - translating between them. + stderr will be configured to use. ``None`` means matching default + behavior for the given Python version. + + :param bool return_bytes: If False, stdout/stderr is native string, + matching native behavior. If True, stdout/stderr are returned as + bytes. """ nodeargs = kwargs.pop("nodeargs", []) - encoding = kwargs.pop("encoding", None) + encoding = kwargs.pop("encoding", "utf-8") + return_bytes = kwargs.pop("return_bytes", False) verb = maybe_unicode_to_argv(verb) args = [maybe_unicode_to_argv(a) for a in args] nodeargs = [maybe_unicode_to_argv(a) for a in nodeargs] @@ -93,36 +99,42 @@ def run_cli_native(verb, *args, **kwargs): ) argv = nodeargs + [verb] + list(args) stdin = kwargs.get("stdin", "") - if encoding is None: - if PY2: - # The original behavior, the Python 2 behavior, is to accept either - # bytes or unicode and try to automatically encode or decode as - # necessary. This works okay for ASCII and if LANG is set - # appropriately. These aren't great constraints so we should move - # away from this behavior. - stdout = StringIO() - stderr = StringIO() - else: - # Default on Python 3 is accepting text. - stdout = TextIOWrapper(BytesIO(), "utf-8") - stderr = TextIOWrapper(BytesIO(), "utf-8") + if PY2: + # The original behavior, the Python 2 behavior, is to accept either + # bytes or unicode and try to automatically encode or decode as + # necessary. This works okay for ASCII and if LANG is set + # appropriately. These aren't great constraints so we should move + # away from this behavior. + stdin = StringIO(stdin) + stdout = StringIO() + stderr = StringIO() else: # The new behavior, the Python 3 behavior, is to accept unicode and - # encode it using a specific encoding. For older versions of Python - # 3, the encoding is determined from LANG (bad) but for newer Python - # 3, the encoding is always utf-8 (good). Tests can pass in different - # encodings to exercise different behaviors. + # encode it using a specific encoding. For older versions of Python 3, + # the encoding is determined from LANG (bad) but for newer Python 3, + # the encoding is either LANG if it supports full Unicode, otherwise + # utf-8 (good). Tests can pass in different encodings to exercise + # different behaviors. + if isinstance(stdin, str): + stdin = stdin.encode(encoding) + stdin = TextIOWrapper(BytesIO(stdin), encoding) stdout = TextIOWrapper(BytesIO(), encoding) stderr = TextIOWrapper(BytesIO(), encoding) d = defer.succeed(argv) d.addCallback(runner.parse_or_exit_with_explanation, stdout=stdout) d.addCallback(runner.dispatch, - stdin=StringIO(stdin), + stdin=stdin, stdout=stdout, stderr=stderr) - def _done(rc): + def _done(rc, stdout=stdout, stderr=stderr): + if return_bytes and PY3: + stdout = stdout.buffer + stderr = stderr.buffer return 0, _getvalue(stdout), _getvalue(stderr) - def _err(f): + def _err(f, stdout=stdout, stderr=stderr): f.trap(SystemExit) + if return_bytes and PY3: + stdout = stdout.buffer + stderr = stderr.buffer return f.value.code, _getvalue(stdout), _getvalue(stderr) d.addCallbacks(_done, _err) return d From 3846df8e4fb3a8677d77ebc7ae62b9d8b1079507 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Apr 2021 11:06:31 -0400 Subject: [PATCH 205/463] All test_create_alias tests pass on Python 3. --- src/allmydata/scripts/tahoe_put.py | 12 ++++++-- src/allmydata/test/cli/test_create_alias.py | 31 ++++++++++----------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/src/allmydata/scripts/tahoe_put.py b/src/allmydata/scripts/tahoe_put.py index b64283a81..8db705d01 100644 --- a/src/allmydata/scripts/tahoe_put.py +++ b/src/allmydata/scripts/tahoe_put.py @@ -1,8 +1,9 @@ from __future__ import print_function +from future.utils import PY2 from past.builtins import unicode -from six.moves import cStringIO as StringIO +from io import BytesIO from urllib.parse import quote as url_quote from allmydata.scripts.common_http import do_http, format_http_success, format_http_error @@ -83,8 +84,13 @@ def put(options): # Content-Length field. So we currently must copy it. if verbosity > 0: print("waiting for file data on stdin..", file=stderr) - data = stdin.read() - infileobj = StringIO(data) + # We're uploading arbitrary files, so this had better be bytes: + if PY2: + stdinb = stdin + else: + stdinb = stdin.buffer + data = stdinb.read() + infileobj = BytesIO(data) resp = do_http("PUT", url, infileobj) diff --git a/src/allmydata/test/cli/test_create_alias.py b/src/allmydata/test/cli/test_create_alias.py index 2718a706f..30f613486 100644 --- a/src/allmydata/test/cli/test_create_alias.py +++ b/src/allmydata/test/cli/test_create_alias.py @@ -162,48 +162,47 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): self.basedir = "cli/CreateAlias/create_unicode" self.set_up_grid(oneshare=True) - try: - etudes_arg = u"\u00E9tudes".encode(get_io_encoding()) - lumiere_arg = u"lumi\u00E8re.txt".encode(get_io_encoding()) - except UnicodeEncodeError: - raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.") + etudes_arg = u"\u00E9tudes" + lumiere_arg = u"lumi\u00E8re.txt" d = self.do_cli("create-alias", etudes_arg) def _check_create_unicode(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessIn("Alias %s created" % quote_output(u"\u00E9tudes"), out) aliases = get_aliases(self.get_clientdir()) - self.failUnless(aliases[u"\u00E9tudes"].startswith("URI:DIR2:")) + self.failUnless(aliases[u"\u00E9tudes"].startswith(b"URI:DIR2:")) d.addCallback(_check_create_unicode) d.addCallback(lambda res: self.do_cli("ls", etudes_arg + ":")) def _check_ls1(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(err), 0, err) + self.assertEqual(len(out), 0, out) d.addCallback(_check_ls1) + DATA = b"Blah blah blah \xff blah \x00 blah" d.addCallback(lambda res: self.do_cli("put", "-", etudes_arg + ":uploaded.txt", - stdin="Blah blah blah")) + stdin=DATA)) d.addCallback(lambda res: self.do_cli("ls", etudes_arg + ":")) def _check_ls2(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(out, "uploaded.txt\n") d.addCallback(_check_ls2) - d.addCallback(lambda res: self.do_cli("get", etudes_arg + ":uploaded.txt")) + d.addCallback(lambda res: self.do_cli("get", etudes_arg + ":uploaded.txt", + return_bytes=True)) def _check_get(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") - self.failUnlessReallyEqual(out, "Blah blah blah") + self.assertEqual(len(err), 0, err) + self.failUnlessReallyEqual(out, DATA) d.addCallback(_check_get) # Ensure that an Unicode filename in an Unicode alias works as expected @@ -211,11 +210,11 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): stdin="Let the sunshine In!")) d.addCallback(lambda res: self.do_cli("get", - get_aliases(self.get_clientdir())[u"\u00E9tudes"] + "/" + lumiere_arg)) + str(get_aliases(self.get_clientdir())[u"\u00E9tudes"], "ascii") + "/" + lumiere_arg)) def _check_get2(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(out, "Let the sunshine In!") d.addCallback(_check_get2) From da8e0d61aa7b7a695ab69a72f43071b76cc81548 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Apr 2021 11:15:48 -0400 Subject: [PATCH 206/463] Port to Python 3. --- src/allmydata/test/cli/test_create_alias.py | 28 +++++++++++++++------ src/allmydata/util/_python3.py | 1 + 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/cli/test_create_alias.py b/src/allmydata/test/cli/test_create_alias.py index 30f613486..e82ecf60c 100644 --- a/src/allmydata/test/cli/test_create_alias.py +++ b/src/allmydata/test/cli/test_create_alias.py @@ -1,4 +1,16 @@ -from future.builtins import str +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from six import ensure_str from six.moves import StringIO import os.path from twisted.trial import unittest @@ -170,7 +182,7 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) self.assertEqual(len(err), 0, err) - self.failUnlessIn("Alias %s created" % quote_output(u"\u00E9tudes"), out) + self.failUnlessIn(ensure_str("Alias %s created") % quote_output(etudes_arg), out) aliases = get_aliases(self.get_clientdir()) self.failUnless(aliases[u"\u00E9tudes"].startswith(b"URI:DIR2:")) @@ -193,7 +205,7 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) self.assertEqual(len(err), 0, err) - self.failUnlessReallyEqual(out, "uploaded.txt\n") + self.assertEqual(out, "uploaded.txt\n") d.addCallback(_check_ls2) d.addCallback(lambda res: self.do_cli("get", etudes_arg + ":uploaded.txt", @@ -207,15 +219,17 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): # Ensure that an Unicode filename in an Unicode alias works as expected d.addCallback(lambda res: self.do_cli("put", "-", etudes_arg + ":" + lumiere_arg, - stdin="Let the sunshine In!")) + stdin=b"Let the sunshine In!")) - d.addCallback(lambda res: self.do_cli("get", - str(get_aliases(self.get_clientdir())[u"\u00E9tudes"], "ascii") + "/" + lumiere_arg)) + d.addCallback(lambda res: self.do_cli( + "get", + str(get_aliases(self.get_clientdir())[u"\u00E9tudes"], "ascii") + "/" + lumiere_arg, + return_bytes=True)) def _check_get2(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) self.assertEqual(len(err), 0, err) - self.failUnlessReallyEqual(out, "Let the sunshine In!") + self.failUnlessReallyEqual(out, b"Let the sunshine In!") d.addCallback(_check_get2) return d diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 471c7e913..1e6111e6c 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -179,6 +179,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.cli.test_check", "allmydata.test.cli.test_cp", "allmydata.test.cli.test_create", + "allmydata.test.cli.test_create_alias", "allmydata.test.cli.test_invite", "allmydata.test.cli.test_status", From 4eb9be1996b8fa091e26b16a6882e99ae74f6688 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Apr 2021 11:17:09 -0400 Subject: [PATCH 207/463] News file. --- newsfragments/3687.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3687.minor diff --git a/newsfragments/3687.minor b/newsfragments/3687.minor new file mode 100644 index 000000000..e69de29bb From 463f9fe80266d2a8934a92e549503bf06ae9c68c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Apr 2021 11:31:25 -0400 Subject: [PATCH 208/463] Tests pass on Python 3. --- src/allmydata/test/cli/test_put.py | 57 +++++++++++++++++------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/src/allmydata/test/cli/test_put.py b/src/allmydata/test/cli/test_put.py index 3392e67b4..ef26f5990 100644 --- a/src/allmydata/test/cli/test_put.py +++ b/src/allmydata/test/cli/test_put.py @@ -1,3 +1,5 @@ +from past.builtins import unicode + import os.path from twisted.trial import unittest from twisted.python import usage @@ -17,7 +19,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): # tahoe get `echo DATA | tahoe put` # tahoe get `echo DATA | tahoe put -` self.basedir = "cli/Put/unlinked_immutable_stdin" - DATA = "data" * 100 + DATA = b"data\xff" * 100 self.set_up_grid(oneshare=True) d = self.do_cli("put", stdin=DATA) def _uploaded(res): @@ -27,10 +29,11 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.readcap = out self.failUnless(self.readcap.startswith("URI:CHK:")) d.addCallback(_uploaded) - d.addCallback(lambda res: self.do_cli("get", self.readcap)) + d.addCallback(lambda res: self.do_cli("get", self.readcap, + return_bytes=True)) def _downloaded(res): (rc, out, err) = res - self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(err, b"") self.failUnlessReallyEqual(out, DATA) d.addCallback(_downloaded) d.addCallback(lambda res: self.do_cli("put", "-", stdin=DATA)) @@ -49,7 +52,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): rel_fn = unicode(os.path.join(self.basedir, "DATAFILE")) abs_fn = abspath_expanduser_unicode(rel_fn) # we make the file small enough to fit in a LIT file, for speed - fileutil.write(rel_fn, "short file") + fileutil.write(rel_fn, b"short file has some bytes \xff yes") d = self.do_cli_unicode(u"put", [rel_fn]) def _uploaded(args): (rc, out, err) = args @@ -79,8 +82,8 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): rel_fn = os.path.join(self.basedir, "DATAFILE") # we make the file small enough to fit in a LIT file, for speed - DATA = "short file" - DATA2 = "short file two" + DATA = b"short file" + DATA2 = b"short file two" fileutil.write(rel_fn, DATA) d = self.do_cli("create-alias", "tahoe") @@ -95,7 +98,8 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.readcap = readcap d.addCallback(_uploaded) d.addCallback(lambda res: - self.do_cli("get", "tahoe:uploaded.txt")) + self.do_cli("get", "tahoe:uploaded.txt", + return_bytes=True)) d.addCallback(lambda rc_stdout_stderr: self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA)) @@ -110,32 +114,36 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("put", rel_fn, "subdir/uploaded2.txt")) - d.addCallback(lambda res: self.do_cli("get", "subdir/uploaded2.txt")) + d.addCallback(lambda res: self.do_cli("get", "subdir/uploaded2.txt", + return_bytes=True)) d.addCallback(lambda rc_stdout_stderr: self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA)) d.addCallback(lambda res: self.do_cli("put", rel_fn, "tahoe:uploaded3.txt")) - d.addCallback(lambda res: self.do_cli("get", "tahoe:uploaded3.txt")) + d.addCallback(lambda res: self.do_cli("get", "tahoe:uploaded3.txt", + return_bytes=True)) d.addCallback(lambda rc_stdout_stderr: self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA)) d.addCallback(lambda res: self.do_cli("put", rel_fn, "tahoe:subdir/uploaded4.txt")) d.addCallback(lambda res: - self.do_cli("get", "tahoe:subdir/uploaded4.txt")) + self.do_cli("get", "tahoe:subdir/uploaded4.txt", + return_bytes=True)) d.addCallback(lambda rc_stdout_stderr: self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA)) def _get_dircap(res): - self.dircap = get_aliases(self.get_clientdir())["tahoe"] + self.dircap = unicode(get_aliases(self.get_clientdir())["tahoe"], "ascii") d.addCallback(_get_dircap) d.addCallback(lambda res: self.do_cli("put", rel_fn, self.dircap+":./uploaded5.txt")) d.addCallback(lambda res: - self.do_cli("get", "tahoe:uploaded5.txt")) + self.do_cli("get", "tahoe:uploaded5.txt", + return_bytes=True)) d.addCallback(lambda rc_stdout_stderr: self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA)) @@ -143,7 +151,8 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.do_cli("put", rel_fn, self.dircap+":./subdir/uploaded6.txt")) d.addCallback(lambda res: - self.do_cli("get", "tahoe:subdir/uploaded6.txt")) + self.do_cli("get", "tahoe:subdir/uploaded6.txt", + return_bytes=True)) d.addCallback(lambda rc_stdout_stderr: self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA)) @@ -158,10 +167,10 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.basedir = "cli/Put/mutable_unlinked" self.set_up_grid(oneshare=True) - DATA = "data" * 100 - DATA2 = "two" * 100 + DATA = b"data" * 100 + DATA2 = b"two" * 100 rel_fn = os.path.join(self.basedir, "DATAFILE") - DATA3 = "three" * 100 + DATA3 = b"three" * 100 fileutil.write(rel_fn, DATA3) d = self.do_cli("put", "--mutable", stdin=DATA) @@ -172,7 +181,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.filecap = out self.failUnless(self.filecap.startswith("URI:SSK:"), self.filecap) d.addCallback(_created) - d.addCallback(lambda res: self.do_cli("get", self.filecap)) + d.addCallback(lambda res: self.do_cli("get", self.filecap, return_bytes=True)) d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA)) d.addCallback(lambda res: self.do_cli("put", "-", self.filecap, stdin=DATA2)) @@ -182,7 +191,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessIn("200 OK", err) self.failUnlessReallyEqual(self.filecap, out) d.addCallback(_replaced) - d.addCallback(lambda res: self.do_cli("get", self.filecap)) + d.addCallback(lambda res: self.do_cli("get", self.filecap, return_bytes=True)) d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA2)) d.addCallback(lambda res: self.do_cli("put", rel_fn, self.filecap)) @@ -191,7 +200,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessIn("200 OK", err) self.failUnlessReallyEqual(self.filecap, out) d.addCallback(_replaced2) - d.addCallback(lambda res: self.do_cli("get", self.filecap)) + d.addCallback(lambda res: self.do_cli("get", self.filecap, return_bytes=True)) d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA3)) return d @@ -436,10 +445,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): def test_immutable_from_file_unicode(self): # tahoe put "\u00E0 trier.txt" "\u00E0 trier.txt" - try: - a_trier_arg = u"\u00E0 trier.txt".encode(get_io_encoding()) - except UnicodeEncodeError: - raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.") + a_trier_arg = u"\u00E0 trier.txt" skip_if_cannot_represent_filename(u"\u00E0 trier.txt") @@ -448,7 +454,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): rel_fn = os.path.join(unicode(self.basedir), u"\u00E0 trier.txt") # we make the file small enough to fit in a LIT file, for speed - DATA = "short file" + DATA = b"short file \xff bytes" fileutil.write(rel_fn, DATA) d = self.do_cli("create-alias", "tahoe") @@ -464,7 +470,8 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(_uploaded) d.addCallback(lambda res: - self.do_cli("get", "tahoe:" + a_trier_arg)) + self.do_cli("get", "tahoe:" + a_trier_arg, + return_bytes=True)) d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA)) From f6b5628ce120c72066bd3870fae0a6c57174cdac Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Apr 2021 11:33:51 -0400 Subject: [PATCH 209/463] Port to Python 3. --- src/allmydata/test/cli/test_put.py | 26 ++++++++++++++++++-------- src/allmydata/util/_python3.py | 1 + 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/cli/test_put.py b/src/allmydata/test/cli/test_put.py index ef26f5990..c6a577074 100644 --- a/src/allmydata/test/cli/test_put.py +++ b/src/allmydata/test/cli/test_put.py @@ -1,4 +1,14 @@ -from past.builtins import unicode +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os.path from twisted.trial import unittest @@ -49,7 +59,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.basedir = "cli/Put/unlinked_immutable_from_file" self.set_up_grid(oneshare=True) - rel_fn = unicode(os.path.join(self.basedir, "DATAFILE")) + rel_fn = str(os.path.join(self.basedir, "DATAFILE")) abs_fn = abspath_expanduser_unicode(rel_fn) # we make the file small enough to fit in a LIT file, for speed fileutil.write(rel_fn, b"short file has some bytes \xff yes") @@ -135,7 +145,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA)) def _get_dircap(res): - self.dircap = unicode(get_aliases(self.get_clientdir())["tahoe"], "ascii") + self.dircap = str(get_aliases(self.get_clientdir())["tahoe"], "ascii") d.addCallback(_get_dircap) d.addCallback(lambda res: @@ -213,10 +223,10 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.basedir = "cli/Put/mutable" self.set_up_grid(oneshare=True) - DATA1 = "data" * 100 + DATA1 = b"data" * 100 fn1 = os.path.join(self.basedir, "DATA1") fileutil.write(fn1, DATA1) - DATA2 = "two" * 100 + DATA2 = b"two\xff" * 100 fn2 = os.path.join(self.basedir, "DATA2") fileutil.write(fn2, DATA2) @@ -238,7 +248,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessEqual(out, self.uri, str(res)) d.addCallback(_check2) d.addCallback(lambda res: - self.do_cli("get", "tahoe:uploaded.txt")) + self.do_cli("get", "tahoe:uploaded.txt", return_bytes=True)) d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA2)) return d @@ -438,7 +448,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) return d @@ -452,7 +462,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.basedir = "cli/Put/immutable_from_file_unicode" self.set_up_grid(oneshare=True) - rel_fn = os.path.join(unicode(self.basedir), u"\u00E0 trier.txt") + rel_fn = os.path.join(str(self.basedir), u"\u00E0 trier.txt") # we make the file small enough to fit in a LIT file, for speed DATA = b"short file \xff bytes" fileutil.write(rel_fn, DATA) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 1e6111e6c..6695b685f 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -181,6 +181,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.cli.test_create", "allmydata.test.cli.test_create_alias", "allmydata.test.cli.test_invite", + "allmydata.test.cli.test_put", "allmydata.test.cli.test_status", "allmydata.test.mutable.test_checker", From b3ede6b9f232338b0d5ed6a3f4650d7648df2ab9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 May 2021 10:25:48 -0400 Subject: [PATCH 210/463] Nicer way to say the same thing. --- src/allmydata/scripts/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 826c36e1f..645b57f41 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -224,7 +224,7 @@ class CpOptions(FileStoreOptions): def parseArgs(self, *args): if len(args) < 2: raise usage.UsageError("cp requires at least two arguments") - self.sources = list(map(argv_to_unicode, args[:-1])) + self.sources = [argv_to_unicode(arg) for arg in args[:-1]] self.destination = argv_to_unicode(args[-1]) synopsis = "[options] FROM.. TO" From 99543877d6d0e45c55230249548749fb1e3e5666 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 May 2021 10:27:43 -0400 Subject: [PATCH 211/463] Fix flake. --- src/allmydata/test/cli/test_create_alias.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_create_alias.py b/src/allmydata/test/cli/test_create_alias.py index e82ecf60c..4a252f372 100644 --- a/src/allmydata/test/cli/test_create_alias.py +++ b/src/allmydata/test/cli/test_create_alias.py @@ -20,7 +20,7 @@ from allmydata.util import fileutil from allmydata.scripts.common import get_aliases from allmydata.scripts import cli, runner from ..no_network import GridTestMixin -from allmydata.util.encodingutil import quote_output, get_io_encoding +from allmydata.util.encodingutil import quote_output from .common import CLITestMixin class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): From c275f9ae54a5d256655824ec1099b4610a63cd60 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 May 2021 10:47:30 -0400 Subject: [PATCH 212/463] Tests pass on Python 3. --- src/allmydata/scripts/tahoe_ls.py | 3 +- src/allmydata/test/cli/test_list.py | 62 +++++++++++++---------------- 2 files changed, 30 insertions(+), 35 deletions(-) diff --git a/src/allmydata/scripts/tahoe_ls.py b/src/allmydata/scripts/tahoe_ls.py index 71db3f5cb..92d5adfef 100644 --- a/src/allmydata/scripts/tahoe_ls.py +++ b/src/allmydata/scripts/tahoe_ls.py @@ -64,13 +64,14 @@ def list(options): print(quote_output(data, quotemarks=False), file=stderr) return 1 + path = unicode(path, "utf-8") nodetype, d = parsed children = {} if nodetype == "dirnode": children = d['children'] else: # paths returned from get_alias are always valid UTF-8 - childname = path.split("/")[-1].decode('utf-8') + childname = path.split("/")[-1] children = {childname: (nodetype, d)} if "metadata" not in d: d["metadata"] = {} diff --git a/src/allmydata/test/cli/test_list.py b/src/allmydata/test/cli/test_list.py index fff57cdc9..68571b49b 100644 --- a/src/allmydata/test/cli/test_list.py +++ b/src/allmydata/test/cli/test_list.py @@ -1,3 +1,6 @@ +from future.utils import PY3 +from past.builtins import unicode + from twisted.trial import unittest from twisted.internet import defer @@ -8,30 +11,26 @@ from ..no_network import GridTestMixin from allmydata.util.encodingutil import quote_output, get_io_encoding from .common import CLITestMixin + class List(GridTestMixin, CLITestMixin, unittest.TestCase): def test_list(self): self.basedir = "cli/List/list" self.set_up_grid() c0 = self.g.clients[0] - small = "small" + small = b"small" - # u"g\u00F6\u00F6d" might not be representable in the argv and/or output encodings. - # It is initially included in the directory in any case. - try: - good_arg = u"g\u00F6\u00F6d".encode(get_io_encoding()) - except UnicodeEncodeError: - good_arg = None + good_arg = u"g\u00F6\u00F6d" + good_out = u"g\u00F6\u00F6d" - try: - good_out = u"g\u00F6\u00F6d".encode(get_io_encoding()) - except UnicodeEncodeError: - good_out = None + # On Python 2 we get bytes, so we need encoded version. On Python 3 + # stdio is unicode so can leave unchanged. + good_out_encoded = good_out if PY3 else good_out.encode(get_io_encoding()) d = c0.create_dirnode() def _stash_root_and_create_file(n): self.rootnode = n - self.rooturi = n.get_uri() - return n.add_file(u"g\u00F6\u00F6d", upload.Data(small, convergence="")) + self.rooturi = unicode(n.get_uri(), "utf-8") + return n.add_file(u"g\u00F6\u00F6d", upload.Data(small, convergence=b"")) d.addCallback(_stash_root_and_create_file) def _stash_goodcap(n): self.goodcap = n.get_uri() @@ -47,15 +46,10 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("ls")) def _check1(args): (rc, out, err) = args - if good_out is None: - self.failUnlessReallyEqual(rc, 1) - self.failUnlessIn("files whose names could not be converted", err) - self.failUnlessIn(quote_output(u"g\u00F6\u00F6d"), err) - self.failUnlessReallyEqual(sorted(out.splitlines()), sorted(["0share", "1share"])) - else: - self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") - self.failUnlessReallyEqual(sorted(out.splitlines()), sorted(["0share", "1share", good_out])) + self.failUnlessReallyEqual(rc, 0) + self.assertEqual(len(err), 0, err) + self.failUnlessReallyEqual(sorted(out.splitlines()), + sorted(["0share", "1share", good_out_encoded])) d.addCallback(_check1) d.addCallback(lambda ign: self.do_cli("ls", "missing")) def _check2(args): @@ -87,7 +81,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): # listing a file (as dir/filename) should have the edge metadata, # including the filename self.failUnlessReallyEqual(rc, 0) - self.failUnlessIn(good_out, out) + self.failUnlessIn(good_out_encoded, out) self.failIfIn("-r-- %d -" % len(small), out, "trailing hyphen means unknown date") @@ -139,7 +133,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("ls", "-l", self.rooturi + ":./good")) d.addCallback(_check4_ascii) - unknown_immcap = "imm.URI:unknown" + unknown_immcap = b"imm.URI:unknown" def _create_unknown(ign): nm = c0.nodemaker kids = {u"unknownchild-imm": (nm.create_from_cap(unknown_immcap), {})} @@ -226,8 +220,8 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): # The uploaders may run at the same time, so we need two # MutableData instances or they'll fight over offsets &c and # break. - mutable_data = MutableData("data" * 100000) - mutable_data2 = MutableData("data" * 100000) + mutable_data = MutableData(b"data" * 100000) + mutable_data2 = MutableData(b"data" * 100000) # Add both kinds of mutable node. d1 = nm.create_mutable_file(mutable_data, version=MDMF_VERSION) @@ -235,8 +229,8 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): version=SDMF_VERSION) # Add an immutable node. We do this through the directory, # with add_file. - immutable_data = upload.Data("immutable data" * 100000, - convergence="") + immutable_data = upload.Data(b"immutable data" * 100000, + convergence=b"") d3 = n.add_file(u"immutable", immutable_data) ds = [d1, d2, d3] dl = defer.DeferredList(ds) @@ -294,12 +288,12 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): def _got_json(args): (rc, out, err) = args self.failUnlessEqual(rc, 0) - self.failUnlessEqual(err, "") - self.failUnlessIn(self._mdmf_uri, out) - self.failUnlessIn(self._mdmf_readonly_uri, out) - self.failUnlessIn(self._sdmf_uri, out) - self.failUnlessIn(self._sdmf_readonly_uri, out) - self.failUnlessIn(self._imm_uri, out) + self.assertEqual(len(err), 0, err) + self.failUnlessIn(unicode(self._mdmf_uri, "ascii"), out) + self.failUnlessIn(unicode(self._mdmf_readonly_uri, "ascii"), out) + self.failUnlessIn(unicode(self._sdmf_uri, "ascii"), out) + self.failUnlessIn(unicode(self._sdmf_readonly_uri, "ascii"), out) + self.failUnlessIn(unicode(self._imm_uri, "ascii"), out) self.failUnlessIn('"format": "SDMF"', out) self.failUnlessIn('"format": "MDMF"', out) d.addCallback(_got_json) From 2b751c44dbf3091ba1b213e0298d8a0b75fbbf15 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 May 2021 11:10:54 -0400 Subject: [PATCH 213/463] Port to Python 3. --- src/allmydata/test/cli/test_list.py | 52 +++++++++++++++++------------ src/allmydata/util/_python3.py | 1 + 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/src/allmydata/test/cli/test_list.py b/src/allmydata/test/cli/test_list.py index 68571b49b..1206579f1 100644 --- a/src/allmydata/test/cli/test_list.py +++ b/src/allmydata/test/cli/test_list.py @@ -1,5 +1,15 @@ -from future.utils import PY3 -from past.builtins import unicode +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2, PY3 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from six import ensure_str from twisted.trial import unittest from twisted.internet import defer @@ -29,7 +39,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): d = c0.create_dirnode() def _stash_root_and_create_file(n): self.rootnode = n - self.rooturi = unicode(n.get_uri(), "utf-8") + self.rooturi = str(n.get_uri(), "utf-8") return n.add_file(u"g\u00F6\u00F6d", upload.Data(small, convergence=b"")) d.addCallback(_stash_root_and_create_file) def _stash_goodcap(n): @@ -37,10 +47,10 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(_stash_goodcap) d.addCallback(lambda ign: self.rootnode.create_subdirectory(u"1share")) d.addCallback(lambda n: - self.delete_shares_numbered(n.get_uri(), range(1,10))) + self.delete_shares_numbered(n.get_uri(), list(range(1,10)))) d.addCallback(lambda ign: self.rootnode.create_subdirectory(u"0share")) d.addCallback(lambda n: - self.delete_shares_numbered(n.get_uri(), range(0,10))) + self.delete_shares_numbered(n.get_uri(), list(range(0,10)))) d.addCallback(lambda ign: self.do_cli("add-alias", "tahoe", self.rooturi)) d.addCallback(lambda ign: self.do_cli("ls")) @@ -48,15 +58,15 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) self.assertEqual(len(err), 0, err) - self.failUnlessReallyEqual(sorted(out.splitlines()), - sorted(["0share", "1share", good_out_encoded])) + expected = sorted([ensure_str("0share"), ensure_str("1share"), good_out_encoded]) + self.assertEqual(sorted(out.splitlines()), expected) d.addCallback(_check1) d.addCallback(lambda ign: self.do_cli("ls", "missing")) def _check2(args): (rc, out, err) = args self.failIfEqual(rc, 0) - self.failUnlessReallyEqual(err.strip(), "No such file or directory") - self.failUnlessReallyEqual(out, "") + self.assertEqual(err.strip(), "No such file or directory") + self.assertEqual(len(out), 0, out) d.addCallback(_check2) d.addCallback(lambda ign: self.do_cli("ls", "1share")) def _check3(args): @@ -66,7 +76,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessIn("UnrecoverableFileError:", err) self.failUnlessIn("could not be retrieved, because there were " "insufficient good shares.", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check3) d.addCallback(lambda ign: self.do_cli("ls", "0share")) d.addCallback(_check3) @@ -76,13 +86,13 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("files whose names could not be converted", err) self.failUnlessIn(quote_output(u"g\u00F6\u00F6d"), err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) else: # listing a file (as dir/filename) should have the edge metadata, # including the filename self.failUnlessReallyEqual(rc, 0) self.failUnlessIn(good_out_encoded, out) - self.failIfIn("-r-- %d -" % len(small), out, + self.failIfIn(ensure_str("-r-- %d -" % len(small)), out, "trailing hyphen means unknown date") if good_arg is not None: @@ -100,7 +110,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): # metadata, just the size (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual("-r-- %d -" % len(small), out.strip()) + self.assertEqual("-r-- %d -" % len(small), out.strip()) d.addCallback(lambda ign: self.do_cli("ls", "-l", self.goodcap)) d.addCallback(_check5) @@ -112,7 +122,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): def _check1_ascii(args): (rc,out,err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(sorted(out.splitlines()), sorted(["0share", "1share", "good"])) d.addCallback(_check1_ascii) def _check4_ascii(args): @@ -172,7 +182,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) return d @@ -187,7 +197,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) return d @@ -289,11 +299,11 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessEqual(rc, 0) self.assertEqual(len(err), 0, err) - self.failUnlessIn(unicode(self._mdmf_uri, "ascii"), out) - self.failUnlessIn(unicode(self._mdmf_readonly_uri, "ascii"), out) - self.failUnlessIn(unicode(self._sdmf_uri, "ascii"), out) - self.failUnlessIn(unicode(self._sdmf_readonly_uri, "ascii"), out) - self.failUnlessIn(unicode(self._imm_uri, "ascii"), out) + self.failUnlessIn(str(self._mdmf_uri, "ascii"), out) + self.failUnlessIn(str(self._mdmf_readonly_uri, "ascii"), out) + self.failUnlessIn(str(self._sdmf_uri, "ascii"), out) + self.failUnlessIn(str(self._sdmf_readonly_uri, "ascii"), out) + self.failUnlessIn(str(self._imm_uri, "ascii"), out) self.failUnlessIn('"format": "SDMF"', out) self.failUnlessIn('"format": "MDMF"', out) d.addCallback(_got_json) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 6695b685f..400a0ea9e 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -181,6 +181,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.cli.test_create", "allmydata.test.cli.test_create_alias", "allmydata.test.cli.test_invite", + "allmydata.test.cli.test_list", "allmydata.test.cli.test_put", "allmydata.test.cli.test_status", From 9dcfa2171ef11e8e7ed111a6613840d52d965fb3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 May 2021 11:13:27 -0400 Subject: [PATCH 214/463] Tests pass on Python 3. --- src/allmydata/scripts/tahoe_mv.py | 3 +++ src/allmydata/test/cli/test_mv.py | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/allmydata/scripts/tahoe_mv.py b/src/allmydata/scripts/tahoe_mv.py index bdaf134ff..580475c91 100644 --- a/src/allmydata/scripts/tahoe_mv.py +++ b/src/allmydata/scripts/tahoe_mv.py @@ -1,5 +1,7 @@ from __future__ import print_function +from past.builtins import unicode + import re from urllib.parse import quote as url_quote import json @@ -47,6 +49,7 @@ def mv(options, mode="move"): if path: to_url += "/" + escape_path(path) + from_path = unicode(from_path, "utf-8") if to_url.endswith("/"): # "mv foo.txt bar/" == "mv foo.txt bar/foo.txt" to_url += escape_path(from_path[from_path.rfind("/")+1:]) diff --git a/src/allmydata/test/cli/test_mv.py b/src/allmydata/test/cli/test_mv.py index 9d1a64974..21378c353 100644 --- a/src/allmydata/test/cli/test_mv.py +++ b/src/allmydata/test/cli/test_mv.py @@ -10,10 +10,10 @@ class Mv(GridTestMixin, CLITestMixin, unittest.TestCase): self.basedir = "cli/Mv/mv_behavior" self.set_up_grid(oneshare=True) fn1 = os.path.join(self.basedir, "file1") - DATA1 = "Nuclear launch codes" + DATA1 = b"Nuclear launch codes" fileutil.write(fn1, DATA1) fn2 = os.path.join(self.basedir, "file2") - DATA2 = "UML diagrams" + DATA2 = b"UML diagrams" fileutil.write(fn2, DATA2) # copy both files to the grid d = self.do_cli("create-alias", "tahoe") @@ -104,11 +104,11 @@ class Mv(GridTestMixin, CLITestMixin, unittest.TestCase): self.basedir = "cli/Mv/mv_error_if_DELETE_fails" self.set_up_grid(oneshare=True) fn1 = os.path.join(self.basedir, "file1") - DATA1 = "Nuclear launch codes" + DATA1 = b"Nuclear launch codes" fileutil.write(fn1, DATA1) original_do_http = tahoe_mv.do_http - def mock_do_http(method, url, body=""): + def mock_do_http(method, url, body=b""): if method == "DELETE": class FakeResponse(object): def read(self): From a7d4fed1bada5abcc96e0dfb8b15746585f2d41b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 May 2021 11:15:10 -0400 Subject: [PATCH 215/463] Port to Python 3. --- src/allmydata/test/cli/test_mv.py | 17 +++++++++++++++-- src/allmydata/util/_python3.py | 1 + 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_mv.py b/src/allmydata/test/cli/test_mv.py index 21378c353..0bb9ba369 100644 --- a/src/allmydata/test/cli/test_mv.py +++ b/src/allmydata/test/cli/test_mv.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os.path from twisted.trial import unittest from allmydata.util import fileutil @@ -5,6 +17,7 @@ from ..no_network import GridTestMixin from allmydata.scripts import tahoe_mv from .common import CLITestMixin + class Mv(GridTestMixin, CLITestMixin, unittest.TestCase): def test_mv_behavior(self): self.basedir = "cli/Mv/mv_behavior" @@ -152,7 +165,7 @@ class Mv(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) # check to see that the validation extends to the # target argument by making an alias that will work with the first @@ -180,7 +193,7 @@ class Mv(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("fake", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) # check to see that the validation extends to the # target argument by making an alias that will work with the first diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 400a0ea9e..919192452 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -182,6 +182,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.cli.test_create_alias", "allmydata.test.cli.test_invite", "allmydata.test.cli.test_list", + "allmydata.test.cli.test_mv", "allmydata.test.cli.test_put", "allmydata.test.cli.test_status", From 02edef01a9e28ee853ce844761596946deb352e9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 May 2021 11:15:24 -0400 Subject: [PATCH 216/463] News file. --- newsfragments/3691.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3691.minor diff --git a/newsfragments/3691.minor b/newsfragments/3691.minor new file mode 100644 index 000000000..e69de29bb From f9ae91a94ec5b078c48440be047ab318a159f015 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 May 2021 11:20:16 -0400 Subject: [PATCH 217/463] Tests pass on Python 3. --- src/allmydata/test/cli/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index d27791f34..0e12483dd 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -50,7 +50,7 @@ class DaemonizeTheRealServiceTests(SyncTestCase): """ nodedir = FilePath(self.mktemp()) nodedir.makedirs() - nodedir.child("tahoe.cfg").setContent(config) + nodedir.child("tahoe.cfg").setContent(config.encode("ascii")) nodedir.child("tahoe-client.tac").touch() options = parse_options(["run", nodedir.path]) From 1f70d5c13a5d094a32f2f38fb699fecfefdbc40e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 May 2021 11:21:23 -0400 Subject: [PATCH 218/463] Port to Python 3. --- src/allmydata/test/cli/test_run.py | 10 ++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 11 insertions(+) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index 0e12483dd..6100d2568 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -1,6 +1,16 @@ """ Tests for ``allmydata.scripts.tahoe_run``. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six.moves import ( StringIO, diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 919192452..bbafc144f 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -184,6 +184,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.cli.test_list", "allmydata.test.cli.test_mv", "allmydata.test.cli.test_put", + "allmydata.test.cli.test_run", "allmydata.test.cli.test_status", "allmydata.test.mutable.test_checker", From d3be3ce1e6978976dc800a98b80c4af27d600c3b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 May 2021 11:43:31 -0400 Subject: [PATCH 219/463] Start making tests pass on Python 3. --- src/allmydata/scripts/admin.py | 15 ++++++++++----- src/allmydata/test/cli/test_cli.py | 11 +++++++---- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 50dde9e43..0c860e93a 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -1,5 +1,7 @@ from __future__ import print_function +from past.builtins import unicode + try: from allmydata.scripts.types_ import SubCommands except ImportError: @@ -22,8 +24,10 @@ def print_keypair(options): from allmydata.crypto import ed25519 out = options.stdout private_key, public_key = ed25519.create_signing_keypair() - print("private:", ed25519.string_from_signing_key(private_key), file=out) - print("public:", ed25519.string_from_verifying_key(public_key), file=out) + print("private:", unicode(ed25519.string_from_signing_key(private_key), "ascii"), + file=out) + print("public:", unicode(ed25519.string_from_verifying_key(public_key), "ascii"), + file=out) class DerivePubkeyOptions(BaseOptions): def parseArgs(self, privkey): @@ -45,9 +49,10 @@ def derive_pubkey(options): out = options.stdout from allmydata.crypto import ed25519 privkey_vs = options.privkey - private_key, public_key = ed25519.signing_keypair_from_string(privkey_vs) - print("private:", ed25519.string_from_signing_key(private_key), file=out) - print("public:", ed25519.string_from_verifying_key(public_key), file=out) + private_key, public_key = ed25519.signing_keypair_from_string( + privkey_vs.encode("ascii")) + print("private:", unicode(ed25519.string_from_signing_key(private_key), "ascii"), file=out) + print("public:", unicode(ed25519.string_from_verifying_key(public_key), "ascii"), file=out) return 0 class AdminCommand(BaseOptions): diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index 2b1bc1c86..2e3c4d86a 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -1,3 +1,5 @@ +from past.builtins import unicode + import os.path from six.moves import cStringIO as StringIO import urllib, sys @@ -718,8 +720,9 @@ class Admin(unittest.TestCase): self.failUnlessEqual(pubkey_bits[0], vk_header, lines[1]) self.failUnless(privkey_bits[1].startswith("priv-v0-"), lines[0]) self.failUnless(pubkey_bits[1].startswith("pub-v0-"), lines[1]) - sk, pk = ed25519.signing_keypair_from_string(privkey_bits[1]) - vk_bytes = pubkey_bits[1] + sk, pk = ed25519.signing_keypair_from_string( + privkey_bits[1].encode("ascii")) + vk_bytes = pubkey_bits[1].encode("ascii") self.assertEqual( ed25519.string_from_verifying_key(pk), vk_bytes, @@ -729,8 +732,8 @@ class Admin(unittest.TestCase): def test_derive_pubkey(self): priv_key, pub_key = ed25519.create_signing_keypair() - priv_key_str = ed25519.string_from_signing_key(priv_key) - pub_key_str = ed25519.string_from_verifying_key(pub_key) + priv_key_str = unicode(ed25519.string_from_signing_key(priv_key), "ascii") + pub_key_str = unicode(ed25519.string_from_verifying_key(pub_key), "ascii") d = run_cli("admin", "derive-pubkey", priv_key_str) def _done(args): (rc, stdout, stderr) = args From ae739dfd9e1f81d4b5b62a1e0570b24305827c50 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 May 2021 11:48:02 -0400 Subject: [PATCH 220/463] Python 3 updates. --- src/allmydata/test/cli/test_cli.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index 2e3c4d86a..d08d46a34 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -363,11 +363,11 @@ class CLI(CLITestMixin, unittest.TestCase): "didn't see 'mqfblse6m5a6dh45isu2cg7oji' in '%s'" % err) def test_alias(self): - def s128(c): return base32.b2a(c*(128/8)) - def s256(c): return base32.b2a(c*(256/8)) - TA = "URI:DIR2:%s:%s" % (s128("T"), s256("T")) - WA = "URI:DIR2:%s:%s" % (s128("W"), s256("W")) - CA = "URI:DIR2:%s:%s" % (s128("C"), s256("C")) + def s128(c): return base32.b2a(c*(128//8)) + def s256(c): return base32.b2a(c*(256//8)) + TA = b"URI:DIR2:%s:%s" % (s128(b"T"), s256(b"T")) + WA = b"URI:DIR2:%s:%s" % (s128(b"W"), s256(b"W")) + CA = b"URI:DIR2:%s:%s" % (s128(b"C"), s256(b"C")) aliases = {"tahoe": TA, "work": WA, "c": CA} From 7349855ce4a417fcce23f9192354e733803f4bd1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 May 2021 10:00:27 -0400 Subject: [PATCH 221/463] Move unicode conversion higher up. --- src/allmydata/scripts/tahoe_ls.py | 3 ++- src/allmydata/scripts/tahoe_mv.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/tahoe_ls.py b/src/allmydata/scripts/tahoe_ls.py index 92d5adfef..b3e09b699 100644 --- a/src/allmydata/scripts/tahoe_ls.py +++ b/src/allmydata/scripts/tahoe_ls.py @@ -27,6 +27,8 @@ def list(options): except UnknownAliasError as e: e.display(stderr) return 1 + + path = unicode(path, "utf-8") url = nodeurl + "uri/%s" % url_quote(rootcap) if path: # move where.endswith check here? @@ -64,7 +66,6 @@ def list(options): print(quote_output(data, quotemarks=False), file=stderr) return 1 - path = unicode(path, "utf-8") nodetype, d = parsed children = {} if nodetype == "dirnode": diff --git a/src/allmydata/scripts/tahoe_mv.py b/src/allmydata/scripts/tahoe_mv.py index 580475c91..84f83edcd 100644 --- a/src/allmydata/scripts/tahoe_mv.py +++ b/src/allmydata/scripts/tahoe_mv.py @@ -27,6 +27,7 @@ def mv(options, mode="move"): except UnknownAliasError as e: e.display(stderr) return 1 + from_path = unicode(from_path, "utf-8") from_url = nodeurl + "uri/%s" % url_quote(rootcap) if from_path: from_url += "/" + escape_path(from_path) @@ -46,10 +47,10 @@ def mv(options, mode="move"): e.display(stderr) return 1 to_url = nodeurl + "uri/%s" % url_quote(rootcap) + path = unicode(path, "utf-8") if path: to_url += "/" + escape_path(path) - from_path = unicode(from_path, "utf-8") if to_url.endswith("/"): # "mv foo.txt bar/" == "mv foo.txt bar/foo.txt" to_url += escape_path(from_path[from_path.rfind("/")+1:]) From 91f8575d29a64c8702095c22865e738da3e775dd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 May 2021 10:04:45 -0400 Subject: [PATCH 222/463] News file --- newsfragments/3692.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3692.minor diff --git a/newsfragments/3692.minor b/newsfragments/3692.minor new file mode 100644 index 000000000..e69de29bb From 75deef906d849255d0ffd9b1efadc7121c3ad2d9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 May 2021 10:12:07 -0400 Subject: [PATCH 223/463] More progress towards running tests on Python 3. --- src/allmydata/test/cli/test_cli.py | 129 +++++++++++++++-------------- 1 file changed, 65 insertions(+), 64 deletions(-) diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index d08d46a34..405340115 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -2,9 +2,10 @@ from past.builtins import unicode import os.path from six.moves import cStringIO as StringIO -import urllib, sys +import sys import re from mock import patch, Mock +from urllib.parse import quote as url_quote from twisted.trial import unittest from twisted.python.monkey import MonkeyPatcher @@ -55,8 +56,8 @@ class CLI(CLITestMixin, unittest.TestCase): return output def test_dump_cap_chk(self): - key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" - uri_extension_hash = hashutil.uri_extension_hash("stuff") + key = b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + uri_extension_hash = hashutil.uri_extension_hash(b"stuff") needed_shares = 25 total_shares = 100 size = 1234 @@ -65,7 +66,7 @@ class CLI(CLITestMixin, unittest.TestCase): needed_shares=needed_shares, total_shares=total_shares, size=size) - output = self._dump_cap(u.to_string()) + output = self._dump_cap(unicode(u.to_string(), "ascii")) self.failUnless("CHK File:" in output, output) self.failUnless("key: aaaqeayeaudaocajbifqydiob4" in output, output) self.failUnless("UEB hash: nf3nimquen7aeqm36ekgxomalstenpkvsdmf6fplj7swdatbv5oa" in output, output) @@ -84,7 +85,7 @@ class CLI(CLITestMixin, unittest.TestCase): self.failUnless("k/N: 25/100" in output, output) self.failUnless("storage index: hdis5iaveku6lnlaiccydyid7q" in output, output) - prefixed_u = "http://127.0.0.1/uri/%s" % urllib.quote(u.to_string()) + prefixed_u = "http://127.0.0.1/uri/%s" % url_quote(u.to_string()) output = self._dump_cap(prefixed_u) self.failUnless("CHK File:" in output, output) self.failUnless("key: aaaqeayeaudaocajbifqydiob4" in output, output) @@ -374,60 +375,60 @@ class CLI(CLITestMixin, unittest.TestCase): def ga1(path): return get_alias(aliases, path, u"tahoe") uses_lettercolon = common.platform_uses_lettercolon_drivename() - self.failUnlessReallyEqual(ga1(u"bare"), (TA, "bare")) - self.failUnlessReallyEqual(ga1(u"baredir/file"), (TA, "baredir/file")) - self.failUnlessReallyEqual(ga1(u"baredir/file:7"), (TA, "baredir/file:7")) - self.failUnlessReallyEqual(ga1(u"tahoe:"), (TA, "")) - self.failUnlessReallyEqual(ga1(u"tahoe:file"), (TA, "file")) - self.failUnlessReallyEqual(ga1(u"tahoe:dir/file"), (TA, "dir/file")) - self.failUnlessReallyEqual(ga1(u"work:"), (WA, "")) - self.failUnlessReallyEqual(ga1(u"work:file"), (WA, "file")) - self.failUnlessReallyEqual(ga1(u"work:dir/file"), (WA, "dir/file")) + self.failUnlessReallyEqual(ga1(u"bare"), (TA, b"bare")) + self.failUnlessReallyEqual(ga1(u"baredir/file"), (TA, b"baredir/file")) + self.failUnlessReallyEqual(ga1(u"baredir/file:7"), (TA, b"baredir/file:7")) + self.failUnlessReallyEqual(ga1(u"tahoe:"), (TA, b"")) + self.failUnlessReallyEqual(ga1(u"tahoe:file"), (TA, b"file")) + self.failUnlessReallyEqual(ga1(u"tahoe:dir/file"), (TA, b"dir/file")) + self.failUnlessReallyEqual(ga1(u"work:"), (WA, b"")) + self.failUnlessReallyEqual(ga1(u"work:file"), (WA, b"file")) + self.failUnlessReallyEqual(ga1(u"work:dir/file"), (WA, b"dir/file")) # default != None means we really expect a tahoe path, regardless of # whether we're on windows or not. This is what 'tahoe get' uses. - self.failUnlessReallyEqual(ga1(u"c:"), (CA, "")) - self.failUnlessReallyEqual(ga1(u"c:file"), (CA, "file")) - self.failUnlessReallyEqual(ga1(u"c:dir/file"), (CA, "dir/file")) - self.failUnlessReallyEqual(ga1(u"URI:stuff"), ("URI:stuff", "")) - self.failUnlessReallyEqual(ga1(u"URI:stuff/file"), ("URI:stuff", "file")) - self.failUnlessReallyEqual(ga1(u"URI:stuff:./file"), ("URI:stuff", "file")) - self.failUnlessReallyEqual(ga1(u"URI:stuff/dir/file"), ("URI:stuff", "dir/file")) - self.failUnlessReallyEqual(ga1(u"URI:stuff:./dir/file"), ("URI:stuff", "dir/file")) + self.failUnlessReallyEqual(ga1(u"c:"), (CA, b"")) + self.failUnlessReallyEqual(ga1(u"c:file"), (CA, b"file")) + self.failUnlessReallyEqual(ga1(u"c:dir/file"), (CA, b"dir/file")) + self.failUnlessReallyEqual(ga1(u"URI:stuff"), (b"URI:stuff", b"")) + self.failUnlessReallyEqual(ga1(u"URI:stuff/file"), (b"URI:stuff", b"file")) + self.failUnlessReallyEqual(ga1(u"URI:stuff:./file"), (b"URI:stuff", b"file")) + self.failUnlessReallyEqual(ga1(u"URI:stuff/dir/file"), (b"URI:stuff", b"dir/file")) + self.failUnlessReallyEqual(ga1(u"URI:stuff:./dir/file"), (b"URI:stuff", b"dir/file")) self.failUnlessRaises(common.UnknownAliasError, ga1, u"missing:") self.failUnlessRaises(common.UnknownAliasError, ga1, u"missing:dir") self.failUnlessRaises(common.UnknownAliasError, ga1, u"missing:dir/file") def ga2(path): return get_alias(aliases, path, None) - self.failUnlessReallyEqual(ga2(u"bare"), (DefaultAliasMarker, "bare")) + self.failUnlessReallyEqual(ga2(u"bare"), (DefaultAliasMarker, b"bare")) self.failUnlessReallyEqual(ga2(u"baredir/file"), - (DefaultAliasMarker, "baredir/file")) + (DefaultAliasMarker, b"baredir/file")) self.failUnlessReallyEqual(ga2(u"baredir/file:7"), - (DefaultAliasMarker, "baredir/file:7")) + (DefaultAliasMarker, b"baredir/file:7")) self.failUnlessReallyEqual(ga2(u"baredir/sub:1/file:7"), - (DefaultAliasMarker, "baredir/sub:1/file:7")) - self.failUnlessReallyEqual(ga2(u"tahoe:"), (TA, "")) - self.failUnlessReallyEqual(ga2(u"tahoe:file"), (TA, "file")) - self.failUnlessReallyEqual(ga2(u"tahoe:dir/file"), (TA, "dir/file")) + (DefaultAliasMarker, b"baredir/sub:1/file:7")) + self.failUnlessReallyEqual(ga2(u"tahoe:"), (TA, b"")) + self.failUnlessReallyEqual(ga2(u"tahoe:file"), (TA, b"file")) + self.failUnlessReallyEqual(ga2(u"tahoe:dir/file"), (TA, b"dir/file")) # on windows, we really want c:foo to indicate a local file. # default==None is what 'tahoe cp' uses. if uses_lettercolon: - self.failUnlessReallyEqual(ga2(u"c:"), (DefaultAliasMarker, "c:")) - self.failUnlessReallyEqual(ga2(u"c:file"), (DefaultAliasMarker, "c:file")) + self.failUnlessReallyEqual(ga2(u"c:"), (DefaultAliasMarker, b"c:")) + self.failUnlessReallyEqual(ga2(u"c:file"), (DefaultAliasMarker, b"c:file")) self.failUnlessReallyEqual(ga2(u"c:dir/file"), - (DefaultAliasMarker, "c:dir/file")) + (DefaultAliasMarker, b"c:dir/file")) else: - self.failUnlessReallyEqual(ga2(u"c:"), (CA, "")) - self.failUnlessReallyEqual(ga2(u"c:file"), (CA, "file")) - self.failUnlessReallyEqual(ga2(u"c:dir/file"), (CA, "dir/file")) - self.failUnlessReallyEqual(ga2(u"work:"), (WA, "")) - self.failUnlessReallyEqual(ga2(u"work:file"), (WA, "file")) - self.failUnlessReallyEqual(ga2(u"work:dir/file"), (WA, "dir/file")) - self.failUnlessReallyEqual(ga2(u"URI:stuff"), ("URI:stuff", "")) - self.failUnlessReallyEqual(ga2(u"URI:stuff/file"), ("URI:stuff", "file")) - self.failUnlessReallyEqual(ga2(u"URI:stuff:./file"), ("URI:stuff", "file")) - self.failUnlessReallyEqual(ga2(u"URI:stuff/dir/file"), ("URI:stuff", "dir/file")) - self.failUnlessReallyEqual(ga2(u"URI:stuff:./dir/file"), ("URI:stuff", "dir/file")) + self.failUnlessReallyEqual(ga2(u"c:"), (CA, b"")) + self.failUnlessReallyEqual(ga2(u"c:file"), (CA, b"file")) + self.failUnlessReallyEqual(ga2(u"c:dir/file"), (CA, b"dir/file")) + self.failUnlessReallyEqual(ga2(u"work:"), (WA, b"")) + self.failUnlessReallyEqual(ga2(u"work:file"), (WA, b"file")) + self.failUnlessReallyEqual(ga2(u"work:dir/file"), (WA, b"dir/file")) + self.failUnlessReallyEqual(ga2(u"URI:stuff"), (b"URI:stuff", b"")) + self.failUnlessReallyEqual(ga2(u"URI:stuff/file"), (b"URI:stuff", b"file")) + self.failUnlessReallyEqual(ga2(u"URI:stuff:./file"), (b"URI:stuff", b"file")) + self.failUnlessReallyEqual(ga2(u"URI:stuff/dir/file"), (b"URI:stuff", b"dir/file")) + self.failUnlessReallyEqual(ga2(u"URI:stuff:./dir/file"), (b"URI:stuff", b"dir/file")) self.failUnlessRaises(common.UnknownAliasError, ga2, u"missing:") self.failUnlessRaises(common.UnknownAliasError, ga2, u"missing:dir") self.failUnlessRaises(common.UnknownAliasError, ga2, u"missing:dir/file") @@ -440,26 +441,26 @@ class CLI(CLITestMixin, unittest.TestCase): finally: common.pretend_platform_uses_lettercolon = old return retval - self.failUnlessReallyEqual(ga3(u"bare"), (DefaultAliasMarker, "bare")) + self.failUnlessReallyEqual(ga3(u"bare"), (DefaultAliasMarker, b"bare")) self.failUnlessReallyEqual(ga3(u"baredir/file"), - (DefaultAliasMarker, "baredir/file")) + (DefaultAliasMarker, b"baredir/file")) self.failUnlessReallyEqual(ga3(u"baredir/file:7"), - (DefaultAliasMarker, "baredir/file:7")) + (DefaultAliasMarker, b"baredir/file:7")) self.failUnlessReallyEqual(ga3(u"baredir/sub:1/file:7"), - (DefaultAliasMarker, "baredir/sub:1/file:7")) - self.failUnlessReallyEqual(ga3(u"tahoe:"), (TA, "")) - self.failUnlessReallyEqual(ga3(u"tahoe:file"), (TA, "file")) - self.failUnlessReallyEqual(ga3(u"tahoe:dir/file"), (TA, "dir/file")) - self.failUnlessReallyEqual(ga3(u"c:"), (DefaultAliasMarker, "c:")) - self.failUnlessReallyEqual(ga3(u"c:file"), (DefaultAliasMarker, "c:file")) + (DefaultAliasMarker, b"baredir/sub:1/file:7")) + self.failUnlessReallyEqual(ga3(u"tahoe:"), (TA, b"")) + self.failUnlessReallyEqual(ga3(u"tahoe:file"), (TA, b"file")) + self.failUnlessReallyEqual(ga3(u"tahoe:dir/file"), (TA, b"dir/file")) + self.failUnlessReallyEqual(ga3(u"c:"), (DefaultAliasMarker, b"c:")) + self.failUnlessReallyEqual(ga3(u"c:file"), (DefaultAliasMarker, b"c:file")) self.failUnlessReallyEqual(ga3(u"c:dir/file"), - (DefaultAliasMarker, "c:dir/file")) - self.failUnlessReallyEqual(ga3(u"work:"), (WA, "")) - self.failUnlessReallyEqual(ga3(u"work:file"), (WA, "file")) - self.failUnlessReallyEqual(ga3(u"work:dir/file"), (WA, "dir/file")) - self.failUnlessReallyEqual(ga3(u"URI:stuff"), ("URI:stuff", "")) - self.failUnlessReallyEqual(ga3(u"URI:stuff:./file"), ("URI:stuff", "file")) - self.failUnlessReallyEqual(ga3(u"URI:stuff:./dir/file"), ("URI:stuff", "dir/file")) + (DefaultAliasMarker, b"c:dir/file")) + self.failUnlessReallyEqual(ga3(u"work:"), (WA, b"")) + self.failUnlessReallyEqual(ga3(u"work:file"), (WA, b"file")) + self.failUnlessReallyEqual(ga3(u"work:dir/file"), (WA, b"dir/file")) + self.failUnlessReallyEqual(ga3(u"URI:stuff"), (b"URI:stuff", b"")) + self.failUnlessReallyEqual(ga3(u"URI:stuff:./file"), (b"URI:stuff", b"file")) + self.failUnlessReallyEqual(ga3(u"URI:stuff:./dir/file"), (b"URI:stuff", b"dir/file")) self.failUnlessRaises(common.UnknownAliasError, ga3, u"missing:") self.failUnlessRaises(common.UnknownAliasError, ga3, u"missing:dir") self.failUnlessRaises(common.UnknownAliasError, ga3, u"missing:dir/file") @@ -482,14 +483,14 @@ class CLI(CLITestMixin, unittest.TestCase): self.failUnlessRaises(common.UnknownAliasError, ga5, u"C:\\Windows") def test_alias_tolerance(self): - def s128(c): return base32.b2a(c*(128/8)) - def s256(c): return base32.b2a(c*(256/8)) - TA = "URI:DIR2:%s:%s" % (s128("T"), s256("T")) + def s128(c): return base32.b2a(c*(128//8)) + def s256(c): return base32.b2a(c*(256//8)) + TA = b"URI:DIR2:%s:%s" % (s128(b"T"), s256(b"T")) aliases = {"present": TA, - "future": "URI-FROM-FUTURE:ooh:aah"} + "future": b"URI-FROM-FUTURE:ooh:aah"} def ga1(path): return get_alias(aliases, path, u"tahoe") - self.failUnlessReallyEqual(ga1(u"present:file"), (TA, "file")) + self.failUnlessReallyEqual(ga1(u"present:file"), (TA, b"file")) # this throws, via assert IDirnodeURI.providedBy(), since get_alias() # wants a dirnode, and the future cap gives us UnknownURI instead. self.failUnlessRaises(AssertionError, ga1, u"future:stuff") From deaaa8c727fff53ca4e4f9c1b389feabd5b75108 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 May 2021 10:27:26 -0400 Subject: [PATCH 224/463] More tests passing on Python 3. --- src/allmydata/scripts/debug.py | 72 +++++++++++++++--------------- src/allmydata/test/cli/test_cli.py | 30 +++++++------ 2 files changed, 52 insertions(+), 50 deletions(-) diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index b8aeee91e..ba40519de 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -452,7 +452,7 @@ def dump_cap(options): from allmydata import uri from allmydata.util import base32 from base64 import b32decode - import urlparse, urllib + from urllib.parse import unquote, urlparse out = options.stdout cap = options.cap @@ -461,18 +461,18 @@ def dump_cap(options): nodeid = b32decode(options['nodeid'].upper()) secret = None if options['client-secret']: - secret = base32.a2b(options['client-secret']) + secret = base32.a2b(options['client-secret'].encode("ascii")) elif options['client-dir']: secretfile = os.path.join(options['client-dir'], "private", "secret") try: - secret = base32.a2b(open(secretfile, "r").read().strip()) + secret = base32.a2b(open(secretfile, "rb").read().strip()) except EnvironmentError: pass if cap.startswith("http"): - scheme, netloc, path, params, query, fragment = urlparse.urlparse(cap) + scheme, netloc, path, params, query, fragment = urlparse(cap) assert path.startswith("/uri/") - cap = urllib.unquote(path[len("/uri/"):]) + cap = unquote(path[len("/uri/"):]) u = uri.from_string(cap) @@ -485,19 +485,19 @@ def _dump_secrets(storage_index, secret, nodeid, out): if secret: crs = hashutil.my_renewal_secret_hash(secret) - print(" client renewal secret:", base32.b2a(crs), file=out) + print(" client renewal secret:", unicode(base32.b2a(crs), "ascii"), file=out) frs = hashutil.file_renewal_secret_hash(crs, storage_index) - print(" file renewal secret:", base32.b2a(frs), file=out) + print(" file renewal secret:", unicode(base32.b2a(frs), "ascii"), file=out) if nodeid: renew = hashutil.bucket_renewal_secret_hash(frs, nodeid) - print(" lease renewal secret:", base32.b2a(renew), file=out) + print(" lease renewal secret:", unicode(base32.b2a(renew), "ascii"), file=out) ccs = hashutil.my_cancel_secret_hash(secret) - print(" client cancel secret:", base32.b2a(ccs), file=out) + print(" client cancel secret:", unicode(base32.b2a(ccs), "ascii"), file=out) fcs = hashutil.file_cancel_secret_hash(ccs, storage_index) - print(" file cancel secret:", base32.b2a(fcs), file=out) + print(" file cancel secret:", unicode(base32.b2a(fcs), "ascii"), file=out) if nodeid: cancel = hashutil.bucket_cancel_secret_hash(fcs, nodeid) - print(" lease cancel secret:", base32.b2a(cancel), file=out) + print(" lease cancel secret:", unicode(base32.b2a(cancel), "ascii"), file=out) def dump_uri_instance(u, nodeid, secret, out, show_header=True): from allmydata import uri @@ -508,19 +508,19 @@ def dump_uri_instance(u, nodeid, secret, out, show_header=True): if isinstance(u, uri.CHKFileURI): if show_header: print("CHK File:", file=out) - print(" key:", base32.b2a(u.key), file=out) - print(" UEB hash:", base32.b2a(u.uri_extension_hash), file=out) + print(" key:", unicode(base32.b2a(u.key), "ascii"), file=out) + print(" UEB hash:", unicode(base32.b2a(u.uri_extension_hash), "ascii"), file=out) print(" size:", u.size, file=out) print(" k/N: %d/%d" % (u.needed_shares, u.total_shares), file=out) - print(" storage index:", si_b2a(u.get_storage_index()), file=out) + print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) _dump_secrets(u.get_storage_index(), secret, nodeid, out) elif isinstance(u, uri.CHKFileVerifierURI): if show_header: print("CHK Verifier URI:", file=out) - print(" UEB hash:", base32.b2a(u.uri_extension_hash), file=out) + print(" UEB hash:", unicode(base32.b2a(u.uri_extension_hash), "ascii"), file=out) print(" size:", u.size, file=out) print(" k/N: %d/%d" % (u.needed_shares, u.total_shares), file=out) - print(" storage index:", si_b2a(u.get_storage_index()), file=out) + print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) elif isinstance(u, uri.LiteralFileURI): if show_header: @@ -530,52 +530,52 @@ def dump_uri_instance(u, nodeid, secret, out, show_header=True): elif isinstance(u, uri.WriteableSSKFileURI): # SDMF if show_header: print("SDMF Writeable URI:", file=out) - print(" writekey:", base32.b2a(u.writekey), file=out) - print(" readkey:", base32.b2a(u.readkey), file=out) - print(" storage index:", si_b2a(u.get_storage_index()), file=out) - print(" fingerprint:", base32.b2a(u.fingerprint), file=out) + print(" writekey:", unicode(base32.b2a(u.writekey), "ascii"), file=out) + print(" readkey:", unicode(base32.b2a(u.readkey), "ascii"), file=out) + print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out) print(file=out) if nodeid: we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid) - print(" write_enabler:", base32.b2a(we), file=out) + print(" write_enabler:", unicode(base32.b2a(we), "ascii"), file=out) print(file=out) _dump_secrets(u.get_storage_index(), secret, nodeid, out) elif isinstance(u, uri.ReadonlySSKFileURI): if show_header: print("SDMF Read-only URI:", file=out) - print(" readkey:", base32.b2a(u.readkey), file=out) - print(" storage index:", si_b2a(u.get_storage_index()), file=out) - print(" fingerprint:", base32.b2a(u.fingerprint), file=out) + print(" readkey:", unicode(base32.b2a(u.readkey), "ascii"), file=out) + print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out) elif isinstance(u, uri.SSKVerifierURI): if show_header: print("SDMF Verifier URI:", file=out) - print(" storage index:", si_b2a(u.get_storage_index()), file=out) - print(" fingerprint:", base32.b2a(u.fingerprint), file=out) + print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out) elif isinstance(u, uri.WriteableMDMFFileURI): # MDMF if show_header: print("MDMF Writeable URI:", file=out) - print(" writekey:", base32.b2a(u.writekey), file=out) - print(" readkey:", base32.b2a(u.readkey), file=out) - print(" storage index:", si_b2a(u.get_storage_index()), file=out) - print(" fingerprint:", base32.b2a(u.fingerprint), file=out) + print(" writekey:", unicode(base32.b2a(u.writekey), "ascii"), file=out) + print(" readkey:", unicode(base32.b2a(u.readkey), "ascii"), file=out) + print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out) print(file=out) if nodeid: we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid) - print(" write_enabler:", base32.b2a(we), file=out) + print(" write_enabler:", unicode(base32.b2a(we), "ascii"), file=out) print(file=out) _dump_secrets(u.get_storage_index(), secret, nodeid, out) elif isinstance(u, uri.ReadonlyMDMFFileURI): if show_header: print("MDMF Read-only URI:", file=out) - print(" readkey:", base32.b2a(u.readkey), file=out) - print(" storage index:", si_b2a(u.get_storage_index()), file=out) - print(" fingerprint:", base32.b2a(u.fingerprint), file=out) + print(" readkey:", unicode(base32.b2a(u.readkey), "ascii"), file=out) + print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out) elif isinstance(u, uri.MDMFVerifierURI): if show_header: print("MDMF Verifier URI:", file=out) - print(" storage index:", si_b2a(u.get_storage_index()), file=out) - print(" fingerprint:", base32.b2a(u.fingerprint), file=out) + print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out) elif isinstance(u, uri.ImmutableDirectoryURI): # CHK-based directory diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index 405340115..d1fc48899 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -47,6 +47,8 @@ from allmydata.util.encodingutil import listdir_unicode, get_io_encoding class CLI(CLITestMixin, unittest.TestCase): def _dump_cap(self, *args): + args = [(unicode(s, "ascii") if isinstance(s, bytes) else s) + for s in args] config = debug.DumpCapOptions() config.stdout,config.stderr = StringIO(), StringIO() config.parseOptions(args) @@ -66,7 +68,7 @@ class CLI(CLITestMixin, unittest.TestCase): needed_shares=needed_shares, total_shares=total_shares, size=size) - output = self._dump_cap(unicode(u.to_string(), "ascii")) + output = self._dump_cap(u.to_string()) self.failUnless("CHK File:" in output, output) self.failUnless("key: aaaqeayeaudaocajbifqydiob4" in output, output) self.failUnless("UEB hash: nf3nimquen7aeqm36ekgxomalstenpkvsdmf6fplj7swdatbv5oa" in output, output) @@ -78,7 +80,7 @@ class CLI(CLITestMixin, unittest.TestCase): u.to_string()) self.failUnless("client renewal secret: znxmki5zdibb5qlt46xbdvk2t55j7hibejq3i5ijyurkr6m6jkhq" in output, output) - output = self._dump_cap(u.get_verify_cap().to_string()) + output = self._dump_cap(unicode(u.get_verify_cap().to_string(), "ascii")) self.failIf("key: " in output, output) self.failUnless("UEB hash: nf3nimquen7aeqm36ekgxomalstenpkvsdmf6fplj7swdatbv5oa" in output, output) self.failUnless("size: 1234" in output, output) @@ -95,14 +97,14 @@ class CLI(CLITestMixin, unittest.TestCase): self.failUnless("storage index: hdis5iaveku6lnlaiccydyid7q" in output, output) def test_dump_cap_lit(self): - u = uri.LiteralFileURI("this is some data") + u = uri.LiteralFileURI(b"this is some data") output = self._dump_cap(u.to_string()) self.failUnless("Literal File URI:" in output, output) self.failUnless("data: 'this is some data'" in output, output) def test_dump_cap_sdmf(self): - writekey = "\x01" * 16 - fingerprint = "\xfe" * 32 + writekey = b"\x01" * 16 + fingerprint = b"\xfe" * 32 u = uri.WriteableSSKFileURI(writekey, fingerprint) output = self._dump_cap(u.to_string()) @@ -152,8 +154,8 @@ class CLI(CLITestMixin, unittest.TestCase): self.failUnless("fingerprint: 737p57x6737p57x6737p57x6737p57x6737p57x6737p57x6737a" in output, output) def test_dump_cap_mdmf(self): - writekey = "\x01" * 16 - fingerprint = "\xfe" * 32 + writekey = b"\x01" * 16 + fingerprint = b"\xfe" * 32 u = uri.WriteableMDMFFileURI(writekey, fingerprint) output = self._dump_cap(u.to_string()) @@ -204,8 +206,8 @@ class CLI(CLITestMixin, unittest.TestCase): def test_dump_cap_chk_directory(self): - key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" - uri_extension_hash = hashutil.uri_extension_hash("stuff") + key = b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + uri_extension_hash = hashutil.uri_extension_hash(b"stuff") needed_shares = 25 total_shares = 100 size = 1234 @@ -238,8 +240,8 @@ class CLI(CLITestMixin, unittest.TestCase): self.failUnless("storage index: hdis5iaveku6lnlaiccydyid7q" in output, output) def test_dump_cap_sdmf_directory(self): - writekey = "\x01" * 16 - fingerprint = "\xfe" * 32 + writekey = b"\x01" * 16 + fingerprint = b"\xfe" * 32 u1 = uri.WriteableSSKFileURI(writekey, fingerprint) u = uri.DirectoryURI(u1) @@ -282,8 +284,8 @@ class CLI(CLITestMixin, unittest.TestCase): self.failUnless("fingerprint: 737p57x6737p57x6737p57x6737p57x6737p57x6737p57x6737a" in output, output) def test_dump_cap_mdmf_directory(self): - writekey = "\x01" * 16 - fingerprint = "\xfe" * 32 + writekey = b"\x01" * 16 + fingerprint = b"\xfe" * 32 u1 = uri.WriteableMDMFFileURI(writekey, fingerprint) u = uri.MDMFDirectoryURI(u1) @@ -343,7 +345,7 @@ class CLI(CLITestMixin, unittest.TestCase): fileutil.write("cli/test_catalog_shares/node1/storage/shares/mq/not-a-dir", "") # write a bogus share that looks a little bit like CHK fileutil.write(os.path.join(sharedir, "8"), - "\x00\x00\x00\x01" + "\xff" * 200) # this triggers an assert + b"\x00\x00\x00\x01" + b"\xff" * 200) # this triggers an assert nodedir2 = "cli/test_catalog_shares/node2" fileutil.make_dirs(nodedir2) From a4af4d8e5bb3b21a6e0c37290678bb78a6f2db51 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 May 2021 10:36:27 -0400 Subject: [PATCH 225/463] Even more passing tests on Python 3. --- src/allmydata/scripts/cli.py | 4 +++- src/allmydata/test/cli/common.py | 3 ++- src/allmydata/test/cli/test_cli.py | 33 +++++++++++++++--------------- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 645b57f41..011dc3b21 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -1,5 +1,7 @@ from __future__ import print_function +from past.builtins import unicode + import os.path, re, fnmatch try: @@ -36,7 +38,7 @@ class FileStoreOptions(BaseOptions): # compute a node-url from the existing options, put in self['node-url'] if self['node-url']: - if (not isinstance(self['node-url'], basestring) + if (not isinstance(self['node-url'], (bytes, unicode)) or not NODEURL_RE.match(self['node-url'])): msg = ("--node-url is required to be a string and look like " "\"http://HOSTNAMEORADDR:PORT\", not: %r" % diff --git a/src/allmydata/test/cli/common.py b/src/allmydata/test/cli/common.py index 8796f815f..7b97312b0 100644 --- a/src/allmydata/test/cli/common.py +++ b/src/allmydata/test/cli/common.py @@ -1,9 +1,10 @@ -from six import ensure_str +from six import ensure_str, ensure_text from ...scripts import runner from ..common_util import ReallyEqualMixin, run_cli, run_cli_unicode def parse_options(basedir, command, args): + args = [ensure_text(s) for s in args] o = runner.Options() o.parseOptions(["--node-directory", basedir, command] + args) while hasattr(o, "subOptions"): diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index d1fc48899..85a3ea11e 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -1,7 +1,9 @@ from past.builtins import unicode -import os.path from six.moves import cStringIO as StringIO +from six import ensure_text + +import os.path import sys import re from mock import patch, Mock @@ -47,8 +49,7 @@ from allmydata.util.encodingutil import listdir_unicode, get_io_encoding class CLI(CLITestMixin, unittest.TestCase): def _dump_cap(self, *args): - args = [(unicode(s, "ascii") if isinstance(s, bytes) else s) - for s in args] + args = [ensure_text(s) for s in args] config = debug.DumpCapOptions() config.stdout,config.stderr = StringIO(), StringIO() config.parseOptions(args) @@ -759,8 +760,8 @@ class Errors(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() c0 = self.g.clients[0] self.fileurls = {} - DATA = "data" * 100 - d = c0.upload(upload.Data(DATA, convergence="")) + DATA = b"data" * 100 + d = c0.upload(upload.Data(DATA, convergence=b"")) def _stash_bad(ur): self.uri_1share = ur.get_uri() self.delete_shares_numbered(ur.get_uri(), range(1,10)) @@ -1201,31 +1202,31 @@ class Options(ReallyEqualMixin, unittest.TestCase): fileutil.make_dirs("cli/test_options") fileutil.make_dirs("cli/test_options/private") fileutil.write("cli/test_options/node.url", "http://localhost:8080/\n") - filenode_uri = uri.WriteableSSKFileURI(writekey="\x00"*16, - fingerprint="\x00"*32) + filenode_uri = uri.WriteableSSKFileURI(writekey=b"\x00"*16, + fingerprint=b"\x00"*32) private_uri = uri.DirectoryURI(filenode_uri).to_string() - fileutil.write("cli/test_options/private/root_dir.cap", private_uri + "\n") + fileutil.write("cli/test_options/private/root_dir.cap", private_uri + b"\n") def parse2(args): return parse_options("cli/test_options", "ls", args) o = parse2([]) self.failUnlessEqual(o['node-url'], "http://localhost:8080/") - self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], private_uri) + self.failUnlessEqual(o.aliases[DEFAULT_ALIAS].encode("ascii"), private_uri) self.failUnlessEqual(o.where, u"") o = parse2(["--node-url", "http://example.org:8111/"]) self.failUnlessEqual(o['node-url'], "http://example.org:8111/") - self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], private_uri) + self.failUnlessEqual(o.aliases[DEFAULT_ALIAS].encode("ascii"), private_uri) self.failUnlessEqual(o.where, u"") # -u for --node-url used to clash with -u for --uri (tickets #1949 and #2137). o = parse2(["-u", "http://example.org:8111/"]) self.failUnlessEqual(o['node-url'], "http://example.org:8111/") - self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], private_uri) + self.failUnlessEqual(o.aliases[DEFAULT_ALIAS].encode("ascii"), private_uri) self.failUnlessEqual(o.where, u"") self.failIf(o["uri"]) o = parse2(["-u", "http://example.org:8111/", "--uri"]) self.failUnlessEqual(o['node-url'], "http://example.org:8111/") - self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], private_uri) + self.failUnlessEqual(o.aliases[DEFAULT_ALIAS].encode("ascii"), private_uri) self.failUnlessEqual(o.where, u"") self.failUnless(o["uri"]) @@ -1234,17 +1235,17 @@ class Options(ReallyEqualMixin, unittest.TestCase): self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], "root") self.failUnlessEqual(o.where, u"") - other_filenode_uri = uri.WriteableSSKFileURI(writekey="\x11"*16, - fingerprint="\x11"*32) + other_filenode_uri = uri.WriteableSSKFileURI(writekey=b"\x11"*16, + fingerprint=b"\x11"*32) other_uri = uri.DirectoryURI(other_filenode_uri).to_string() o = parse2(["--dir-cap", other_uri]) self.failUnlessEqual(o['node-url'], "http://localhost:8080/") - self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], other_uri) + self.failUnlessEqual(o.aliases[DEFAULT_ALIAS].encode("ascii"), other_uri) self.failUnlessEqual(o.where, u"") o = parse2(["--dir-cap", other_uri, "subdir"]) self.failUnlessEqual(o['node-url'], "http://localhost:8080/") - self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], other_uri) + self.failUnlessEqual(o.aliases[DEFAULT_ALIAS].encode("ascii"), other_uri) self.failUnlessEqual(o.where, u"subdir") self.failUnlessRaises(usage.UsageError, parse2, From c589e97cdeff2c566e9664792700789898afa062 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 May 2021 10:40:43 -0400 Subject: [PATCH 226/463] All tests pass on Python 3. --- src/allmydata/test/cli/test_cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index 85a3ea11e..3035bdb0e 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -1,7 +1,7 @@ from past.builtins import unicode from six.moves import cStringIO as StringIO -from six import ensure_text +from six import ensure_text, ensure_str import os.path import sys @@ -1151,7 +1151,7 @@ class Webopen(GridTestMixin, CLITestMixin, unittest.TestCase): # TODO: replace with @patch that supports Deferreds. import webbrowser def call_webbrowser_open(url): - self.failUnlessIn(self.alias_uri.replace(':', '%3A'), url) + self.failUnlessIn(unicode(self.alias_uri, "ascii").replace(':', '%3A'), url) self.webbrowser_open_called = True def _cleanup(res): webbrowser.open = self.old_webbrowser_open @@ -1332,7 +1332,7 @@ class Run(unittest.TestCase): If the pidfile exists but does not contain a numeric value, a complaint to this effect is written to stderr. """ - basedir = FilePath(self.mktemp().decode("ascii")) + basedir = FilePath(ensure_str(self.mktemp())) basedir.makedirs() basedir.child(u"twistd.pid").setContent(b"foo") basedir.child(u"tahoe-client.tac").setContent(b"") @@ -1340,7 +1340,7 @@ class Run(unittest.TestCase): config = tahoe_run.RunOptions() config.stdout = StringIO() config.stderr = StringIO() - config['basedir'] = basedir.path + config['basedir'] = ensure_text(basedir.path) config.twistd_args = [] result_code = tahoe_run.run(config) From 4c6d55b2600c61ceb9e736b6a8ae821917cd1104 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 May 2021 10:52:55 -0400 Subject: [PATCH 227/463] Port to Python 3. --- src/allmydata/scripts/admin.py | 5 ++- src/allmydata/test/cli/test_cli.py | 68 +++++++++++++++++------------- src/allmydata/util/_python3.py | 1 + 3 files changed, 43 insertions(+), 31 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 0c860e93a..abe3d093c 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -1,6 +1,7 @@ from __future__ import print_function from past.builtins import unicode +from six import ensure_binary try: from allmydata.scripts.types_ import SubCommands @@ -49,8 +50,8 @@ def derive_pubkey(options): out = options.stdout from allmydata.crypto import ed25519 privkey_vs = options.privkey - private_key, public_key = ed25519.signing_keypair_from_string( - privkey_vs.encode("ascii")) + privkey_vs = ensure_binary(privkey_vs) + private_key, public_key = ed25519.signing_keypair_from_string(privkey_vs) print("private:", unicode(ed25519.string_from_signing_key(private_key), "ascii"), file=out) print("public:", unicode(ed25519.string_from_verifying_key(public_key), "ascii"), file=out) return 0 diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index 3035bdb0e..8a9b4dfd6 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -1,4 +1,14 @@ -from past.builtins import unicode +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six.moves import cStringIO as StringIO from six import ensure_text, ensure_str @@ -81,7 +91,7 @@ class CLI(CLITestMixin, unittest.TestCase): u.to_string()) self.failUnless("client renewal secret: znxmki5zdibb5qlt46xbdvk2t55j7hibejq3i5ijyurkr6m6jkhq" in output, output) - output = self._dump_cap(unicode(u.get_verify_cap().to_string(), "ascii")) + output = self._dump_cap(str(u.get_verify_cap().to_string(), "ascii")) self.failIf("key: " in output, output) self.failUnless("UEB hash: nf3nimquen7aeqm36ekgxomalstenpkvsdmf6fplj7swdatbv5oa" in output, output) self.failUnless("size: 1234" in output, output) @@ -354,7 +364,7 @@ class CLI(CLITestMixin, unittest.TestCase): # now make sure that the 'catalog-shares' commands survives the error out, err = self._catalog_shares(nodedir1, nodedir2) - self.failUnlessReallyEqual(out, "", out) + self.assertEqual(out, "") self.failUnless("Error processing " in err, "didn't see 'error processing' in '%s'" % err) #self.failUnless(nodedir1 in err, @@ -508,9 +518,9 @@ class CLI(CLITestMixin, unittest.TestCase): fileutil.make_dirs(basedir) for name in filenames: - open(os.path.join(unicode(basedir), name), "wb").close() + open(os.path.join(str(basedir), name), "wb").close() - for file in listdir_unicode(unicode(basedir)): + for file in listdir_unicode(str(basedir)): self.failUnlessIn(normalize(file), filenames) def test_exception_catcher(self): @@ -677,7 +687,7 @@ class Ln(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) # Make sure that validation extends to the "to" parameter d.addCallback(lambda ign: self.do_cli("create-alias", "havasu")) @@ -736,8 +746,8 @@ class Admin(unittest.TestCase): def test_derive_pubkey(self): priv_key, pub_key = ed25519.create_signing_keypair() - priv_key_str = unicode(ed25519.string_from_signing_key(priv_key), "ascii") - pub_key_str = unicode(ed25519.string_from_verifying_key(pub_key), "ascii") + priv_key_str = str(ed25519.string_from_signing_key(priv_key), "ascii") + pub_key_str = str(ed25519.string_from_verifying_key(pub_key), "ascii") d = run_cli("admin", "derive-pubkey", priv_key_str) def _done(args): (rc, stdout, stderr) = args @@ -764,7 +774,7 @@ class Errors(GridTestMixin, CLITestMixin, unittest.TestCase): d = c0.upload(upload.Data(DATA, convergence=b"")) def _stash_bad(ur): self.uri_1share = ur.get_uri() - self.delete_shares_numbered(ur.get_uri(), range(1,10)) + self.delete_shares_numbered(ur.get_uri(), list(range(1,10))) d.addCallback(_stash_bad) # the download is abandoned as soon as it's clear that we won't get @@ -828,7 +838,7 @@ class Get(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) return d @@ -843,7 +853,7 @@ class Get(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) return d @@ -860,7 +870,7 @@ class Manifest(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) return d @@ -875,7 +885,7 @@ class Manifest(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) return d @@ -890,7 +900,7 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase): def _check(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") + self.assertEqual(err, "") self.failUnlessIn("URI:", out) d.addCallback(_check) @@ -903,7 +913,7 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase): def _check(args, st): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") + self.assertEqual(err, "") self.failUnlessIn(st, out) return out @@ -939,7 +949,7 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase): def _check(args, st): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") + self.assertEqual(err, "") self.failUnlessIn(st, out) return out d.addCallback(_check, "URI:DIR2") @@ -983,7 +993,7 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase): def _check(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") + self.assertEqual(err, "") self.failUnlessIn("URI:", out) d.addCallback(_check) @@ -999,7 +1009,7 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) return d @@ -1023,7 +1033,7 @@ class Unlink(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) d.addCallback(lambda ign: self.do_cli(self.command, "afile")) @@ -1041,7 +1051,7 @@ class Unlink(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) d.addCallback(lambda ign: self.do_cli(self.command, "nonexistent:afile")) @@ -1067,7 +1077,7 @@ class Unlink(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("'tahoe %s'" % (self.command,), err) self.failUnlessIn("path must be given", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) return d @@ -1088,7 +1098,7 @@ class Stats(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("stats", self.rooturi)) def _check_stats(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(err, "") self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnlessIn(" count-immutable-files: 0", lines) @@ -1112,7 +1122,7 @@ class Stats(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) return d @@ -1126,7 +1136,7 @@ class Stats(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) return d @@ -1143,7 +1153,7 @@ class Webopen(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) return d @@ -1151,7 +1161,7 @@ class Webopen(GridTestMixin, CLITestMixin, unittest.TestCase): # TODO: replace with @patch that supports Deferreds. import webbrowser def call_webbrowser_open(url): - self.failUnlessIn(unicode(self.alias_uri, "ascii").replace(':', '%3A'), url) + self.failUnlessIn(str(self.alias_uri, "ascii").replace(':', '%3A'), url) self.webbrowser_open_called = True def _cleanup(res): webbrowser.open = self.old_webbrowser_open @@ -1168,15 +1178,15 @@ class Webopen(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0, repr((rc, out, err))) self.failUnlessIn("Alias 'alias' created", out) - self.failUnlessReallyEqual(err, "") + self.assertEqual(err, "") self.alias_uri = get_aliases(self.get_clientdir())["alias"] d.addCallback(_check_alias) d.addCallback(lambda res: self.do_cli("webopen", "alias:")) def _check_webopen(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0, repr((rc, out, err))) - self.failUnlessReallyEqual(out, "") - self.failUnlessReallyEqual(err, "") + self.assertEqual(out, "") + self.assertEqual(err, "") self.failUnless(self.webbrowser_open_called) d.addCallback(_check_webopen) d.addBoth(_cleanup) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index bbafc144f..f74222189 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -177,6 +177,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.cli.test_backup", "allmydata.test.cli.test_backupdb", "allmydata.test.cli.test_check", + "allmydata.test.cli.test_cli", "allmydata.test.cli.test_cp", "allmydata.test.cli.test_create", "allmydata.test.cli.test_create_alias", From 7411da1b88ed3ae69cc0ec7947f26d537667c43f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 May 2021 10:57:45 -0400 Subject: [PATCH 228/463] Port to Python 3. --- src/allmydata/test/cli/common.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+) diff --git a/src/allmydata/test/cli/common.py b/src/allmydata/test/cli/common.py index 7b97312b0..ed066c6b6 100644 --- a/src/allmydata/test/cli/common.py +++ b/src/allmydata/test/cli/common.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from six import ensure_str, ensure_text from ...scripts import runner diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index f74222189..d708a52ac 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -98,6 +98,7 @@ PORTED_MODULES = [ "allmydata.storage.shares", "allmydata.test", "allmydata.test.cli", + "allmydata.test.cli.common", "allmydata.test.cli_node_api", "allmydata.test.common", "allmydata.test.common_util", From cc176342d4eaf9d06082a8a95c62de2ce3cf4d7e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 May 2021 11:22:12 -0400 Subject: [PATCH 229/463] Some progress towards test_system.py fully running on Python 3. --- src/allmydata/scripts/tahoe_unlink.py | 4 +-- src/allmydata/test/common_util.py | 2 +- src/allmydata/test/test_system.py | 38 ++++++++++++--------------- src/allmydata/util/_python3.py | 5 ---- 4 files changed, 20 insertions(+), 29 deletions(-) diff --git a/src/allmydata/scripts/tahoe_unlink.py b/src/allmydata/scripts/tahoe_unlink.py index bc1d43c9e..1ec92c69e 100644 --- a/src/allmydata/scripts/tahoe_unlink.py +++ b/src/allmydata/scripts/tahoe_unlink.py @@ -1,6 +1,6 @@ from __future__ import print_function -import urllib +from urllib.parse import quote as url_quote from allmydata.scripts.common_http import do_http, format_http_success, format_http_error from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError @@ -27,7 +27,7 @@ def unlink(options, command="unlink"): 'tahoe %s' can only unlink directory entries, so a path must be given.""" % (command,), file=stderr) return 1 - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + url = nodeurl + "uri/%s" % url_quote(rootcap) url += "/" + escape_path(path) resp = do_http("DELETE", url) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 95065710c..caafbb81d 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -85,7 +85,7 @@ def run_cli_native(verb, *args, **kwargs): bytes. """ nodeargs = kwargs.pop("nodeargs", []) - encoding = kwargs.pop("encoding", "utf-8") + encoding = kwargs.pop("encoding", None) or "utf-8" return_bytes = kwargs.pop("return_bytes", False) verb = maybe_unicode_to_argv(verb) args = [maybe_unicode_to_argv(a) for a in args] diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 0ff1e06e9..8ffbf7063 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1,12 +1,12 @@ """ -Ported to Python 3, partially: test_filesystem* will be done in a future round. +Ported to Python 3. """ from __future__ import print_function from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals -from future.utils import PY2, PY3 +from future.utils import PY2 if PY2: # Don't import bytes since it causes issues on (so far unported) modules on Python 2. from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min, str # noqa: F401 @@ -16,7 +16,6 @@ from six import ensure_text, ensure_str import os, re, sys, time, json from functools import partial -from unittest import skipIf from bs4 import BeautifulSoup @@ -1665,9 +1664,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): d.addCallback(self.log, "did _check_publish_private") d.addCallback(self._test_web) d.addCallback(self._test_control) - if PY2: - # TODO when CLI is ported to Python 3, reenable. - d.addCallback(self._test_cli) + d.addCallback(self._test_cli) # P now has four top-level children: # P/personal/sekrit data # P/s2-ro/ @@ -2298,7 +2295,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def _check_aliases_1(out_and_err): (out, err) = out_and_err self.failUnlessEqual(err, "") - self.failUnlessEqual(out.strip(" \n"), "tahoe: %s" % private_uri) + self.failUnlessEqual(out.strip(" \n"), "tahoe: %s" % str(private_uri, "ascii")) d.addCallback(_check_aliases_1) # now that that's out of the way, remove root_dir.cap and work with @@ -2355,7 +2352,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): (out, err) = out_and_err self.failUnlessEqual(err, "") if filenum is not None: - self.failUnlessEqual(out, datas[filenum]) + self.failUnlessEqual(out, str(datas[filenum], "ascii")) if data is not None: self.failUnlessEqual(out, data) @@ -2369,7 +2366,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): uri0 = out.strip() return run(None, "get", uri0) d.addCallback(_put_out) - d.addCallback(lambda out_err: self.failUnlessEqual(out_err[0], datas[0])) + d.addCallback(lambda out_err: self.failUnlessEqual(out_err[0], str(datas[0], "ascii"))) d.addCallback(run, "put", files[1], "subdir/tahoe-file1") # tahoe put bar tahoe:FOO @@ -2411,14 +2408,14 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def _check_outfile0(out_and_err): (out, err) = out_and_err data = open(outfile0,"rb").read() - self.failUnlessEqual(data, "data to be uploaded: file2\n") + self.failUnlessEqual(data, b"data to be uploaded: file2\n") d.addCallback(_check_outfile0) outfile1 = os.path.join(self.basedir, "outfile0") d.addCallback(run, "get", "tahoe:subdir/tahoe-file1", outfile1) def _check_outfile1(out_and_err): (out, err) = out_and_err data = open(outfile1,"rb").read() - self.failUnlessEqual(data, "data to be uploaded: file1\n") + self.failUnlessEqual(data, b"data to be uploaded: file1\n") d.addCallback(_check_outfile1) d.addCallback(run, "unlink", "tahoe-file0") @@ -2455,7 +2452,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): if "file3" in l: rw_uri = self._mutable_file3_uri u = uri.from_string_mutable_filenode(rw_uri) - ro_uri = u.get_readonly().to_string() + ro_uri = str(u.get_readonly().to_string(), "ascii") self.failUnless(ro_uri in l) d.addCallback(_check_ls_rouri) @@ -2528,17 +2525,17 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): dn = os.path.join(self.basedir, "dir1") os.makedirs(dn) with open(os.path.join(dn, "rfile1"), "wb") as f: - f.write("rfile1") + f.write(b"rfile1") with open(os.path.join(dn, "rfile2"), "wb") as f: - f.write("rfile2") + f.write(b"rfile2") with open(os.path.join(dn, "rfile3"), "wb") as f: - f.write("rfile3") + f.write(b"rfile3") sdn2 = os.path.join(dn, "subdir2") os.makedirs(sdn2) with open(os.path.join(sdn2, "rfile4"), "wb") as f: - f.write("rfile4") + f.write(b"rfile4") with open(os.path.join(sdn2, "rfile5"), "wb") as f: - f.write("rfile5") + f.write(b"rfile5") # from disk into tahoe d.addCallback(run, "cp", "-r", dn, "tahoe:") @@ -2615,7 +2612,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): return d - @skipIf(PY3, "Python 3 CLI support hasn't happened yet.") def test_filesystem_with_cli_in_subprocess(self): # We do this in a separate test so that test_filesystem doesn't skip if we can't run bin/tahoe. @@ -2659,9 +2655,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def _check_ls(res): out, err, rc_or_sig = res self.failUnlessEqual(rc_or_sig, 0, str(res)) - self.failUnlessEqual(err, "", str(res)) - self.failUnlessIn("tahoe-moved", out) - self.failIfIn("tahoe-file", out) + self.failUnlessEqual(err, b"", str(res)) + self.failUnlessIn(b"tahoe-moved", out) + self.failIfIn(b"tahoe-file", out) d.addCallback(_check_ls) return d diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index d708a52ac..a18f50635 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -260,12 +260,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_storage", "allmydata.test.test_storage_client", "allmydata.test.test_storage_web", - - # Only partially ported, test_filesystem_with_cli_in_subprocess isn't - # ported yet, nor is part of test_filesystem (the call to _test_cli). This - # should be done once CLI is ported. "allmydata.test.test_system", - "allmydata.test.test_testing", "allmydata.test.test_time_format", "allmydata.test.test_tor_provider", From 07b58e36192dfbc8cdf26ef8907b1c61e76eca3b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 May 2021 11:34:58 -0400 Subject: [PATCH 230/463] All tests pass on Python 3. --- src/allmydata/scripts/tahoe_cp.py | 19 ++++++++++++++----- src/allmydata/test/test_system.py | 2 +- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/allmydata/scripts/tahoe_cp.py b/src/allmydata/scripts/tahoe_cp.py index 0a6c67942..fb86ff0ec 100644 --- a/src/allmydata/scripts/tahoe_cp.py +++ b/src/allmydata/scripts/tahoe_cp.py @@ -5,7 +5,8 @@ from past.builtins import unicode import os.path from urllib.parse import quote as url_quote from collections import defaultdict -from six.moves import cStringIO as StringIO +from io import BytesIO + from twisted.python.failure import Failure from allmydata.scripts.common import get_alias, escape_path, \ DefaultAliasMarker, TahoeError @@ -200,13 +201,21 @@ class TahoeFileSource(object): def open(self, caps_only): if caps_only: - return StringIO(self.readcap) + return BytesIO(self.readcap) url = self.nodeurl + "uri/" + url_quote(self.readcap) return GET_to_file(url) def bestcap(self): return self.writecap or self.readcap + +def seekable(file_like): + """Return whether the file-like object is seekable.""" + return hasattr(file_like, "seek") and ( + not hasattr(file_like, "seekable") or file_like.seekable() + ) + + class TahoeFileTarget(object): def __init__(self, nodeurl, mutable, writecap, readcap, url): self.nodeurl = nodeurl @@ -220,7 +229,7 @@ class TahoeFileTarget(object): assert self.url # our do_http() call currently requires a string or a filehandle with # a real .seek - if not hasattr(inf, "seek"): + if not seekable(inf): inf = inf.read() PUT(self.url, inf) # TODO: this always creates immutable files. We might want an option @@ -306,7 +315,7 @@ class TahoeMissingTarget(object): def put_file(self, inf): # We want to replace this object in-place. - if not hasattr(inf, "seek"): + if not seekable(inf): inf = inf.read() PUT(self.url, inf) # TODO: this always creates immutable files. We might want an option @@ -417,7 +426,7 @@ class TahoeDirectoryTarget(object): def put_file(self, name, inf): precondition(isinstance(name, unicode), name) url = self.nodeurl + "uri" - if not hasattr(inf, "seek"): + if not seekable(inf): inf = inf.read() if self.children is None: diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 8ffbf7063..12ae846eb 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -2579,7 +2579,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): (out, err) = out_and_err x = open(os.path.join(dn_copy2, "dir1", "subdir2", "rfile4")).read() y = uri.from_string_filenode(x) - self.failUnlessEqual(y.data, "rfile4") + self.failUnlessEqual(y.data, b"rfile4") d.addCallback(_check_capsonly) # and tahoe-to-tahoe From 90240ae5efb7f6a0ecf4c17f193da95e56ec0c1d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 May 2021 12:03:12 -0400 Subject: [PATCH 231/463] Tests pass on Python 3. --- src/allmydata/scripts/slow_operation.py | 8 +++++++- src/allmydata/scripts/tahoe_manifest.py | 4 ++++ src/allmydata/test/test_deepcheck.py | 21 ++++++++++----------- src/allmydata/util/_python3.py | 4 ---- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py index 5d8c77538..b4b2f8196 100644 --- a/src/allmydata/scripts/slow_operation.py +++ b/src/allmydata/scripts/slow_operation.py @@ -1,5 +1,6 @@ from __future__ import print_function +from future.utils import PY3 from past.builtins import unicode from six import ensure_str @@ -78,8 +79,13 @@ class SlowOperationRunner(object): if not data["finished"]: return False if self.options.get("raw"): + if PY3: + # need to write bytes! + stdout = stdout.buffer if is_printable_ascii(jdata): - print(jdata, file=stdout) + stdout.write(jdata) + stdout.write(b"\n") + stdout.flush() else: print("The JSON response contained unprintable characters:\n%s" % quote_output(jdata), file=stderr) return True diff --git a/src/allmydata/scripts/tahoe_manifest.py b/src/allmydata/scripts/tahoe_manifest.py index b837e648a..966583244 100644 --- a/src/allmydata/scripts/tahoe_manifest.py +++ b/src/allmydata/scripts/tahoe_manifest.py @@ -1,5 +1,6 @@ from __future__ import print_function +from future.utils import PY3 from past.builtins import unicode from urllib.parse import quote as url_quote @@ -51,6 +52,9 @@ class ManifestStreamer(LineOnlyReceiver, object): #print("RESP", dir(resp)) # use Twisted to split this into lines self.in_error = False + # Writing bytes, so need binary stdout. + if PY3: + stdout = stdout.buffer while True: chunk = resp.read(100) if not chunk: diff --git a/src/allmydata/test/test_deepcheck.py b/src/allmydata/test/test_deepcheck.py index baee1acbe..652e51ea5 100644 --- a/src/allmydata/test/test_deepcheck.py +++ b/src/allmydata/test/test_deepcheck.py @@ -17,10 +17,10 @@ from __future__ import unicode_literals # (Pdb) pp data # '334:12:b\'mutable-good\',90:URI:SSK-RO:... from past.builtins import unicode as str -from future.utils import PY3, PY2 +from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 - +from six import ensure_text import os, json from urllib.parse import quote as url_quote @@ -170,7 +170,8 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin, return data def parse_streamed_json(self, s): - for unit in s.split(b"\n"): + s = ensure_text(s) + for unit in s.split("\n"): if not unit: # stream should end with a newline, so split returns "" continue @@ -746,8 +747,6 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def do_test_cli_good(self, ignored): d = defer.succeed(None) - if PY3: # TODO fixme once Python 3 CLI porting is done - return d d.addCallback(lambda ign: self.do_cli_manifest_stream1()) d.addCallback(lambda ign: self.do_cli_manifest_stream2()) d.addCallback(lambda ign: self.do_cli_manifest_stream3()) @@ -758,7 +757,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): return d def _check_manifest_storage_index(self, out): - lines = [l for l in out.split(b"\n") if l] + lines = [l.encode("utf-8") for l in out.split("\n") if l] self.failUnlessEqual(len(lines), 3) self.failUnless(base32.b2a(self.root.get_storage_index()) in lines) self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines) @@ -769,7 +768,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def _check(args): (rc, out, err) = args self.failUnlessEqual(err, "") - lines = [l for l in out.split(b"\n") if l] + lines = [l for l in out.split("\n") if l] self.failUnlessEqual(len(lines), 8) caps = {} for l in lines: @@ -778,7 +777,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): except ValueError: cap = l.strip() path = "" - caps[cap] = path + caps[cap.encode("ascii")] = path self.failUnless(self.root.get_uri() in caps) self.failUnlessEqual(caps[self.root.get_uri()], "") self.failUnlessEqual(caps[self.mutable.get_uri()], "mutable") @@ -814,7 +813,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def _check(args): (rc, out, err) = args self.failUnlessEqual(err, "") - lines = [l for l in out.split(b"\n") if l] + lines = [l.encode("utf-8") for l in out.split("\n") if l] self.failUnlessEqual(len(lines), 3) self.failUnless(self.root.get_verify_cap().to_string() in lines) self.failUnless(self.mutable.get_verify_cap().to_string() in lines) @@ -827,7 +826,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def _check(args): (rc, out, err) = args self.failUnlessEqual(err, "") - lines = [l for l in out.split(b"\n") if l] + lines = [l.encode("utf-8") for l in out.split("\n") if l] self.failUnlessEqual(len(lines), 3) self.failUnless(self.root.get_repair_cap().to_string() in lines) self.failUnless(self.mutable.get_repair_cap().to_string() in lines) @@ -839,7 +838,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): d = self.do_cli("stats", self.root_uri) def _check3(args): (rc, out, err) = args - lines = [l.strip() for l in out.split(b"\n") if l] + lines = [l.strip() for l in out.split("\n") if l] self.failUnless("count-immutable-files: 1" in lines) self.failUnless("count-mutable-files: 1" in lines) self.failUnless("count-literal-files: 3" in lines) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index a18f50635..5f78af626 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -218,11 +218,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_consumer", "allmydata.test.test_crawler", "allmydata.test.test_crypto", - - # Only partially ported, CLI-using test code is disabled for now until CLI - # is ported. "allmydata.test.test_deepcheck", - "allmydata.test.test_deferredutil", "allmydata.test.test_dictutil", "allmydata.test.test_dirnode", From 8f997870c8db9a081bf6f9f06f18a53ce7f14345 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 09:47:28 -0400 Subject: [PATCH 232/463] Test newer versions of Python 3, as well as PyPy 3. --- .github/workflows/ci.yml | 4 ++++ newsfragments/3683.minor | 0 2 files changed, 4 insertions(+) create mode 100644 newsfragments/3683.minor diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f70432267..47e88a9a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,10 @@ jobs: python-version: - 2.7 - 3.6 + - 3.7 + - 3.8 + - 3.9 + - pypy-3.7 steps: diff --git a/newsfragments/3683.minor b/newsfragments/3683.minor new file mode 100644 index 000000000..e69de29bb From ea2db971a6f98f3ff6c3d269c750b2aa13e3e3c2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 10:06:35 -0400 Subject: [PATCH 233/463] Try to make GitHub Actions automatically use appropriate Tox envs by using `tox-gh-actions` package. --- .github/workflows/ci.yml | 16 +++++++--------- tox.ini | 12 +++++++++++- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47e88a9a0..b60b51b4d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,17 +40,15 @@ jobs: - name: Set up Python ${{ matrix.python-version }} if: ${{ matrix.os != 'windows-latest' }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} # See note below about need for using 32-bit Python 2.7 on - # Windows. The extra handling here for Python 3.6 on Windows is - # because I could not figure out the right GitHub Actions - # expression to do this in a better way. + # Windows. - name: Set up Python ${{ matrix.python-version }} [Windows x64] - if: ${{ ( matrix.os == 'windows-latest' ) && ( matrix.python-version == '3.6' ) }} - uses: actions/setup-python@v1 + if: ${{ ( matrix.os == 'windows-latest' ) && ( matrix.python-version != '2.7' ) }} + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} architecture: 'x64' @@ -89,14 +87,14 @@ jobs: - name: Install Python packages run: | - pip install --upgrade codecov tox setuptools + pip install --upgrade codecov tox tox-gh-actions setuptools pip list - name: Display tool versions run: python misc/build_helpers/show-tool-versions.py - - name: Run "tox -e py27-coverage" - run: tox -e py27-coverage + - name: Run tox for corresponding Python version + run: python -m tox - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v1 diff --git a/tox.ini b/tox.ini index a58d9d447..73dbb8e78 100644 --- a/tox.ini +++ b/tox.ini @@ -3,11 +3,21 @@ # test suite on all supported python versions. To use it, "pip install tox" # and then run "tox" from this directory. +# Map Python versions in GitHub Actions to tox environments to run. +[gh-actions] +python = + 2.7: py27-coverage,codechecks + 3.6: py36-coverage + 3.7: py37-coverage + 3.8: py38-coverage + 3.9: py39-coverage,typechecks + pypy-3.7: pypy3 + [pytest] twisted = 1 [tox] -envlist = typechecks,codechecks,py27,py36,pypy27 +envlist = typechecks,codechecks,py27,py36,py37,py38,py39,pypy27,pypy3 minversion = 2.4 [testenv] From ebedc660bd244e8ed0d5c7a06200a0c0d7486ef4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 10:12:42 -0400 Subject: [PATCH 234/463] Note coverage explicitly in envlist. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 73dbb8e78..2274b42e7 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,py27,py36,py37,py38,py39,pypy27,pypy3 +envlist = typechecks,codechecks,py{27,36,37,38,39}-{coverage},pypy27,pypy3 minversion = 2.4 [testenv] From 293f372ea44118e2e9217d9ef161ac739dfe2748 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 10:17:13 -0400 Subject: [PATCH 235/463] Version of Twisted that doesn't break on Python 3 with Windows. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index df917eb40..32b7f25d2 100644 --- a/setup.py +++ b/setup.py @@ -97,7 +97,8 @@ install_requires = [ # an sftp extra in Tahoe-LAFS, there is no point in having one. # * Twisted 19.10 introduces Site.getContentFile which we use to get # temporary upload files placed into a per-node temporary directory. - "Twisted[tls,conch] >= 19.10.0", + # * Twisted 21.2.0 is missing binary wheels for some reason. + "Twisted[tls,conch] >= 19.10.0, != 21.2.0", "PyYAML >= 3.11", From 30c03d085e352baf7419ff6644e7b6411ab86e24 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 10:34:10 -0400 Subject: [PATCH 236/463] Newer versions of setuptools and pip, to work better with newer Pythons. --- tox.ini | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 2274b42e7..4b52fd316 100644 --- a/tox.ini +++ b/tox.ini @@ -35,9 +35,11 @@ deps = # happening at the time. The versions selected here are just the current # versions at the time. Bumping them to keep up with future releases is # fine as long as those releases are known to actually work. - pip==19.1.1 - setuptools==41.0.1 - wheel==0.33.4 + # + # For now these are versions that support Python 2. + pip==20.3.4 + setuptools==44.1.1 + wheel==0.36.2 subunitreporter==19.3.2 # As an exception, we don't pin certifi because it contains CA # certificates which necessarily change over time. Pinning this is From 65159c99616daa1c9d2b54299d9c01f5e1ebfee7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 10:37:49 -0400 Subject: [PATCH 237/463] Use modern (and actually correct on Windows!) API for getting CPU time. --- src/allmydata/stats.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index 7137ba28e..13ed8817c 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -9,7 +9,9 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - + from time import clock as process_time +else: + from time import process_time import time from twisted.application import service @@ -27,7 +29,7 @@ class CPUUsageMonitor(service.MultiService): def __init__(self): service.MultiService.__init__(self) - # we don't use time.clock() here, because the constructor is run by + # we don't use process_time() here, because the constructor is run by # the twistd parent process (as it loads the .tac file), whereas the # rest of the program will be run by the child process, after twistd # forks. Instead, set self.initial_cpu as soon as the reactor starts @@ -39,11 +41,11 @@ class CPUUsageMonitor(service.MultiService): TimerService(self.POLL_INTERVAL, self.check).setServiceParent(self) def _set_initial_cpu(self): - self.initial_cpu = time.clock() + self.initial_cpu = process_time() def check(self): now_wall = time.time() - now_cpu = time.clock() + now_cpu = process_time() self.samples.append( (now_wall, now_cpu) ) while len(self.samples) > self.HISTORY_LENGTH+1: self.samples.pop(0) @@ -68,7 +70,7 @@ class CPUUsageMonitor(service.MultiService): avg = self._average_N_minutes(15) if avg is not None: s["cpu_monitor.15min_avg"] = avg - now_cpu = time.clock() + now_cpu = process_time() s["cpu_monitor.total"] = now_cpu - self.initial_cpu return s From 1ed24a16169b77c844affed59101cc7b896f4a7d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 11:22:17 -0400 Subject: [PATCH 238/463] This API is irrelevant in Python 2.5 or later, and not present in newer Python 3. --- src/allmydata/scripts/tahoe_backup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/scripts/tahoe_backup.py b/src/allmydata/scripts/tahoe_backup.py index 2f3bbeed1..4847f2e8c 100644 --- a/src/allmydata/scripts/tahoe_backup.py +++ b/src/allmydata/scripts/tahoe_backup.py @@ -22,7 +22,6 @@ def get_local_metadata(path): metadata = {} # posix stat(2) metadata, depends on the platform - os.stat_float_times(True) s = os.stat(path) metadata["ctime"] = s.st_ctime metadata["mtime"] = s.st_mtime From fa4be104a63fbeb011b3f25858138b8e5004ad53 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 11:31:19 -0400 Subject: [PATCH 239/463] New name for module. --- src/allmydata/windows/registry.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/allmydata/windows/registry.py b/src/allmydata/windows/registry.py index c1f695d2b..2c7f03156 100644 --- a/src/allmydata/windows/registry.py +++ b/src/allmydata/windows/registry.py @@ -1,5 +1,5 @@ import sys -import _winreg +import winreg _AMD_KEY = r"Software\Allmydata" _BDIR_KEY = 'Base Dir Path' @@ -22,19 +22,19 @@ def get_registry_setting(key, name, _topkey=None): @param name: The name of the setting we are querying. @type name: String """ - topkeys = [_winreg.HKEY_CURRENT_USER, _winreg.HKEY_LOCAL_MACHINE] + topkeys = [winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE] if _topkey: topkeys.insert(0, _topkey) for topkey in topkeys: try: - regkey = _winreg.OpenKey(topkey, key) + regkey = winreg.OpenKey(topkey, key) - sublen, vallen, timestamp = _winreg.QueryInfoKey(regkey) + sublen, vallen, timestamp = winreg.QueryInfoKey(regkey) for validx in xrange(vallen): - keyname, value, keytype = _winreg.EnumValue(regkey, validx) - if keyname == name and keytype == _winreg.REG_SZ: + keyname, value, keytype = winreg.EnumValue(regkey, validx) + if keyname == name and keytype == winreg.REG_SZ: return value except WindowsError: @@ -42,27 +42,27 @@ def get_registry_setting(key, name, _topkey=None): # We didn't find the key: raise KeyError(key, name, "registry setting not found") -def set_registry_setting(key, name, data, reg_type=_winreg.REG_SZ, - _topkey=_winreg.HKEY_LOCAL_MACHINE, create_key_if_missing=True): +def set_registry_setting(key, name, data, reg_type=winreg.REG_SZ, + _topkey=winreg.HKEY_LOCAL_MACHINE, create_key_if_missing=True): """ Sets a registry setting. defaults to string values (REG_SZ) - overridable with reg_type. """ try: - regkey = _winreg.OpenKey(_topkey, key, 0, _winreg.KEY_SET_VALUE) + regkey = winreg.OpenKey(_topkey, key, 0, winreg.KEY_SET_VALUE) except WindowsError: if create_key_if_missing: - regkey = _winreg.CreateKey(_topkey, key) + regkey = winreg.CreateKey(_topkey, key) else: raise KeyError(key, "registry key not found") try: - _winreg.DeleteValue(regkey, name) + winreg.DeleteValue(regkey, name) except: pass - _winreg.SetValueEx(regkey, name, 0, reg_type, data) + winreg.SetValueEx(regkey, name, 0, reg_type, data) def get_registry_value(keyname): """ From fa0ec99052cd920da59272f572a03f0c290692d6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 11:43:59 -0400 Subject: [PATCH 240/463] Maybe newer Twisted is OK, IOCP was supposedly moved out. --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 32b7f25d2..df917eb40 100644 --- a/setup.py +++ b/setup.py @@ -97,8 +97,7 @@ install_requires = [ # an sftp extra in Tahoe-LAFS, there is no point in having one. # * Twisted 19.10 introduces Site.getContentFile which we use to get # temporary upload files placed into a per-node temporary directory. - # * Twisted 21.2.0 is missing binary wheels for some reason. - "Twisted[tls,conch] >= 19.10.0, != 21.2.0", + "Twisted[tls,conch] >= 19.10.0", "PyYAML >= 3.11", From 8af84b7a0cc3c84848e181b933c1e3c4f4f141f7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 11:57:13 -0400 Subject: [PATCH 241/463] Fix tests on Python 3.9 (apparently encoding is no-op on Python 3, and Python always assumes utf-8 by default anyway for JSON). --- src/allmydata/test/test_util.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 4c2e98683..a03845ed6 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -527,7 +527,7 @@ class JSONBytes(unittest.TestCase): x = {u"def\N{SNOWMAN}\uFF00": 123} encoded = jsonbytes.dumps_bytes(x) self.assertIsInstance(encoded, bytes) - self.assertEqual(json.loads(encoded, encoding="utf-8"), x) + self.assertEqual(json.loads(encoded), x) def test_any_bytes_unsupported_by_default(self): """By default non-UTF-8 bytes raise error.""" @@ -554,9 +554,8 @@ class JSONBytes(unittest.TestCase): expected, ) self.assertEqual( - json.loads(jsonbytes.dumps(o, any_bytes=True), - encoding="utf-8"), - expected, + json.loads(jsonbytes.dumps(o, any_bytes=True)), + expected ) From f645715b152e2e0c30b20431e0b404a05c128ad2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 11:59:06 -0400 Subject: [PATCH 242/463] Put off pypy3 to a later ticket. --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b60b51b4d..4644b84c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,6 @@ jobs: - 3.7 - 3.8 - 3.9 - - pypy-3.7 steps: From da5c38b7ca318430198eeafe8742b20130e87580 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 12:39:42 -0400 Subject: [PATCH 243/463] Fix a NameError. --- src/allmydata/windows/fixups.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/windows/fixups.py b/src/allmydata/windows/fixups.py index 3b5437a7b..732a792f5 100644 --- a/src/allmydata/windows/fixups.py +++ b/src/allmydata/windows/fixups.py @@ -1,4 +1,5 @@ from __future__ import print_function +from past.builtins import unicode # This code isn't loadable or sensible except on Windows. Importers all know # this and are careful. Normally I would just let an import error from ctypes From 4ff8a2a09ca4776e1c06f3ef5b8a132c08775e75 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 12:59:06 -0400 Subject: [PATCH 244/463] Pretty sure reactor implements IReactorSocket on Windows on Python 3, but that's still not enough for this functionality to work (and it might be a Twisted bug). --- src/allmydata/test/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index d874d07ae..0f2dc7c62 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -405,7 +405,7 @@ class SameProcessStreamEndpointAssigner(object): :return: A two-tuple of (location hint, port endpoint description) as strings. """ - if IReactorSocket.providedBy(reactor): + if sys.platform != "win32" and IReactorSocket.providedBy(reactor): # On this platform, we can reliable pre-allocate a listening port. # Once it is bound we know it will not fail later with EADDRINUSE. s = socket(AF_INET, SOCK_STREAM) From 7c598d6f028ff0027db720a41179003eb782c666 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 13:00:29 -0400 Subject: [PATCH 245/463] News file. --- newsfragments/3676.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3676.minor diff --git a/newsfragments/3676.minor b/newsfragments/3676.minor new file mode 100644 index 000000000..e69de29bb From 8902868b907b1f9a64886081fcab6a8cee7719b0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 13:15:29 -0400 Subject: [PATCH 246/463] Skip Windows Python 3 for now. --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4644b84c7..1f3d57310 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,6 @@ jobs: matrix: os: - macos-latest - - windows-latest - ubuntu-latest python-version: - 2.7 @@ -27,6 +26,11 @@ jobs: - 3.7 - 3.8 - 3.9 + include: + # For now we're only doing Windows on 2.7, will be fixed in + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3701 + - os: windows-latest + python-version: 2.7 steps: From e76aa273205ea416340b1e71358469a00221c651 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 May 2021 09:35:54 -0400 Subject: [PATCH 247/463] News file. --- newsfragments/3699.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3699.minor diff --git a/newsfragments/3699.minor b/newsfragments/3699.minor new file mode 100644 index 000000000..e69de29bb From d25140b847c863cba3e2d2d211b8a70ab8567e04 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 May 2021 09:36:01 -0400 Subject: [PATCH 248/463] Fix flakes on Python 3. --- src/allmydata/test/_win_subprocess.py | 3 +++ src/allmydata/test/web/test_grid.py | 1 - src/allmydata/uri.py | 2 +- src/allmydata/windows/registry.py | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/_win_subprocess.py b/src/allmydata/test/_win_subprocess.py index 2c2cb60b4..bf9767e73 100644 --- a/src/allmydata/test/_win_subprocess.py +++ b/src/allmydata/test/_win_subprocess.py @@ -7,6 +7,9 @@ from future.utils import PY3 if PY3: raise RuntimeError("Just use subprocess.Popen") +# This is necessary to pacify flake8 on Python 3, while we're still supporting +# Python 2. +from past.builtins import unicode # -*- coding: utf-8 -*- diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index ef2718df4..edcf32268 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -660,7 +660,6 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi if line] except ValueError: print("response is:", res) - print("undecodeable line was '%s'" % line) raise self.failUnlessReallyEqual(len(units), 5+1) # should be parent-first diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index 70742b7b2..5641771d3 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -16,7 +16,7 @@ if PY2: # Don't import bytes or str, to prevent future's newbytes leaking and # breaking code that only expects normal bytes. from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min # noqa: F401 - str = unicode + from past.builtins import unicode as str from past.builtins import unicode, long diff --git a/src/allmydata/windows/registry.py b/src/allmydata/windows/registry.py index 2c7f03156..a5426cb75 100644 --- a/src/allmydata/windows/registry.py +++ b/src/allmydata/windows/registry.py @@ -32,7 +32,7 @@ def get_registry_setting(key, name, _topkey=None): regkey = winreg.OpenKey(topkey, key) sublen, vallen, timestamp = winreg.QueryInfoKey(regkey) - for validx in xrange(vallen): + for validx in range(vallen): keyname, value, keytype = winreg.EnumValue(regkey, validx) if keyname == name and keytype == winreg.REG_SZ: return value From d52e48f2aa22791b62c643d6bc780056f47b85b0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 May 2021 09:41:30 -0400 Subject: [PATCH 249/463] Run flake8 on Python 3. --- tox.ini | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 4b52fd316..e8c783bb2 100644 --- a/tox.ini +++ b/tox.ini @@ -6,18 +6,18 @@ # Map Python versions in GitHub Actions to tox environments to run. [gh-actions] python = - 2.7: py27-coverage,codechecks + 2.7: py27-coverage,codechecks2 3.6: py36-coverage 3.7: py37-coverage 3.8: py38-coverage - 3.9: py39-coverage,typechecks + 3.9: py39-coverage,typechecks,codechecks3 pypy-3.7: pypy3 [pytest] twisted = 1 [tox] -envlist = typechecks,codechecks,py{27,36,37,38,39}-{coverage},pypy27,pypy3 +envlist = typechecks,codechecks2,codechecks3,py{27,36,37,38,39}-{coverage},pypy27,pypy3 minversion = 2.4 [testenv] @@ -97,7 +97,7 @@ commands = coverage report -[testenv:codechecks] +[testenv:codechecks2] basepython = python2.7 # On macOS, git inside of towncrier needs $HOME. passenv = HOME @@ -128,6 +128,16 @@ commands = python -m towncrier.check --config towncrier.pyproject.toml +[testenv:codechecks3] +basepython = python3 +setenv = + # If no positional arguments are given, try to run the checks on the + # entire codebase, including various pieces of supporting code. + DEFAULT_FILES=src integration static misc setup.py +commands = + flake8 {posargs:{env:DEFAULT_FILES}} + + [testenv:typechecks] basepython = python3 skip_install = True From 4711c9fda3d0ea3d2eac9cbdfe68251f20567411 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 May 2021 09:44:44 -0400 Subject: [PATCH 250/463] More flake fixes. --- integration/test_web.py | 2 ++ integration/util.py | 2 ++ misc/coding_tools/make-canary-files.py | 2 ++ .../provisioning/provisioning.py | 4 +-- misc/simulators/hashbasedsig.py | 28 +++++++++---------- misc/simulators/simulate_load.py | 2 ++ misc/simulators/sizes.py | 7 ++++- 7 files changed, 30 insertions(+), 17 deletions(-) diff --git a/integration/test_web.py b/integration/test_web.py index aab11412f..d48ad3aa6 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -9,6 +9,8 @@ WebAPI *should* do in every situation. It's not clear the latter exists anywhere, however. """ +from past.builtins import unicode + import time import json import urllib2 diff --git a/integration/util.py b/integration/util.py index b72e11c72..cd7f15e84 100644 --- a/integration/util.py +++ b/integration/util.py @@ -1,3 +1,5 @@ +from past.builtins import unicode + import sys import time import json diff --git a/misc/coding_tools/make-canary-files.py b/misc/coding_tools/make-canary-files.py index fa813f047..89f274b38 100644 --- a/misc/coding_tools/make-canary-files.py +++ b/misc/coding_tools/make-canary-files.py @@ -52,6 +52,8 @@ system where Tahoe is installed, or in a source tree with setup.py like this: setup.py run_with_pythonpath -p -c 'misc/make-canary-files.py ARGS..' """ +from past.builtins import cmp + import os, hashlib from twisted.python import usage from allmydata.immutable import upload diff --git a/misc/operations_helpers/provisioning/provisioning.py b/misc/operations_helpers/provisioning/provisioning.py index d6dfc4cd7..d2fa44951 100644 --- a/misc/operations_helpers/provisioning/provisioning.py +++ b/misc/operations_helpers/provisioning/provisioning.py @@ -18,7 +18,7 @@ def factorial(n): factorial(n) with n<0 is -factorial(abs(n)) """ result = 1 - for i in xrange(1, abs(n)+1): + for i in range(1, abs(n)+1): result *= i assert n >= 0 return result @@ -30,7 +30,7 @@ def binomial(n, k): # calculate n!/k! as one product, avoiding factors that # just get canceled P = k+1 - for i in xrange(k+2, n+1): + for i in range(k+2, n+1): P *= i # if you are paranoid: # C, rem = divmod(P, factorial(n-k)) diff --git a/misc/simulators/hashbasedsig.py b/misc/simulators/hashbasedsig.py index dc141744f..dbb9ca504 100644 --- a/misc/simulators/hashbasedsig.py +++ b/misc/simulators/hashbasedsig.py @@ -79,7 +79,7 @@ def make_candidate(B, K, K1, K2, q, T, T_min, L_hash, lg_N, sig_bytes, c_sign, c # Winternitz with B < 4 is never optimal. For example, going from B=4 to B=2 halves the # chain depth, but that is cancelled out by doubling (roughly) the number of digits. -range_B = xrange(4, 33) +range_B = range(4, 33) M = pow(2, lg_M) @@ -100,7 +100,7 @@ def calculate(K, K1, K2, q_max, L_hash, trees): T_min = ceil_div(lg_M - lg_K1, lg_K) last_q = None - for T in xrange(T_min, T_min+21): + for T in range(T_min, T_min+21): # lg(total number of leaf private keys) lg_S = lg_K1 + lg_K*T lg_N = lg_S + lg_K2 @@ -137,14 +137,14 @@ def calculate(K, K1, K2, q_max, L_hash, trees): # We approximate lg(M-x) as lg(M) lg_px_step = lg_M + lg_p - lg_1_p - for x in xrange(1, j): + for x in range(1, j): lg_px[x] = lg_px[x-1] - lg(x) + lg_px_step q = None # Find the minimum acceptable value of q. - for q_cand in xrange(1, q_max+1): + for q_cand in range(1, q_max+1): lg_q = lg(q_cand) - lg_pforge = [lg_px[x] + (lg_q*x - lg_K2)*q_cand for x in xrange(1, j)] + lg_pforge = [lg_px[x] + (lg_q*x - lg_K2)*q_cand for x in range(1, j)] if max(lg_pforge) < -L_hash + lg(j) and lg_px[j-1] + 1.0 < -L_hash: #print("K = %d, K1 = %d, K2 = %d, L_hash = %d, lg_K2 = %.3f, q = %d, lg_pforge_1 = %.3f, lg_pforge_2 = %.3f, lg_pforge_3 = %.3f" # % (K, K1, K2, L_hash, lg_K2, q, lg_pforge_1, lg_pforge_2, lg_pforge_3)) @@ -246,13 +246,13 @@ def search(): K_max = 50 c2 = compressions(2*L_hash) c3 = compressions(3*L_hash) - for dau in xrange(0, 10): + for dau in range(0, 10): a = pow(2, dau) - for tri in xrange(0, ceil_log(30-dau, 3)): + for tri in range(0, ceil_log(30-dau, 3)): x = int(a*pow(3, tri)) h = dau + 2*tri c_x = int(sum_powers(2, dau)*c2 + a*sum_powers(3, tri)*c3) - for y in xrange(1, x+1): + for y in range(1, x+1): if tri > 0: # If the bottom level has arity 3, then for every 2 nodes by which the tree is # imperfect, we can save c3 compressions by pruning 3 leaves back to their parent. @@ -267,16 +267,16 @@ def search(): if y not in trees or (h, c_y, (dau, tri)) < trees[y]: trees[y] = (h, c_y, (dau, tri)) - #for x in xrange(1, K_max+1): + #for x in range(1, K_max+1): # print(x, trees[x]) candidates = [] progress = 0 fuzz = 0 complete = (K_max-1)*(2200-200)/100 - for K in xrange(2, K_max+1): - for K2 in xrange(200, 2200, 100): - for K1 in xrange(max(2, K-fuzz), min(K_max, K+fuzz)+1): + for K in range(2, K_max+1): + for K2 in range(200, 2200, 100): + for K1 in range(max(2, K-fuzz), min(K_max, K+fuzz)+1): candidates += calculate(K, K1, K2, q_max, L_hash, trees) progress += 1 print("searching: %3d %% \r" % (100.0 * progress / complete,), end=' ', file=stderr) @@ -285,7 +285,7 @@ def search(): step = 2.0 bins = {} limit = floor_div(limit_cost, step) - for bin in xrange(0, limit+2): + for bin in range(0, limit+2): bins[bin] = [] for c in candidates: @@ -296,7 +296,7 @@ def search(): # For each in a range of signing times, find the best candidate. best = [] - for bin in xrange(0, limit): + for bin in range(0, limit): candidates = bins[bin] + bins[bin+1] + bins[bin+2] if len(candidates) > 0: best += [min(candidates, key=lambda c: c['sig_bytes'])] diff --git a/misc/simulators/simulate_load.py b/misc/simulators/simulate_load.py index 945d96990..ed80ab842 100644 --- a/misc/simulators/simulate_load.py +++ b/misc/simulators/simulate_load.py @@ -4,6 +4,8 @@ from __future__ import print_function +from past.builtins import cmp + import random SERVER_CAPACITY = 10**12 diff --git a/misc/simulators/sizes.py b/misc/simulators/sizes.py index 1719700fa..eb5f3adbf 100644 --- a/misc/simulators/sizes.py +++ b/misc/simulators/sizes.py @@ -2,6 +2,11 @@ from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import input + + import random, math, re from twisted.python import usage @@ -205,7 +210,7 @@ def graph(): series["alacrity"][file_size] = s.bytes_until_some_data g.plot([ (fs, series["overhead"][fs]) for fs in sizes ]) - raw_input("press return") + input("press return") if __name__ == '__main__': From 036a864dc9979f242ea5f98d14fb972fd443c911 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 May 2021 09:48:30 -0400 Subject: [PATCH 251/463] There are many references to this, so stick to old name. --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index e8c783bb2..a6e5e0cc3 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ # Map Python versions in GitHub Actions to tox environments to run. [gh-actions] python = - 2.7: py27-coverage,codechecks2 + 2.7: py27-coverage,codechecks 3.6: py36-coverage 3.7: py37-coverage 3.8: py38-coverage @@ -17,7 +17,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks2,codechecks3,py{27,36,37,38,39}-{coverage},pypy27,pypy3 +envlist = typechecks,codechecks,codechecks3,py{27,36,37,38,39}-{coverage},pypy27,pypy3 minversion = 2.4 [testenv] @@ -97,7 +97,7 @@ commands = coverage report -[testenv:codechecks2] +[testenv:codechecks] basepython = python2.7 # On macOS, git inside of towncrier needs $HOME. passenv = HOME From 0d093c45df583bed9400004bebd2f3becaa7523d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 May 2021 10:06:12 -0400 Subject: [PATCH 252/463] Test fails on Python 3, instead of spinning until timeout. --- integration/conftest.py | 2 +- integration/test_servers_of_happiness.py | 2 +- integration/util.py | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 533cbdb67..918f2d4c9 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -28,7 +28,7 @@ from twisted.internet.error import ( import pytest import pytest_twisted -from util import ( +from .util import ( _CollectOutputProtocol, _MagicTextProtocol, _DumpOutputProtocol, diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index 1f350eb8e..f8f69b1cc 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -3,7 +3,7 @@ from os.path import join from twisted.internet.error import ProcessTerminated -import util +from . import util import pytest_twisted diff --git a/integration/util.py b/integration/util.py index cd7f15e84..2052d2ae6 100644 --- a/integration/util.py +++ b/integration/util.py @@ -116,6 +116,7 @@ class _MagicTextProtocol(ProcessProtocol): self.exited.callback(None) def outReceived(self, data): + data = unicode(data, sys.stdout.encoding) sys.stdout.write(data) self._output.write(data) if not self.magic_seen.called and self._magic_text in self._output.getvalue(): @@ -123,6 +124,7 @@ class _MagicTextProtocol(ProcessProtocol): self.magic_seen.callback(self) def errReceived(self, data): + data = unicode(data, sys.stderr.encoding) sys.stdout.write(data) From 9c7f943271d028373f8967407477121a4cd0e6a2 Mon Sep 17 00:00:00 2001 From: May-Lee Sia Date: Fri, 7 May 2021 16:10:24 +0200 Subject: [PATCH 253/463] Add suggestion for Tahoe Committers --- CONTRIBUTORS.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CONTRIBUTORS.rst b/CONTRIBUTORS.rst index bde4b014d..70d308398 100644 --- a/CONTRIBUTORS.rst +++ b/CONTRIBUTORS.rst @@ -31,8 +31,7 @@ Contributor Checklist This makes the ``Trac`` ticket close when your PR gets approved. -* Request appropriate review. - +* Request appropriate review - we suggest asking `Tahoe Committers `__ References ---------- From 6df076dc10b22e4b42905875df33a929b95937e1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 May 2021 10:16:46 -0400 Subject: [PATCH 254/463] Bit more progress. --- integration/util.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/integration/util.py b/integration/util.py index 2052d2ae6..bb2f9e8e4 100644 --- a/integration/util.py +++ b/integration/util.py @@ -5,7 +5,7 @@ import time import json from os import mkdir, environ from os.path import exists, join -from six.moves import StringIO +from io import StringIO, BytesIO from functools import partial from subprocess import check_output @@ -59,7 +59,7 @@ class _CollectOutputProtocol(ProcessProtocol): """ def __init__(self): self.done = Deferred() - self.output = StringIO() + self.output = BytesIO() def processEnded(self, reason): if not self.done.called: @@ -73,7 +73,7 @@ class _CollectOutputProtocol(ProcessProtocol): self.output.write(data) def errReceived(self, data): - print("ERR: {}".format(data)) + print("ERR: {!r}".format(data)) self.output.write(data) @@ -94,9 +94,11 @@ class _DumpOutputProtocol(ProcessProtocol): self.done.errback(reason) def outReceived(self, data): + data = unicode(data, sys.stdout.encoding) self._out.write(data) def errReceived(self, data): + data = unicode(data, sys.stdout.encoding) self._out.write(data) @@ -284,7 +286,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam config, u'node', u'log_gatherer.furl', - flog_gatherer.decode("utf-8"), + flog_gatherer, ) write_config(FilePath(config_path), config) created_d.addCallback(created) From b11cc9137b87c6ca443b7244f4de4353be5bd250 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 May 2021 10:48:50 -0400 Subject: [PATCH 255/463] Integration test passed on Python 3. --- integration/test_servers_of_happiness.py | 2 +- src/allmydata/web/root.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index f8f69b1cc..594397999 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -42,4 +42,4 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto assert isinstance(e, ProcessTerminated) output = proto.output.getvalue() - assert "shares could be placed on only" in output + assert b"shares could be placed on only" in output diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index b7dc8b5f4..1debc1d10 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -318,7 +318,7 @@ class Root(MultiFormatResource): } version = server.get_version() if version is not None: - description[u"version"] = version["application-version"] + description[u"version"] = version[b"application-version"] return description From 04fc8e704650e1758a2eac6e5f146092aa04f310 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 May 2021 13:21:23 -0400 Subject: [PATCH 256/463] Port to Python 3. --- integration/test_servers_of_happiness.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index 594397999..b9de0c075 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import sys from os.path import join diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 5f78af626..41270b430 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -22,6 +22,12 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +PORTED_INTEGRATION_TESTS = [ + "integration.test_servers_of_happiness", +] + + # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ "allmydata", From ad05abd2f7fb4e43badc2441ab6d187caa31512d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 May 2021 13:25:57 -0400 Subject: [PATCH 257/463] Just run unit tests normally on Python 3. --- src/allmydata/test/python3_tests.py | 37 ----------------------------- src/allmydata/util/_python3.py | 10 ++------ tox.ini | 3 +-- 3 files changed, 3 insertions(+), 47 deletions(-) delete mode 100644 src/allmydata/test/python3_tests.py diff --git a/src/allmydata/test/python3_tests.py b/src/allmydata/test/python3_tests.py deleted file mode 100644 index 9326caa51..000000000 --- a/src/allmydata/test/python3_tests.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -This module defines the subset of the full test suite which is expected to -pass on Python 3 in a way which makes that suite discoverable by trial. - -This module has been ported to Python 3. -""" - -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -from twisted.python.reflect import ( - namedModule, -) -from twisted.trial.runner import ( - TestLoader, -) -from twisted.trial.unittest import ( - TestSuite, -) - -from allmydata.util._python3 import ( - PORTED_TEST_MODULES, -) - -def testSuite(): - loader = TestLoader() - return TestSuite(list( - loader.loadModule(namedModule(module)) - for module - in PORTED_TEST_MODULES - )) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 5f78af626..4a771a4d5 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -1,14 +1,8 @@ """ Track the port to Python 3. -The two easiest ways to run the part of the test suite which is expected to -pass on Python 3 are:: - - $ tox -e py36 - -and:: - - $ trial allmydata.test.python3_tests +At this point all unit tests have been ported to Python 3, so you can just run +them normally. This module has been ported to Python 3. """ diff --git a/tox.ini b/tox.ini index 4b52fd316..84eafd910 100644 --- a/tox.ini +++ b/tox.ini @@ -62,8 +62,7 @@ extras = test setenv = # Define TEST_SUITE in the environment as an aid to constructing the # correct test command below. - !py36: TEST_SUITE = allmydata - py36: TEST_SUITE = allmydata.test.python3_tests + TEST_SUITE = allmydata commands = # As an aid to debugging, dump all of the Python packages and their From 15399b8eba4fc2dadee9de35c34f8b7b4960739c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 May 2021 13:26:14 -0400 Subject: [PATCH 258/463] News file. --- newsfragments/3705.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3705.minor diff --git a/newsfragments/3705.minor b/newsfragments/3705.minor new file mode 100644 index 000000000..e69de29bb From 930329126d4d041ab4166139afe5fee01c04fa4e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 May 2021 13:28:01 -0400 Subject: [PATCH 259/463] One final test module to port to Python 3 (nominally). --- src/allmydata/test/test_python2_regressions.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+) diff --git a/src/allmydata/test/test_python2_regressions.py b/src/allmydata/test/test_python2_regressions.py index 59b16d011..c641d2dba 100644 --- a/src/allmydata/test/test_python2_regressions.py +++ b/src/allmydata/test/test_python2_regressions.py @@ -2,6 +2,16 @@ Tests to check for Python2 regressions """ +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from unittest import skipUnless from inspect import isclass from twisted.python.modules import getModule @@ -37,10 +47,12 @@ def defined_here(cls, where): """ return cls.__module__ == where + class PythonTwoRegressions(TestCase): """ Regression tests for Python 2 behaviors related to Python 3 porting. """ + @skipUnless(PY2, "No point in running on Python 3.") def test_new_style_classes(self): """ All classes in Tahoe-LAFS are new-style. diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 4a771a4d5..18575d624 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -240,6 +240,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_node", "allmydata.test.test_observer", "allmydata.test.test_pipeline", + "allmydata.test.test_python2_regressions", "allmydata.test.test_python3", "allmydata.test.test_repairer", "allmydata.test.test_runner", From 1fbeb6e54eaa54cf8587e4013908adaddab069ea Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 May 2021 13:43:20 -0400 Subject: [PATCH 260/463] It's gone. --- src/allmydata/util/_python3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 18575d624..51c7ad20f 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -102,7 +102,6 @@ PORTED_MODULES = [ "allmydata.test.matchers", "allmydata.test.mutable", "allmydata.test.mutable.util", - "allmydata.test.python3_tests", "allmydata.test.storage_plugin", "allmydata.test.strategies", "allmydata.test.web", From add20d802430113c949c89d6bdccac8d463b653f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 May 2021 13:42:38 -0400 Subject: [PATCH 261/463] First pass at integration support on Python 3. --- src/allmydata/util/_python3.py | 3 +++ tox.ini | 13 ++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 41270b430..c265424c6 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -23,6 +23,9 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +# Every time a module is added here, also add it to tox.ini environment +# integrations3. Bit of duplication, but it's only a handful of files so quite +# temporary. PORTED_INTEGRATION_TESTS = [ "integration.test_servers_of_happiness", ] diff --git a/tox.ini b/tox.ini index a6e5e0cc3..6a0c628d7 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ [gh-actions] python = 2.7: py27-coverage,codechecks - 3.6: py36-coverage + 3.6: py36-coverage,integration3 3.7: py37-coverage 3.8: py38-coverage 3.9: py39-coverage,typechecks,codechecks3 @@ -97,6 +97,17 @@ commands = coverage report +[testenv:integration3] +basepython = python3 +setenv = + COVERAGE_PROCESS_START=.coveragerc +commands = + # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' + py.test --timeout=1800 --coverage -v {posargs:integration/test_servers_of_happiness.py} + coverage combine + coverage report + + [testenv:codechecks] basepython = python2.7 # On macOS, git inside of towncrier needs $HOME. From 01baa13fca6f71931682210dfa1b7170207244c5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 May 2021 10:10:02 -0400 Subject: [PATCH 262/463] News file --- newsfragments/3703.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3703.minor diff --git a/newsfragments/3703.minor b/newsfragments/3703.minor new file mode 100644 index 000000000..e69de29bb From 7c2f9e7d8c1e5e02d5ee46ded0971b891d7bb26d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 May 2021 10:19:21 -0400 Subject: [PATCH 263/463] Fix test to match real-world types. --- src/allmydata/test/web/test_web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index e73cc12f8..1c9d6b65c 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -218,7 +218,7 @@ class FakeDisplayableServer(StubServer): # type: ignore # tahoe-lafs/ticket/35 return self.connected def get_version(self): return { - "application-version": "1.0" + b"application-version": b"1.0" } def get_permutation_seed(self): return b"" From 905ea9cafd7e2fbf403aeebd54d9befff4022c12 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 May 2021 10:30:15 -0400 Subject: [PATCH 264/463] Make stdio emulation more realistic, to trigger problem that was previously missed by tests. --- src/allmydata/test/common_util.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index caafbb81d..c4ebd7384 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -11,6 +11,7 @@ from future.builtins import str as future_str if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 +import sys import os import time import signal @@ -105,9 +106,15 @@ def run_cli_native(verb, *args, **kwargs): # necessary. This works okay for ASCII and if LANG is set # appropriately. These aren't great constraints so we should move # away from this behavior. + # + # The encoding attribute doesn't change StringIO behavior on Python 2, + # but it's there for realism of the emulation. stdin = StringIO(stdin) + stdin.encoding = sys.stdin.encoding stdout = StringIO() + stdout.encoding = sys.stdout.encoding stderr = StringIO() + stderr.encoding = sys.stderr.encoding else: # The new behavior, the Python 3 behavior, is to accept unicode and # encode it using a specific encoding. For older versions of Python 3, From 150b0fd3a356f6646b84c9a33b09d972c5eb0db6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 May 2021 10:31:12 -0400 Subject: [PATCH 265/463] This code should only run on Python 3. --- src/allmydata/scripts/tahoe_get.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/tahoe_get.py b/src/allmydata/scripts/tahoe_get.py index 769a366a5..150b0457b 100644 --- a/src/allmydata/scripts/tahoe_get.py +++ b/src/allmydata/scripts/tahoe_get.py @@ -1,5 +1,7 @@ from __future__ import print_function +from future.utils import PY3 + from urllib.parse import quote as url_quote from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError @@ -32,7 +34,7 @@ def get(options): outf = stdout # Make sure we can write bytes; on Python 3 stdout is Unicode by # default. - if getattr(outf, "encoding", None) is not None: + if PY3 and getattr(outf, "encoding", None) is not None: outf = outf.buffer while True: data = resp.read(4096) From 9ca404151e563bc7728b73ef5470152d2e21cce4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 May 2021 10:31:45 -0400 Subject: [PATCH 266/463] News file. Empty since this was never released. --- newsfragments/3704.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3704.minor diff --git a/newsfragments/3704.minor b/newsfragments/3704.minor new file mode 100644 index 000000000..e69de29bb From 886c5007b5746fef93f2153e6231ce159d02b8a3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 May 2021 10:38:14 -0400 Subject: [PATCH 267/463] News file. --- newsfragments/3707.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3707.minor diff --git a/newsfragments/3707.minor b/newsfragments/3707.minor new file mode 100644 index 000000000..e69de29bb From 1257fc18c8816e024d519f1db6ad9bae79feccc0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 May 2021 11:10:52 -0400 Subject: [PATCH 268/463] Additional test coverage and corresponding bug fixes for password auth on Python 3. --- src/allmydata/frontends/auth.py | 2 +- src/allmydata/test/test_auth.py | 25 ++++++++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/allmydata/frontends/auth.py b/src/allmydata/frontends/auth.py index f2ac99b8f..376cdebe4 100644 --- a/src/allmydata/frontends/auth.py +++ b/src/allmydata/frontends/auth.py @@ -80,5 +80,5 @@ class AccountFileChecker(object): return defer.fail(error.UnauthorizedLogin()) d = defer.maybeDeferred(creds.checkPassword, correct) - d.addCallback(self._cbPasswordMatch, str(creds.username)) + d.addCallback(self._cbPasswordMatch, creds.username) return d diff --git a/src/allmydata/test/test_auth.py b/src/allmydata/test/test_auth.py index f808f72ab..4922ca5f3 100644 --- a/src/allmydata/test/test_auth.py +++ b/src/allmydata/test/test_auth.py @@ -65,7 +65,7 @@ class AccountFileCheckerKeyTests(unittest.TestCase): avatarId = self.checker.requestAvatarId(key_credentials) return self.assertFailure(avatarId, error.UnauthorizedLogin) - def test_password_auth_user(self): + def test_password_auth_user_with_ssh_key(self): """ AccountFileChecker.requestAvatarId returns a Deferred that fires with UnauthorizedLogin if called with an SSHPrivateKey object for a username @@ -76,6 +76,29 @@ class AccountFileCheckerKeyTests(unittest.TestCase): avatarId = self.checker.requestAvatarId(key_credentials) return self.assertFailure(avatarId, error.UnauthorizedLogin) + def test_password_auth_user_with_correct_password(self): + """ + AccountFileChecker.requestAvatarId returns a Deferred that fires with + the user if the correct password is given. + """ + key_credentials = credentials.UsernamePassword(b"alice", b"password") + d = self.checker.requestAvatarId(key_credentials) + def authenticated(avatarId): + self.assertEqual( + (b"alice", + b"URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111"), + (avatarId.username, avatarId.rootcap)) + return d + + def test_password_auth_user_with_wrong_password(self): + """ + AccountFileChecker.requestAvatarId returns a Deferred that fires with + UnauthorizedLogin if the wrong password is given. + """ + key_credentials = credentials.UsernamePassword(b"alice", b"WRONG") + avatarId = self.checker.requestAvatarId(key_credentials) + return self.assertFailure(avatarId, error.UnauthorizedLogin) + def test_unrecognized_key(self): """ AccountFileChecker.requestAvatarId returns a Deferred that fires with From 1c6a324a9258d5a5ad58601696dc70e27c16ba20 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 May 2021 11:11:11 -0400 Subject: [PATCH 269/463] Fix a byte-to-string bug on Python 3. --- integration/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration/util.py b/integration/util.py index bb2f9e8e4..4f3c40666 100644 --- a/integration/util.py +++ b/integration/util.py @@ -532,7 +532,8 @@ def generate_ssh_key(path): key = RSAKey.generate(2048) key.write_private_key_file(path) with open(path + ".pub", "wb") as f: - f.write(b"%s %s" % (key.get_name(), key.get_base64())) + s = "%s %s" % (key.get_name(), key.get_base64()) + f.write(s.encode("ascii")) def run_in_thread(f): From 430bc51e2979937b8c282ce411563494c6f81380 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 May 2021 11:41:59 -0400 Subject: [PATCH 270/463] SFTP integration tests pass on Python 3. --- src/allmydata/frontends/sftpd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py index 17eca993e..d2d614c77 100644 --- a/src/allmydata/frontends/sftpd.py +++ b/src/allmydata/frontends/sftpd.py @@ -1011,8 +1011,8 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin): PrefixingLogMixin.__init__(self, facility="tahoe.sftp", prefix=username) if noisy: self.log(".__init__(%r, %r, %r)" % (client, rootnode, username), level=NOISY) - self.channelLookup["session"] = session.SSHSession - self.subsystemLookup["sftp"] = FileTransferServer + self.channelLookup[b"session"] = session.SSHSession + self.subsystemLookup[b"sftp"] = FileTransferServer self._client = client self._root = rootnode From 02e699347ed7d53508fea773272ad78d69e150f8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 May 2021 11:49:21 -0400 Subject: [PATCH 271/463] Mark tests as ported. --- src/allmydata/util/_python3.py | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index f821045d5..17e74aa4f 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -22,6 +22,7 @@ if PY2: # temporary. PORTED_INTEGRATION_TESTS = [ "integration.test_servers_of_happiness", + "integration.test_sftp", ] diff --git a/tox.ini b/tox.ini index b8deeeb60..32252a346 100644 --- a/tox.ini +++ b/tox.ini @@ -102,7 +102,7 @@ setenv = COVERAGE_PROCESS_START=.coveragerc commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' - py.test --timeout=1800 --coverage -v {posargs:integration/test_servers_of_happiness.py} + py.test --timeout=1800 --coverage -v {posargs:integration/test_servers_of_happiness.py integration/test_sftp.py} coverage combine coverage report From 37bcaf6c88f3f42ac580e36590759a1109cc3c94 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 May 2021 11:51:41 -0400 Subject: [PATCH 272/463] Port initialization "tests". --- integration/test_aaa_aardvark.py | 9 +++++++++ src/allmydata/util/_python3.py | 1 + tox.ini | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/integration/test_aaa_aardvark.py b/integration/test_aaa_aardvark.py index 4a2ef71a6..28ac4c412 100644 --- a/integration/test_aaa_aardvark.py +++ b/integration/test_aaa_aardvark.py @@ -5,6 +5,15 @@ # You can safely skip any of these tests, it'll just appear to "take # longer" to start the first test as the fixtures get built +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + def test_create_flogger(flog_gatherer): print("Created flog_gatherer") diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 17e74aa4f..9b166f5e0 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -21,6 +21,7 @@ if PY2: # integrations3. Bit of duplication, but it's only a handful of files so quite # temporary. PORTED_INTEGRATION_TESTS = [ + "integration.test_aaa_aardvark", "integration.test_servers_of_happiness", "integration.test_sftp", ] diff --git a/tox.ini b/tox.ini index 32252a346..fac52ad43 100644 --- a/tox.ini +++ b/tox.ini @@ -102,7 +102,7 @@ setenv = COVERAGE_PROCESS_START=.coveragerc commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' - py.test --timeout=1800 --coverage -v {posargs:integration/test_servers_of_happiness.py integration/test_sftp.py} + py.test --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py} coverage combine coverage report From 109829dfcba9a0d840f9a2f43dc5174f1909f3fa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 May 2021 11:58:29 -0400 Subject: [PATCH 273/463] BytesWarnings. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index fac52ad43..7f438ef2d 100644 --- a/tox.ini +++ b/tox.ini @@ -102,7 +102,7 @@ setenv = COVERAGE_PROCESS_START=.coveragerc commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' - py.test --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py} + python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py} coverage combine coverage report From 3f2571480c49d40dc1c54e5ee645d95ddf573135 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 May 2021 12:03:58 -0400 Subject: [PATCH 274/463] Some progress towards passing on Python 3. --- integration/test_streaming_logs.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/integration/test_streaming_logs.py b/integration/test_streaming_logs.py index 52c813f9b..e5a3acedd 100644 --- a/integration/test_streaming_logs.py +++ b/integration/test_streaming_logs.py @@ -5,12 +5,14 @@ from __future__ import ( division, ) +from six import ensure_text + import json from os.path import ( join, ) -from urlparse import ( +from urllib.parse import ( urlsplit, ) @@ -68,7 +70,7 @@ def _connect_client(reactor, api_auth_token, ws_url): factory = WebSocketClientFactory( url=ws_url, headers={ - "Authorization": "{} {}".format(SCHEME, api_auth_token), + "Authorization": "{} {}".format(str(SCHEME, "ascii"), api_auth_token), } ) factory.protocol = _StreamingLogClientProtocol @@ -127,7 +129,7 @@ def _test_streaming_logs(reactor, temp_dir, alice): node_url = cfg.get_config_from_file("node.url") api_auth_token = cfg.get_private_config("api_auth_token") - ws_url = node_url.replace("http://", "ws://") + ws_url = ensure_text(node_url).replace("http://", "ws://") log_url = ws_url + "private/logs/v1" print("Connecting to {}".format(log_url)) From 03c1376a3084f0ea462a8fa571a89f41b88f3996 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 May 2021 13:04:44 -0400 Subject: [PATCH 275/463] Try to fix the test. --- src/allmydata/test/cli/test_create_alias.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_create_alias.py b/src/allmydata/test/cli/test_create_alias.py index 4a252f372..6f6881860 100644 --- a/src/allmydata/test/cli/test_create_alias.py +++ b/src/allmydata/test/cli/test_create_alias.py @@ -10,7 +10,7 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from six import ensure_str +from six import ensure_text from six.moves import StringIO import os.path from twisted.trial import unittest @@ -182,7 +182,7 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) self.assertEqual(len(err), 0, err) - self.failUnlessIn(ensure_str("Alias %s created") % quote_output(etudes_arg), out) + self.failUnlessIn(u"Alias %s created" % ensure_text(quote_output(etudes_arg)), out) aliases = get_aliases(self.get_clientdir()) self.failUnless(aliases[u"\u00E9tudes"].startswith(b"URI:DIR2:")) From d80b6d6c56956dc1e44bd3086ba8cfaa073669cf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 May 2021 13:18:08 -0400 Subject: [PATCH 276/463] Add a couple missing environments. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 6a0c628d7..8ffba7713 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,codechecks3,py{27,36,37,38,39}-{coverage},pypy27,pypy3 +envlist = typechecks,codechecks,codechecks3,py{27,36,37,38,39}-{coverage},pypy27,pypy3,integration,integration3 minversion = 2.4 [testenv] From fd5bda67db0c85ada274c7fb93e0560b65878391 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 May 2021 11:02:15 -0400 Subject: [PATCH 277/463] Note package. --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 09f0d9f99..c53bd2a44 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,8 @@ # test suite on all supported python versions. To use it, "pip install tox" # and then run "tox" from this directory. -# Map Python versions in GitHub Actions to tox environments to run. +# Map Python versions in GitHub Actions to tox environments to run, for use by +# the tox-gh-actions package. [gh-actions] python = 2.7: py27-coverage,codechecks From 9baedc97dbdc827198dcd461ef8d47033bd3d4fe Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 May 2021 11:09:59 -0400 Subject: [PATCH 278/463] Auth token needs to be bytes, apparently. --- src/allmydata/client.py | 3 ++- src/allmydata/test/test_client.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index a6c45643f..aabae9065 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -928,7 +928,8 @@ class _Client(node.Node, pollmixin.PollMixin): random data in "api_auth_token" which must be echoed to API calls. """ - return self.config.get_private_config('api_auth_token') + return self.config.get_private_config( + 'api_auth_token').encode("ascii") def _create_auth_token(self): """ diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 957be5197..fd2837f1d 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -415,7 +415,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): f.write("deadbeef") token = c.get_auth_token() - self.assertEqual("deadbeef", token) + self.assertEqual(b"deadbeef", token) @defer.inlineCallbacks def test_web_staticdir(self): From b8b00fa2da2914020e7fe72ccb9767a04d918e2a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 May 2021 11:13:52 -0400 Subject: [PATCH 279/463] Port to Python 3. --- integration/test_streaming_logs.py | 7 +++++++ src/allmydata/util/_python3.py | 5 +++-- tox.ini | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/integration/test_streaming_logs.py b/integration/test_streaming_logs.py index e5a3acedd..036d30715 100644 --- a/integration/test_streaming_logs.py +++ b/integration/test_streaming_logs.py @@ -1,3 +1,6 @@ +""" +Ported to Python 3. +""" from __future__ import ( print_function, unicode_literals, @@ -5,6 +8,10 @@ from __future__ import ( division, ) +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from six import ensure_text import json diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 9b166f5e0..4bbfc0e68 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -18,12 +18,13 @@ if PY2: # Every time a module is added here, also add it to tox.ini environment -# integrations3. Bit of duplication, but it's only a handful of files so quite -# temporary. +# integrations3. Bit of duplication, but it's only a handful of files and quite +# temporary, just until we've ported them all. PORTED_INTEGRATION_TESTS = [ "integration.test_aaa_aardvark", "integration.test_servers_of_happiness", "integration.test_sftp", + "integration.test_streaming_logs", ] diff --git a/tox.ini b/tox.ini index 7f438ef2d..e6b745491 100644 --- a/tox.ini +++ b/tox.ini @@ -102,7 +102,7 @@ setenv = COVERAGE_PROCESS_START=.coveragerc commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' - python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py} + python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py integration/test_streaming_logs.py} coverage combine coverage report From 9b6067dcd1d05f8e4296c83157bd9447df1c0699 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 May 2021 11:39:32 -0400 Subject: [PATCH 280/463] Make test match reality, and fix corresponding bug. --- src/allmydata/test/test_tor_provider.py | 4 +++- src/allmydata/util/tor_provider.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_tor_provider.py b/src/allmydata/test/test_tor_provider.py index 148d813f5..86d54803a 100644 --- a/src/allmydata/test/test_tor_provider.py +++ b/src/allmydata/test/test_tor_provider.py @@ -14,6 +14,7 @@ import os from twisted.trial import unittest from twisted.internet import defer, error from six.moves import StringIO +from six import ensure_str import mock from ..util import tor_provider from ..scripts import create_node, runner @@ -185,7 +186,8 @@ class CreateOnion(unittest.TestCase): protocol))) txtorcon = mock.Mock() ehs = mock.Mock() - ehs.private_key = b"privkey" + # This appears to be a native string in the real txtorcon object... + ehs.private_key = ensure_str("privkey") ehs.hostname = "ONION.onion" txtorcon.EphemeralHiddenService = mock.Mock(return_value=ehs) ehs.add_to_tor = mock.Mock(return_value=defer.succeed(None)) diff --git a/src/allmydata/util/tor_provider.py b/src/allmydata/util/tor_provider.py index c4c63f61a..4ca19c01c 100644 --- a/src/allmydata/util/tor_provider.py +++ b/src/allmydata/util/tor_provider.py @@ -211,6 +211,8 @@ def create_config(reactor, cli_config): "tor_onion.privkey") privkeyfile = os.path.join(private_dir, "tor_onion.privkey") with open(privkeyfile, "wb") as f: + if isinstance(privkey, str): + privkey = privkey.encode("ascii") f.write(privkey) # tahoe_config_tor: this is a dictionary of keys/values to add to the From 070691caa2a8f204eeda04f815162f2f9baa4c79 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 May 2021 11:54:30 -0400 Subject: [PATCH 281/463] Make sure it always goes to stderr. --- src/allmydata/scripts/runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 7122e499e..f63b9b330 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -185,8 +185,8 @@ def _maybe_enable_eliot_logging(options, reactor): def run(): if six.PY3: - warnings.warn("Support for Python 3 is an incomplete work-in-progress." - " Use at your own risk.") + print("Support for Python 3 is an incomplete work-in-progress." + " Use at your own risk.", file=sys.stderr) if sys.platform == "win32": from allmydata.windows.fixups import initialize From 4043b2fe1fffdef95fb6efa449cd1b5f20dc0dce Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 May 2021 11:56:21 -0400 Subject: [PATCH 282/463] Tests pass on Python 3. --- integration/test_tor.py | 12 +++++++----- integration/util.py | 6 ++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index 3b374f669..bea48ba5d 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -6,7 +6,7 @@ from os.path import join import pytest import pytest_twisted -import util +from . import util from twisted.python.filepath import ( FilePath, @@ -55,7 +55,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne cap = proto.output.getvalue().strip().split()[-1] print("TEH CAP!", cap) - proto = util._CollectOutputProtocol() + proto = util._CollectOutputProtocol(capture_stderr=False) reactor.spawnProcess( proto, sys.executable, @@ -68,7 +68,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne yield proto.done dave_got = proto.output.getvalue().strip() - assert dave_got == open(gold_path, 'r').read().strip() + assert dave_got == open(gold_path, 'rb').read().strip() @pytest_twisted.inlineCallbacks @@ -100,7 +100,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ # Which services should this client connect to? write_introducer(node_dir, "default", introducer_furl) with node_dir.child('tahoe.cfg').open('w') as f: - f.write(''' + node_config = ''' [node] nickname = %(name)s web.port = %(web_port)s @@ -125,7 +125,9 @@ shares.total = 2 'log_furl': flog_gatherer, 'control_port': control_port, 'local_port': control_port + 1000, -}) +} + node_config = node_config.encode("utf-8") + f.write(node_config) print("running") yield util._run_node(reactor, node_dir.path, request, None) diff --git a/integration/util.py b/integration/util.py index 4f3c40666..a233471c7 100644 --- a/integration/util.py +++ b/integration/util.py @@ -57,9 +57,10 @@ class _CollectOutputProtocol(ProcessProtocol): self.output, and callback's on done with all of it after the process exits (for any reason). """ - def __init__(self): + def __init__(self, capture_stderr=True): self.done = Deferred() self.output = BytesIO() + self.capture_stderr = capture_stderr def processEnded(self, reason): if not self.done.called: @@ -74,7 +75,8 @@ class _CollectOutputProtocol(ProcessProtocol): def errReceived(self, data): print("ERR: {!r}".format(data)) - self.output.write(data) + if self.capture_err: + self.output.write(data) class _DumpOutputProtocol(ProcessProtocol): From 02897b9968ef5b1e341c8d0600b2b7dd8fde1be6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 May 2021 13:21:47 -0400 Subject: [PATCH 283/463] Fix typo. --- integration/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/util.py b/integration/util.py index a233471c7..c8ba5d16b 100644 --- a/integration/util.py +++ b/integration/util.py @@ -75,7 +75,7 @@ class _CollectOutputProtocol(ProcessProtocol): def errReceived(self, data): print("ERR: {!r}".format(data)) - if self.capture_err: + if self.capture_stderr: self.output.write(data) From fba23ef91cc95fdc0da8429fd38fed802c96c9fc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 May 2021 13:34:35 -0400 Subject: [PATCH 284/463] Port to Python 3. --- integration/test_tor.py | 10 ++++++++++ src/allmydata/util/_python3.py | 1 + tox.ini | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index bea48ba5d..15d888e36 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -1,5 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import sys from os.path import join diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 4bbfc0e68..2bde5941e 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -25,6 +25,7 @@ PORTED_INTEGRATION_TESTS = [ "integration.test_servers_of_happiness", "integration.test_sftp", "integration.test_streaming_logs", + "integration.test_tor", ] diff --git a/tox.ini b/tox.ini index 7d8e90040..04eae7833 100644 --- a/tox.ini +++ b/tox.ini @@ -102,7 +102,7 @@ setenv = COVERAGE_PROCESS_START=.coveragerc commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' - python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py integration/test_streaming_logs.py} + python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py integration/test_streaming_logs.py integration/test_tor.py} coverage combine coverage report From d517304a7914ee3a73ced2462d2743b70b6e617b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 May 2021 13:47:49 -0400 Subject: [PATCH 285/463] A lot closer to passing on Python 3. --- integration/test_web.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/integration/test_web.py b/integration/test_web.py index d48ad3aa6..8fac6909d 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -12,12 +12,12 @@ exists anywhere, however. from past.builtins import unicode import time -import json -import urllib2 +from urllib.parse import unquote as url_unquote, quote as url_quote import allmydata.uri +from allmydata.util import jsonbytes as json -import util +from . import util import requests import html5lib @@ -66,7 +66,7 @@ def test_upload_download(alice): u"filename": u"boom", } ) - assert data == FILE_CONTENTS + assert unicode(data, "utf-8") == FILE_CONTENTS def test_put(alice): @@ -117,10 +117,10 @@ def test_deep_stats(alice): # when creating a directory, we'll be re-directed to a URL # containing our writecap.. - uri = urllib2.unquote(resp.url) + uri = url_unquote(resp.url) assert 'URI:DIR2:' in uri dircap = uri[uri.find("URI:DIR2:"):].rstrip('/') - dircap_uri = util.node_url(alice.node_dir, "uri/{}".format(urllib2.quote(dircap))) + dircap_uri = util.node_url(alice.node_dir, "uri/{}".format(url_quote(dircap))) # POST a file into this directory FILE_CONTENTS = u"a file in a directory" @@ -147,7 +147,7 @@ def test_deep_stats(alice): k, data = d assert k == u"dirnode" assert len(data['children']) == 1 - k, child = data['children'].values()[0] + k, child = list(data['children'].values())[0] assert k == u"filenode" assert child['size'] == len(FILE_CONTENTS) @@ -198,11 +198,11 @@ def test_status(alice): print("Uploaded data, cap={}".format(cap)) resp = requests.get( - util.node_url(alice.node_dir, u"uri/{}".format(urllib2.quote(cap))), + util.node_url(alice.node_dir, u"uri/{}".format(url_quote(cap))), ) print("Downloaded {} bytes of data".format(len(resp.content))) - assert resp.content == FILE_CONTENTS + assert unicode(resp.content, "ascii") == FILE_CONTENTS resp = requests.get( util.node_url(alice.node_dir, "status"), @@ -221,12 +221,12 @@ def test_status(alice): continue resp = requests.get(util.node_url(alice.node_dir, href)) if href.startswith(u"/status/up"): - assert "File Upload Status" in resp.content - if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content: + assert b"File Upload Status" in resp.content + if b"Total Size: %d" % (len(FILE_CONTENTS),) in resp.content: found_upload = True elif href.startswith(u"/status/down"): - assert "File Download Status" in resp.content - if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content: + assert b"File Download Status" in resp.content + if b"Total Size: %d" % (len(FILE_CONTENTS),) in resp.content: found_download = True # download the specialized event information @@ -299,7 +299,7 @@ def test_directory_deep_check(alice): print("Uploaded data1, cap={}".format(cap1)) resp = requests.get( - util.node_url(alice.node_dir, u"uri/{}".format(urllib2.quote(cap0))), + util.node_url(alice.node_dir, u"uri/{}".format(url_quote(cap0))), params={u"t": u"info"}, ) @@ -440,7 +440,7 @@ def test_introducer_info(introducer): resp = requests.get( util.node_url(introducer.node_dir, u""), ) - assert "Introducer" in resp.content + assert b"Introducer" in resp.content resp = requests.get( util.node_url(introducer.node_dir, u""), @@ -513,6 +513,6 @@ def test_mkdir_with_children(alice): params={u"t": "mkdir-with-children"}, data=json.dumps(meta), ) - assert resp.startswith("URI:DIR2") + assert resp.startswith(b"URI:DIR2") cap = allmydata.uri.from_string(resp) assert isinstance(cap, allmydata.uri.DirectoryURI) From 73c28db8fbcc1990ee16a51644123651a2a9b763 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 11 May 2021 11:50:11 -0600 Subject: [PATCH 286/463] news --- newsfragments/3711.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3711.minor diff --git a/newsfragments/3711.minor b/newsfragments/3711.minor new file mode 100644 index 000000000..e69de29bb From 58f4db849e1880c88179a5f4fa3938cb8c8752bb Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 11 May 2021 11:53:11 -0600 Subject: [PATCH 287/463] update actual-release instructions --- docs/release-checklist.rst | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 75ab74bb1..953904f63 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -189,11 +189,16 @@ is appropriate. Once a release-candidate has marinated for some time then it can be made into a the actual release. -XXX Write this section when doing 1.15.0 actual release - -(In general, this means dropping the "rcX" part of the release and the -tag, uploading those artifacts, uploading to PyPI, ... ) +The actual release follows the same steps as above, with some differences: +- there is no "-rcX" on the end of release names +- the release is uploaded to PyPI (using Twine) +- the version is tagged in Git (ideally using "the tahoe release key" + but can be done with any of the authorized core developers' personal + key) +- the release-candidate branches must be merged back to master after + the release is official (e.g. causing newsfragments to be deleted on + master, etc) Announcing the Release From e2ce207fdd4d94de3b785ca22080c87ecaf6646e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 May 2021 09:13:39 -0400 Subject: [PATCH 288/463] Don't convert bytes to unicode implicitly. --- src/allmydata/web/status.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 0401fb586..158d897f9 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1173,7 +1173,8 @@ class MapupdateStatusElement(Element): def privkey_from(self, req, tag): server = self._update_status.get_privkey_from() if server: - return tag(tags.li("Got privkey from: [%s]" % server.get_name())) + return tag(tags.li("Got privkey from: [%s]" % str( + server.get_name(), "utf-8"))) else: return tag From 079041fc1eb6d284382f39bd307ad8fba81b487e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 May 2021 09:20:49 -0400 Subject: [PATCH 289/463] Port to Python 3. --- integration/test_web.py | 21 +++++++++++++++------ src/allmydata/util/_python3.py | 4 +--- tox.ini | 2 +- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/integration/test_web.py b/integration/test_web.py index 8fac6909d..22f08da82 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -7,9 +7,18 @@ Most of the tests have cursory asserts and encode 'what the WebAPI did at the time of testing' -- not necessarily a cohesive idea of what the WebAPI *should* do in every situation. It's not clear the latter exists anywhere, however. + +Ported to Python 3. """ -from past.builtins import unicode +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import time from urllib.parse import unquote as url_unquote, quote as url_quote @@ -66,7 +75,7 @@ def test_upload_download(alice): u"filename": u"boom", } ) - assert unicode(data, "utf-8") == FILE_CONTENTS + assert str(data, "utf-8") == FILE_CONTENTS def test_put(alice): @@ -97,7 +106,7 @@ def test_helper_status(storage_nodes): resp = requests.get(url) assert resp.status_code >= 200 and resp.status_code < 300 dom = BeautifulSoup(resp.content, "html5lib") - assert unicode(dom.h1.string) == u"Helper Status" + assert str(dom.h1.string) == u"Helper Status" def test_deep_stats(alice): @@ -202,7 +211,7 @@ def test_status(alice): ) print("Downloaded {} bytes of data".format(len(resp.content))) - assert unicode(resp.content, "ascii") == FILE_CONTENTS + assert str(resp.content, "ascii") == FILE_CONTENTS resp = requests.get( util.node_url(alice.node_dir, "status"), @@ -400,9 +409,9 @@ def test_directory_deep_check(alice): for _ in range(5): resp = requests.get(deepcheck_uri) dom = BeautifulSoup(resp.content, "html5lib") - if dom.h1 and u'Results' in unicode(dom.h1.string): + if dom.h1 and u'Results' in str(dom.h1.string): break - if dom.h2 and dom.h2.a and u"Reload" in unicode(dom.h2.a.string): + if dom.h2 and dom.h2.a and u"Reload" in str(dom.h2.a.string): dom = None time.sleep(1) assert dom is not None, "Operation never completed" diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 2bde5941e..c15301c3c 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -17,15 +17,13 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -# Every time a module is added here, also add it to tox.ini environment -# integrations3. Bit of duplication, but it's only a handful of files and quite -# temporary, just until we've ported them all. PORTED_INTEGRATION_TESTS = [ "integration.test_aaa_aardvark", "integration.test_servers_of_happiness", "integration.test_sftp", "integration.test_streaming_logs", "integration.test_tor", + "integration.test_web", ] diff --git a/tox.ini b/tox.ini index 04eae7833..365c5d0e9 100644 --- a/tox.ini +++ b/tox.ini @@ -102,7 +102,7 @@ setenv = COVERAGE_PROCESS_START=.coveragerc commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' - python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py integration/test_streaming_logs.py integration/test_tor.py} + python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration} coverage combine coverage report From 22f8b9b4281e15d3851eb7097e2c076907c621be Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 May 2021 09:25:52 -0400 Subject: [PATCH 290/463] Port to Python 3. --- integration/conftest.py | 10 ++++++++++ integration/util.py | 26 ++++++++++++++++++-------- src/allmydata/util/_python3.py | 4 ++++ 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 918f2d4c9..39ff3b42b 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -1,5 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import sys import shutil from time import sleep diff --git a/integration/util.py b/integration/util.py index c8ba5d16b..7c7a1efd2 100644 --- a/integration/util.py +++ b/integration/util.py @@ -1,4 +1,14 @@ -from past.builtins import unicode +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import sys import time @@ -96,11 +106,11 @@ class _DumpOutputProtocol(ProcessProtocol): self.done.errback(reason) def outReceived(self, data): - data = unicode(data, sys.stdout.encoding) + data = str(data, sys.stdout.encoding) self._out.write(data) def errReceived(self, data): - data = unicode(data, sys.stdout.encoding) + data = str(data, sys.stdout.encoding) self._out.write(data) @@ -120,7 +130,7 @@ class _MagicTextProtocol(ProcessProtocol): self.exited.callback(None) def outReceived(self, data): - data = unicode(data, sys.stdout.encoding) + data = str(data, sys.stdout.encoding) sys.stdout.write(data) self._output.write(data) if not self.magic_seen.called and self._magic_text in self._output.getvalue(): @@ -128,7 +138,7 @@ class _MagicTextProtocol(ProcessProtocol): self.magic_seen.callback(self) def errReceived(self, data): - data = unicode(data, sys.stderr.encoding) + data = str(data, sys.stderr.encoding) sys.stdout.write(data) @@ -269,9 +279,9 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam '--hostname', 'localhost', '--listen', 'tcp', '--webport', web_port, - '--shares-needed', unicode(needed), - '--shares-happy', unicode(happy), - '--shares-total', unicode(total), + '--shares-needed', str(needed), + '--shares-happy', str(happy), + '--shares-total', str(total), '--helper', ] if not storage: diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index c15301c3c..f6cd9752c 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -279,4 +279,8 @@ PORTED_TEST_MODULES = [ "allmydata.test.web.test_web", "allmydata.test.web.test_webish", "allmydata.test.test_windows", + + "integration", + "integration.conftest", + "integration.util", ] From ba03a7b436a6f1f9b26c11ff2209f3e71953bc03 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 May 2021 09:26:14 -0400 Subject: [PATCH 291/463] News file. --- newsfragments/3709.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3709.minor diff --git a/newsfragments/3709.minor b/newsfragments/3709.minor new file mode 100644 index 000000000..e69de29bb From a7153baff26dd30eee2aa71f410acc6a396c8f5e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 12 May 2021 09:52:13 -0400 Subject: [PATCH 292/463] Bump the Nix pkg version to 1.15.1-ish Also make this part of release process. Also generate a good version number for inside the package for Nix. --- docs/release-checklist.rst | 4 ++++ newsfragments/3712.installation | 1 + nix/tahoe-lafs.nix | 39 ++++++++++++++++++++++++++++----- 3 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 newsfragments/3712.installation diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 953904f63..48a65ff0a 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -59,6 +59,10 @@ Create Branch and Apply Updates - summarize major changes - commit it +- update "nix/tahoe-lafs.nix" + + - change the value given for `version` from `OLD.post1` to `NEW.post1` + - update "CREDITS" - are there any new contributors in this release? diff --git a/newsfragments/3712.installation b/newsfragments/3712.installation new file mode 100644 index 000000000..b80e1558b --- /dev/null +++ b/newsfragments/3712.installation @@ -0,0 +1 @@ +The Nix package now includes correct version information. \ No newline at end of file diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 6c3c68343..8370b0943 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -7,7 +7,20 @@ , html5lib, pyutil, distro, configparser }: python.pkgs.buildPythonPackage rec { - version = "1.14.0.dev"; + # Most of the time this is not exactly the release version (eg 1.15.1). + # Give it a `post` component to make it look newer than the release version + # and we'll bump this up at the time of each release. + # + # It's difficult to read the version from Git the way the Python code does + # for two reasons. First, doing so involves populating the Nix expression + # with values from the source. Nix calls this "import from derivation" or + # "IFD" (). This is + # discouraged in most cases - including this one, I think. Second, the + # Python code reads the contents of `.git` to determine its version. `.git` + # is not a reproducable artifact (in the sense of "reproducable builds") so + # it is excluded from the source tree by default. When it is included, the + # package tends to be frequently spuriously rebuilt. + version = "1.15.1.post1"; name = "tahoe-lafs-${version}"; src = lib.cleanSourceWith { src = ../.; @@ -22,20 +35,22 @@ python.pkgs.buildPythonPackage rec { # Build up a bunch of knowledge about what kind of file this is. isTox = type == "directory" && basename == ".tox"; isTrialTemp = type == "directory" && basename == "_trial_temp"; - isVersion = basename == "version.py"; + isVersion = basename == "_version.py"; isBytecode = ext == "pyc" || ext == "pyo"; isBackup = lib.hasSuffix "~" basename; isTemporary = lib.hasPrefix "#" basename && lib.hasSuffix "#" basename; isSymlink = type == "symlink"; + isGit = type == "directory" && basename == ".git"; in # Exclude all these things - ! (isTrialTemp - || isTox + ! (isTox + || isTrialTemp || isVersion || isBytecode || isBackup || isTemporary || isSymlink + || isGit ); }; @@ -60,7 +75,21 @@ python.pkgs.buildPythonPackage rec { # Since we're deleting files, this complains they're missing. For now Nix # is Python 2-only, anyway, so these tests don't add anything yet. rm src/allmydata/test/test_python3.py - ''; + + # Generate _version.py ourselves since we can't rely on the Python code + # extracting the information from the .git directory we excluded. + cat > src/allmydata/_version.py < Date: Wed, 12 May 2021 09:56:57 -0400 Subject: [PATCH 293/463] Test the Nix package version --- nix/tahoe-lafs.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 8370b0943..55804ff2b 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -114,6 +114,13 @@ EOF ]; checkPhase = '' + if ! $out/bin/tahoe --version | grep --fixed-strings "${version}"; then + echo "Package version:" + $out/bin/tahoe --version + echo "Did not contain expected:" + echo "${version}" + exit 1 + fi ${python}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata ''; } From 684989afdebbd0d158860760c8f6ee7cb5a38bfb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 12 May 2021 10:22:13 -0400 Subject: [PATCH 294/463] news fragment --- newsfragments/3713.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3713.minor diff --git a/newsfragments/3713.minor b/newsfragments/3713.minor new file mode 100644 index 000000000..e69de29bb From 15cd4f9335985cd77c764844f8c41dee67bdec68 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 12 May 2021 10:33:39 -0400 Subject: [PATCH 295/463] Replace TravisCI with GA --- README.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index b1f6d2563..97f53190d 100644 --- a/README.rst +++ b/README.rst @@ -6,7 +6,7 @@ Free and Open decentralized data store `Tahoe-LAFS `__ (Tahoe Least-Authority File Store) is the first free software / open-source storage technology that distributes your data across multiple servers. Even if some servers fail or are taken over by an attacker, the entire file store continues to function correctly, preserving your privacy and security. -|Contributor Covenant| |readthedocs| |travis| |circleci| |coveralls| +|Contributor Covenant| |readthedocs| |circleci| |githubactions| |coveralls| Table of contents @@ -118,13 +118,12 @@ See `TGPPL.PDF `__ for why the TGPPL ex :alt: documentation status :target: http://tahoe-lafs.readthedocs.io/en/latest/?badge=latest -.. |travis| image:: https://travis-ci.org/tahoe-lafs/tahoe-lafs.png?branch=master - :alt: build status - :target: https://travis-ci.org/tahoe-lafs/tahoe-lafs - .. |circleci| image:: https://circleci.com/gh/tahoe-lafs/tahoe-lafs.svg?style=svg :target: https://circleci.com/gh/tahoe-lafs/tahoe-lafs +.. |githubactions| image:: https://github.com/tahoe-lafs/tahoe-lafs/actions/workflows/ci/badge.svg + :target: https://github.com/tahoe-lafs/tahoe-lafs/actions + .. |coveralls| image:: https://coveralls.io/repos/github/tahoe-lafs/tahoe-lafs/badge.svg :alt: code coverage :target: https://coveralls.io/github/tahoe-lafs/tahoe-lafs From 2b3bd2d72b12c4948f7372f2079f964af8203e1f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 12 May 2021 10:37:15 -0400 Subject: [PATCH 296/463] maybe it is case sensitive --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 97f53190d..4e049e3f3 100644 --- a/README.rst +++ b/README.rst @@ -121,7 +121,7 @@ See `TGPPL.PDF `__ for why the TGPPL ex .. |circleci| image:: https://circleci.com/gh/tahoe-lafs/tahoe-lafs.svg?style=svg :target: https://circleci.com/gh/tahoe-lafs/tahoe-lafs -.. |githubactions| image:: https://github.com/tahoe-lafs/tahoe-lafs/actions/workflows/ci/badge.svg +.. |githubactions| image:: https://github.com/tahoe-lafs/tahoe-lafs/actions/workflows/CI/badge.svg :target: https://github.com/tahoe-lafs/tahoe-lafs/actions .. |coveralls| image:: https://coveralls.io/repos/github/tahoe-lafs/tahoe-lafs/badge.svg From d1090569e893b96d97fa1f47272513e83fa28b21 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 12 May 2021 10:40:52 -0400 Subject: [PATCH 297/463] literally the basename of the path to the file defining the workflow because sure why not have that extra coupling? --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 4e049e3f3..6c8951fd4 100644 --- a/README.rst +++ b/README.rst @@ -121,7 +121,7 @@ See `TGPPL.PDF `__ for why the TGPPL ex .. |circleci| image:: https://circleci.com/gh/tahoe-lafs/tahoe-lafs.svg?style=svg :target: https://circleci.com/gh/tahoe-lafs/tahoe-lafs -.. |githubactions| image:: https://github.com/tahoe-lafs/tahoe-lafs/actions/workflows/CI/badge.svg +.. |githubactions| image:: https://github.com/tahoe-lafs/tahoe-lafs/actions/workflows/ci.yml/badge.svg :target: https://github.com/tahoe-lafs/tahoe-lafs/actions .. |coveralls| image:: https://coveralls.io/repos/github/tahoe-lafs/tahoe-lafs/badge.svg From 7aa3c9c3baa80287175dfacd5cff2852e02ddfbc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 May 2021 11:06:26 -0400 Subject: [PATCH 298/463] Use passed-in encoding. --- src/allmydata/test/common_util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index c4ebd7384..952e9ce05 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -110,11 +110,11 @@ def run_cli_native(verb, *args, **kwargs): # The encoding attribute doesn't change StringIO behavior on Python 2, # but it's there for realism of the emulation. stdin = StringIO(stdin) - stdin.encoding = sys.stdin.encoding + stdin.encoding = encoding stdout = StringIO() - stdout.encoding = sys.stdout.encoding + stdout.encoding = encoding stderr = StringIO() - stderr.encoding = sys.stderr.encoding + stderr.encoding = encoding else: # The new behavior, the Python 3 behavior, is to accept unicode and # encode it using a specific encoding. For older versions of Python 3, From a0bdc57a5e169a85b71e6d525c4aa4b3f47eb069 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 May 2021 11:10:42 -0400 Subject: [PATCH 299/463] Flake fix. --- src/allmydata/test/common_util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 952e9ce05..9e7f0618c 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -11,7 +11,6 @@ from future.builtins import str as future_str if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 -import sys import os import time import signal From d7ad72f651cd16158364921a573170771142d36c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 May 2021 11:21:22 -0400 Subject: [PATCH 300/463] Just do integration tests as part of normal CI runners. --- .github/workflows/ci.yml | 89 ++++++---------------------------------- tox.ini | 4 +- 2 files changed, 15 insertions(+), 78 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f3d57310..6fd542e0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,19 @@ jobs: python-version: 2.7 steps: + - name: Install Tor [Ubuntu] + if: matrix.os == 'ubuntu-latest' + run: sudo apt install tor + + - name: Install Tor [macOS] + if: matrix.os == 'macos-latest' + run: brew install tor + + - name: Install Tor [Windows] + if: matrix.os == 'windows-latest' + uses: crazy-max/ghaction-chocolatey@v1 + with: + args: install tor # See https://github.com/actions/checkout. A fetch-depth of 0 # fetches all tags and branches. @@ -172,82 +185,6 @@ jobs: # Some magic value required for some magic reason. GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - integration: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: - - macos-latest - - windows-latest - python-version: - - 2.7 - - steps: - - - name: Install Tor [Ubuntu] - if: matrix.os == 'ubuntu-latest' - run: sudo apt install tor - - - name: Install Tor [macOS] - if: matrix.os == 'macos-latest' - run: brew install tor - - - name: Install Tor [Windows] - if: matrix.os == 'windows-latest' - uses: crazy-max/ghaction-chocolatey@v1 - with: - args: install tor - - - name: Check out Tahoe-LAFS sources - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up Python ${{ matrix.python-version }} - if: ${{ matrix.os != 'windows-latest' }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - # See this step under coverage job. - - name: Set up Python ${{ matrix.python-version }} [Windows x86] - if: ${{ matrix.os == 'windows-latest' }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - architecture: 'x86' - - - name: Get pip cache directory - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Use pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Install Python packages - run: | - pip install --upgrade tox - pip list - - - name: Display tool versions - run: python misc/build_helpers/show-tool-versions.py - - - name: Run "tox -e integration" - run: tox -e integration - - - name: Upload eliot.log in case of failure - uses: actions/upload-artifact@v1 - if: failure() - with: - name: integration.eliot.json - path: integration.eliot.json packaging: runs-on: ${{ matrix.os }} diff --git a/tox.ini b/tox.ini index 365c5d0e9..2eddfd766 100644 --- a/tox.ini +++ b/tox.ini @@ -6,11 +6,11 @@ # Map Python versions in GitHub Actions to tox environments to run. [gh-actions] python = - 2.7: py27-coverage,codechecks + 2.7: py27-coverage,codechecks,integration 3.6: py36-coverage,integration3 3.7: py37-coverage 3.8: py38-coverage - 3.9: py39-coverage,typechecks,codechecks3 + 3.9: py39-coverage,typechecks,codechecks3,integration3 pypy-3.7: pypy3 [pytest] From c4dd8dde27237592c87a90d6502bc8fd4a9153b1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 May 2021 11:51:47 -0400 Subject: [PATCH 301/463] Flake. --- src/allmydata/scripts/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index f63b9b330..a17ce3ef4 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -6,7 +6,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -import warnings + import os, sys from six.moves import StringIO import six From 8741c33b3562391e9035ff469621d74d8282fb1d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 May 2021 11:58:02 -0400 Subject: [PATCH 302/463] Needs to be in separate list since it's not checked. --- src/allmydata/util/_python3.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index f6cd9752c..53e5998fb 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -26,6 +26,11 @@ PORTED_INTEGRATION_TESTS = [ "integration.test_web", ] +PORTED_INTEGRATION_MODULES = [ + "integration", + "integration.conftest", + "integration.util", +] # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ @@ -279,8 +284,4 @@ PORTED_TEST_MODULES = [ "allmydata.test.web.test_web", "allmydata.test.web.test_webish", "allmydata.test.test_windows", - - "integration", - "integration.conftest", - "integration.util", ] From c4c053a37d5da1398ab18884436f6856098c9685 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 May 2021 12:02:22 -0400 Subject: [PATCH 303/463] Fix failing tests on Python 3. --- src/allmydata/scripts/runner.py | 6 ++++-- src/allmydata/test/test_system.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index a17ce3ef4..43a3b8807 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -183,10 +183,12 @@ def _maybe_enable_eliot_logging(options, reactor): # Pass on the options so we can dispatch the subcommand. return options +PYTHON_3_WARNING = ("Support for Python 3 is an incomplete work-in-progress." + " Use at your own risk.") + def run(): if six.PY3: - print("Support for Python 3 is an incomplete work-in-progress." - " Use at your own risk.", file=sys.stderr) + print(PYTHON_3_WARNING, file=sys.stderr) if sys.platform == "win32": from allmydata.windows.fixups import initialize diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 12ae846eb..627b6ef29 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -43,6 +43,7 @@ from allmydata.monitor import Monitor from allmydata.mutable.common import NotWriteableError from allmydata.mutable import layout as mutable_layout from allmydata.mutable.publish import MutableData +from allmydata.scripts.runner import PYTHON_3_WARNING from foolscap.api import DeadReferenceError, fireEventually, flushEventualQueue from twisted.python.failure import Failure @@ -2635,7 +2636,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): out, err, rc_or_sig = res self.failUnlessEqual(rc_or_sig, 0, str(res)) if check_stderr: - self.failUnlessEqual(err, b"") + self.assertIn(err.strip(), (b"", PYTHON_3_WARNING.encode("ascii"))) d.addCallback(_run_in_subprocess, "create-alias", "newalias") d.addCallback(_check_succeeded) @@ -2655,7 +2656,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def _check_ls(res): out, err, rc_or_sig = res self.failUnlessEqual(rc_or_sig, 0, str(res)) - self.failUnlessEqual(err, b"", str(res)) + self.assertIn(err.strip(), (b"", PYTHON_3_WARNING.encode("ascii"))) self.failUnlessIn(b"tahoe-moved", out) self.failIfIn(b"tahoe-file", out) d.addCallback(_check_ls) From f2ac7814f650075ac2a728b0358d2c7b015ca2d5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 13 May 2021 08:48:08 -0400 Subject: [PATCH 304/463] make positive version check result visible in build logs --- nix/tahoe-lafs.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 55804ff2b..8005ca50b 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -120,6 +120,8 @@ EOF echo "Did not contain expected:" echo "${version}" exit 1 + else + echo "Version string contained expected value \"${version}.\"" fi ${python}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata ''; From 914f1d171cf241b3c55b69f1572a9b9f7ba28011 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 May 2021 10:19:11 -0400 Subject: [PATCH 305/463] Let's not run integration tests as part of unit tests. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8ffba7713..ca5a2c689 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ [gh-actions] python = 2.7: py27-coverage,codechecks - 3.6: py36-coverage,integration3 + 3.6: py36-coverage 3.7: py37-coverage 3.8: py38-coverage 3.9: py39-coverage,typechecks,codechecks3 From dc7caa00c5b17467b87c709c4e55df7817ad8d9a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 May 2021 10:25:48 -0400 Subject: [PATCH 306/463] Run Python 3 integration tests. --- .github/workflows/ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f3d57310..89d35fdc1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -182,6 +182,9 @@ jobs: - windows-latest python-version: - 2.7 + include: + - os: ubuntu-latest + python-version: 3.6 steps: @@ -239,9 +242,14 @@ jobs: - name: Display tool versions run: python misc/build_helpers/show-tool-versions.py - - name: Run "tox -e integration" + - name: Run "Python 2 integration tests" + if: ${{ matrix.python-version == '2.7' }} run: tox -e integration + - name: Run "Python 3 integration tests" + if: ${{ matrix.python-version != '2.7' }} + run: tox -e integration3 + - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v1 if: failure() From 2f7ba77a9f3433f5b3be6856e0459670b44161a8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 May 2021 10:26:30 -0400 Subject: [PATCH 307/463] Note the version being run. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index ca5a2c689..34698e8be 100644 --- a/tox.ini +++ b/tox.ini @@ -102,6 +102,7 @@ basepython = python3 setenv = COVERAGE_PROCESS_START=.coveragerc commands = + python --version # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' py.test --timeout=1800 --coverage -v {posargs:integration/test_servers_of_happiness.py} coverage combine From 4c1f65d97a7884bd1fdac2f7c00d46cce3d1f994 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 May 2021 10:52:12 -0400 Subject: [PATCH 308/463] Test for leading slash check. --- src/allmydata/test/cli/test_put.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/allmydata/test/cli/test_put.py b/src/allmydata/test/cli/test_put.py index c6a577074..d916ae7a2 100644 --- a/src/allmydata/test/cli/test_put.py +++ b/src/allmydata/test/cli/test_put.py @@ -486,3 +486,21 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc_out_err[1], DATA)) return d + + def test_no_leading_slash(self): + self.basedir = "cli/Put/leading_slash" + self.set_up_grid(oneshare=True) + + DATA1 = b"data" * 100 + fn1 = os.path.join(self.basedir, "DATA1") + + d = self.do_cli("create-alias", "tahoe") + d.addCallback(lambda res: + self.do_cli("put", fn1, "tahoe:/uploaded.txt")) + def _check(args): + (rc, out, err) = args + self.assertEqual(rc, 1) + self.failUnlessIn("must not start with a slash", err) + self.assertEqual(len(out), 0, out) + d.addCallback(_check) + return d From 73a3e82856047954103890c6e30e429051591473 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 May 2021 11:29:11 -0400 Subject: [PATCH 309/463] Don't run integration tests here either. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index cb134ffd2..964403a40 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ python = 3.6: py36-coverage 3.7: py37-coverage 3.8: py38-coverage - 3.9: py39-coverage,typechecks,codechecks3,integration3 + 3.9: py39-coverage,typechecks,codechecks3 pypy-3.7: pypy3 [pytest] From c98d90bf74675694cfa56bbcf55710a91b399c20 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 May 2021 09:41:49 -0400 Subject: [PATCH 310/463] News file. --- newsfragments/3716.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3716.minor diff --git a/newsfragments/3716.minor b/newsfragments/3716.minor new file mode 100644 index 000000000..e69de29bb From 250ca3170db995c0f8579b239c4692070e0a7d14 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 May 2021 09:42:13 -0400 Subject: [PATCH 311/463] Port to Python 3. --- src/allmydata/scripts/admin.py | 19 ++++++++++++++----- src/allmydata/util/_python3.py | 1 + 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index abe3d093c..a9feed0dd 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -1,6 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from past.builtins import unicode +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from six import ensure_binary try: @@ -25,9 +34,9 @@ def print_keypair(options): from allmydata.crypto import ed25519 out = options.stdout private_key, public_key = ed25519.create_signing_keypair() - print("private:", unicode(ed25519.string_from_signing_key(private_key), "ascii"), + print("private:", str(ed25519.string_from_signing_key(private_key), "ascii"), file=out) - print("public:", unicode(ed25519.string_from_verifying_key(public_key), "ascii"), + print("public:", str(ed25519.string_from_verifying_key(public_key), "ascii"), file=out) class DerivePubkeyOptions(BaseOptions): @@ -52,8 +61,8 @@ def derive_pubkey(options): privkey_vs = options.privkey privkey_vs = ensure_binary(privkey_vs) private_key, public_key = ed25519.signing_keypair_from_string(privkey_vs) - print("private:", unicode(ed25519.string_from_signing_key(private_key), "ascii"), file=out) - print("public:", unicode(ed25519.string_from_verifying_key(public_key), "ascii"), file=out) + print("private:", str(ed25519.string_from_signing_key(private_key), "ascii"), file=out) + print("public:", str(ed25519.string_from_verifying_key(public_key), "ascii"), file=out) return 0 class AdminCommand(BaseOptions): diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 4bbfc0e68..dbdd8b92b 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -88,6 +88,7 @@ PORTED_MODULES = [ "allmydata.node", "allmydata.nodemaker", "allmydata.scripts", + "allmydata.scripts.admin", "allmydata.scripts.create_node", "allmydata.scripts.runner", "allmydata.scripts.types_", From 66f53fada8a8e5d0d5de4140358cb517a2370a6c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 May 2021 09:46:15 -0400 Subject: [PATCH 312/463] Port to Python 3. --- src/allmydata/scripts/backupdb.py | 10 ++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 11 insertions(+) diff --git a/src/allmydata/scripts/backupdb.py b/src/allmydata/scripts/backupdb.py index 1bffbbfc3..c7827e56e 100644 --- a/src/allmydata/scripts/backupdb.py +++ b/src/allmydata/scripts/backupdb.py @@ -1,5 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os.path, sys, time, random, stat from allmydata.util.netstring import netstring diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index dbdd8b92b..510e50d35 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -89,6 +89,7 @@ PORTED_MODULES = [ "allmydata.nodemaker", "allmydata.scripts", "allmydata.scripts.admin", + "allmydata.scripts.backupdb", "allmydata.scripts.create_node", "allmydata.scripts.runner", "allmydata.scripts.types_", From 6a619b1ec74cf8c4d76dbb20fd51f97d24c582aa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 May 2021 10:05:08 -0400 Subject: [PATCH 313/463] Make sure not to install Python 3-only version of pefile. --- newsfragments/3717.minor | 0 tox.ini | 2 ++ 2 files changed, 2 insertions(+) create mode 100644 newsfragments/3717.minor diff --git a/newsfragments/3717.minor b/newsfragments/3717.minor new file mode 100644 index 000000000..e69de29bb diff --git a/tox.ini b/tox.ini index 5f500b49d..e7ce93f88 100644 --- a/tox.ini +++ b/tox.ini @@ -281,6 +281,8 @@ deps = # PyInstaller 4.0 drops Python 2 support. When we finish porting to # Python 3 we can reconsider this constraint. pyinstaller < 4.0 + # 2021.5.13 broke on Windows. See https://github.com/erocarrera/pefile/issues/318 + pefile < 2021.5.13 ; platform_system == "Windows" # Setting PYTHONHASHSEED to a known value assists with reproducible builds. # See https://pyinstaller.readthedocs.io/en/stable/advanced-topics.html#creating-a-reproducible-build setenv=PYTHONHASHSEED=1 From a2280b7660baaa7623faf2c91151819bbebe285a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 May 2021 09:57:09 -0400 Subject: [PATCH 314/463] Port to Python 3. --- src/allmydata/scripts/cli.py | 18 ++++++++++++++---- src/allmydata/test/cli/test_backup.py | 10 ++++++---- src/allmydata/util/_python3.py | 3 ++- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 011dc3b21..41695aa47 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -1,6 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from past.builtins import unicode +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os.path, re, fnmatch @@ -38,7 +47,7 @@ class FileStoreOptions(BaseOptions): # compute a node-url from the existing options, put in self['node-url'] if self['node-url']: - if (not isinstance(self['node-url'], (bytes, unicode)) + if (not isinstance(self['node-url'], (bytes, str)) or not NODEURL_RE.match(self['node-url'])): msg = ("--node-url is required to be a string and look like " "\"http://HOSTNAMEORADDR:PORT\", not: %r" % @@ -354,8 +363,9 @@ class BackupOptions(FileStoreOptions): abs_filepath = argv_to_abspath(filepath) try: exclude_file = open(abs_filepath) - except: - raise BackupConfigurationError('Error opening exclude file %s.' % quote_local_unicode_path(abs_filepath)) + except Exception as e: + raise BackupConfigurationError('Error opening exclude file %s. (Error: %s)' % ( + quote_local_unicode_path(abs_filepath), e)) try: for line in exclude_file: self.opt_exclude(line) diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index 4bd4b8bfa..4cc7e865f 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -9,9 +9,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import __builtin__ as builtins -else: - import builtins + import os.path from six.moves import cStringIO as StringIO @@ -430,7 +428,11 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): else: return original_open(name, *args) - patcher = MonkeyPatcher((builtins, 'open', call_file)) + if PY2: + from allmydata.scripts import cli as module_to_patch + else: + import builtins as module_to_patch + patcher = MonkeyPatcher((module_to_patch, 'open', call_file)) patcher.runWithPatches(parse_options, basedir, "backup", ['--exclude-from', unicode_to_argv(exclude_file), 'from', 'to']) self.failUnless(ns.called) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 510e50d35..7767a52d8 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -89,7 +89,8 @@ PORTED_MODULES = [ "allmydata.nodemaker", "allmydata.scripts", "allmydata.scripts.admin", - "allmydata.scripts.backupdb", + "allmydata.scripts.backupdb" + "allmydata.scripts.cli", "allmydata.scripts.create_node", "allmydata.scripts.runner", "allmydata.scripts.types_", From bcf2374dfe0ff83f9014c84519436f31c6b6922d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 May 2021 10:27:07 -0400 Subject: [PATCH 315/463] Port to Python 3. --- src/allmydata/scripts/common_http.py | 10 ++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 11 insertions(+) diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py index a53f6baa8..bf47f2dd6 100644 --- a/src/allmydata/scripts/common_http.py +++ b/src/allmydata/scripts/common_http.py @@ -1,5 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os from io import BytesIO from six.moves import urllib, http_client diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 7767a52d8..67a93198b 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -91,6 +91,7 @@ PORTED_MODULES = [ "allmydata.scripts.admin", "allmydata.scripts.backupdb" "allmydata.scripts.cli", + "allmydata.scripts.common_http", "allmydata.scripts.create_node", "allmydata.scripts.runner", "allmydata.scripts.types_", From e3f079555086e0180df1d9eb8750d7c42658766a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 May 2021 10:31:26 -0400 Subject: [PATCH 316/463] Port to Python 3. --- src/allmydata/scripts/common.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index bc969de93..e2f13b262 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -1,7 +1,19 @@ # coding: utf-8 +""" +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + + import os, sys, textwrap import codecs from os.path import join diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 67a93198b..eee8bf65b 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -92,6 +92,7 @@ PORTED_MODULES = [ "allmydata.scripts.backupdb" "allmydata.scripts.cli", "allmydata.scripts.common_http", + "allmydata.scripts.common", "allmydata.scripts.create_node", "allmydata.scripts.runner", "allmydata.scripts.types_", From 7c170317c00a33764b5967e6607384f6fde9afd1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 May 2021 10:41:53 -0400 Subject: [PATCH 317/463] Remove some cruft. --- src/allmydata/scripts/common.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index e2f13b262..0a9ab8714 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -12,6 +12,8 @@ from __future__ import print_function from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +else: + from typing import Union import os, sys, textwrap @@ -29,13 +31,6 @@ from yaml import ( safe_dump, ) -# Python 2 compatibility -from future.utils import PY2 -if PY2: - from future.builtins import str # noqa: F401 -else: - from typing import Union - from twisted.python import usage from allmydata.util.assertutil import precondition From 0655b020505529f0182e808d992fd10224f35b52 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 May 2021 10:50:04 -0400 Subject: [PATCH 318/463] Port to Python 3. --- src/allmydata/scripts/debug.py | 104 ++++++++++++++++++--------------- src/allmydata/util/_python3.py | 1 + 2 files changed, 57 insertions(+), 48 deletions(-) diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index ba40519de..2d6ba4602 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -1,12 +1,20 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2, bchr +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + try: from allmydata.scripts.types_ import SubCommands except ImportError: pass -from future.utils import bchr -from past.builtins import unicode # do not import any allmydata modules at this level. Do that from inside # individual functions instead. @@ -94,7 +102,7 @@ def dump_immutable_chk_share(f, out, options): def to_string(v): if isinstance(v, bytes): - return unicode(v, "utf-8") + return str(v, "utf-8") else: return str(v) @@ -173,9 +181,9 @@ def format_expiration_time(expiration_time): remains = expiration_time - now when = "%ds" % remains if remains > 24*3600: - when += " (%d days)" % (remains / (24*3600)) + when += " (%d days)" % (remains // (24*3600)) elif remains > 3600: - when += " (%d hours)" % (remains / 3600) + when += " (%d hours)" % (remains // 3600) return when @@ -205,7 +213,7 @@ def dump_mutable_share(options): print(file=out) print("Mutable slot found:", file=out) print(" share_type: %s" % share_type, file=out) - print(" write_enabler: %s" % unicode(base32.b2a(WE), "utf-8"), file=out) + print(" write_enabler: %s" % str(base32.b2a(WE), "utf-8"), file=out) print(" WE for nodeid: %s" % idlib.nodeid_b2a(nodeid), file=out) print(" num_extra_leases: %d" % num_extra_leases, file=out) print(" container_size: %d" % container_size, file=out) @@ -217,8 +225,8 @@ def dump_mutable_share(options): print(" ownerid: %d" % lease.owner_num, file=out) when = format_expiration_time(lease.expiration_time) print(" expires in %s" % when, file=out) - print(" renew_secret: %s" % unicode(base32.b2a(lease.renew_secret), "utf-8"), file=out) - print(" cancel_secret: %s" % unicode(base32.b2a(lease.cancel_secret), "utf-8"), file=out) + print(" renew_secret: %s" % str(base32.b2a(lease.renew_secret), "utf-8"), file=out) + print(" cancel_secret: %s" % str(base32.b2a(lease.cancel_secret), "utf-8"), file=out) print(" secrets are for nodeid: %s" % idlib.nodeid_b2a(lease.nodeid), file=out) else: print("No leases.", file=out) @@ -266,8 +274,8 @@ def dump_SDMF_share(m, length, options): print(" SDMF contents:", file=out) print(" seqnum: %d" % seqnum, file=out) - print(" root_hash: %s" % unicode(base32.b2a(root_hash), "utf-8"), file=out) - print(" IV: %s" % unicode(base32.b2a(IV), "utf-8"), file=out) + print(" root_hash: %s" % str(base32.b2a(root_hash), "utf-8"), file=out) + print(" IV: %s" % str(base32.b2a(IV), "utf-8"), file=out) print(" required_shares: %d" % k, file=out) print(" total_shares: %d" % N, file=out) print(" segsize: %d" % segsize, file=out) @@ -360,7 +368,7 @@ def dump_MDMF_share(m, length, options): print(" MDMF contents:", file=out) print(" seqnum: %d" % seqnum, file=out) - print(" root_hash: %s" % unicode(base32.b2a(root_hash), "utf-8"), file=out) + print(" root_hash: %s" % str(base32.b2a(root_hash), "utf-8"), file=out) #print(" IV: %s" % base32.b2a(IV), file=out) print(" required_shares: %d" % k, file=out) print(" total_shares: %d" % N, file=out) @@ -485,19 +493,19 @@ def _dump_secrets(storage_index, secret, nodeid, out): if secret: crs = hashutil.my_renewal_secret_hash(secret) - print(" client renewal secret:", unicode(base32.b2a(crs), "ascii"), file=out) + print(" client renewal secret:", str(base32.b2a(crs), "ascii"), file=out) frs = hashutil.file_renewal_secret_hash(crs, storage_index) - print(" file renewal secret:", unicode(base32.b2a(frs), "ascii"), file=out) + print(" file renewal secret:", str(base32.b2a(frs), "ascii"), file=out) if nodeid: renew = hashutil.bucket_renewal_secret_hash(frs, nodeid) - print(" lease renewal secret:", unicode(base32.b2a(renew), "ascii"), file=out) + print(" lease renewal secret:", str(base32.b2a(renew), "ascii"), file=out) ccs = hashutil.my_cancel_secret_hash(secret) - print(" client cancel secret:", unicode(base32.b2a(ccs), "ascii"), file=out) + print(" client cancel secret:", str(base32.b2a(ccs), "ascii"), file=out) fcs = hashutil.file_cancel_secret_hash(ccs, storage_index) - print(" file cancel secret:", unicode(base32.b2a(fcs), "ascii"), file=out) + print(" file cancel secret:", str(base32.b2a(fcs), "ascii"), file=out) if nodeid: cancel = hashutil.bucket_cancel_secret_hash(fcs, nodeid) - print(" lease cancel secret:", unicode(base32.b2a(cancel), "ascii"), file=out) + print(" lease cancel secret:", str(base32.b2a(cancel), "ascii"), file=out) def dump_uri_instance(u, nodeid, secret, out, show_header=True): from allmydata import uri @@ -508,19 +516,19 @@ def dump_uri_instance(u, nodeid, secret, out, show_header=True): if isinstance(u, uri.CHKFileURI): if show_header: print("CHK File:", file=out) - print(" key:", unicode(base32.b2a(u.key), "ascii"), file=out) - print(" UEB hash:", unicode(base32.b2a(u.uri_extension_hash), "ascii"), file=out) + print(" key:", str(base32.b2a(u.key), "ascii"), file=out) + print(" UEB hash:", str(base32.b2a(u.uri_extension_hash), "ascii"), file=out) print(" size:", u.size, file=out) print(" k/N: %d/%d" % (u.needed_shares, u.total_shares), file=out) - print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) _dump_secrets(u.get_storage_index(), secret, nodeid, out) elif isinstance(u, uri.CHKFileVerifierURI): if show_header: print("CHK Verifier URI:", file=out) - print(" UEB hash:", unicode(base32.b2a(u.uri_extension_hash), "ascii"), file=out) + print(" UEB hash:", str(base32.b2a(u.uri_extension_hash), "ascii"), file=out) print(" size:", u.size, file=out) print(" k/N: %d/%d" % (u.needed_shares, u.total_shares), file=out) - print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) elif isinstance(u, uri.LiteralFileURI): if show_header: @@ -530,52 +538,52 @@ def dump_uri_instance(u, nodeid, secret, out, show_header=True): elif isinstance(u, uri.WriteableSSKFileURI): # SDMF if show_header: print("SDMF Writeable URI:", file=out) - print(" writekey:", unicode(base32.b2a(u.writekey), "ascii"), file=out) - print(" readkey:", unicode(base32.b2a(u.readkey), "ascii"), file=out) - print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) - print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out) + print(" writekey:", str(base32.b2a(u.writekey), "ascii"), file=out) + print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out) print(file=out) if nodeid: we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid) - print(" write_enabler:", unicode(base32.b2a(we), "ascii"), file=out) + print(" write_enabler:", str(base32.b2a(we), "ascii"), file=out) print(file=out) _dump_secrets(u.get_storage_index(), secret, nodeid, out) elif isinstance(u, uri.ReadonlySSKFileURI): if show_header: print("SDMF Read-only URI:", file=out) - print(" readkey:", unicode(base32.b2a(u.readkey), "ascii"), file=out) - print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) - print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out) + print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out) elif isinstance(u, uri.SSKVerifierURI): if show_header: print("SDMF Verifier URI:", file=out) - print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) - print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out) elif isinstance(u, uri.WriteableMDMFFileURI): # MDMF if show_header: print("MDMF Writeable URI:", file=out) - print(" writekey:", unicode(base32.b2a(u.writekey), "ascii"), file=out) - print(" readkey:", unicode(base32.b2a(u.readkey), "ascii"), file=out) - print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) - print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out) + print(" writekey:", str(base32.b2a(u.writekey), "ascii"), file=out) + print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out) print(file=out) if nodeid: we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid) - print(" write_enabler:", unicode(base32.b2a(we), "ascii"), file=out) + print(" write_enabler:", str(base32.b2a(we), "ascii"), file=out) print(file=out) _dump_secrets(u.get_storage_index(), secret, nodeid, out) elif isinstance(u, uri.ReadonlyMDMFFileURI): if show_header: print("MDMF Read-only URI:", file=out) - print(" readkey:", unicode(base32.b2a(u.readkey), "ascii"), file=out) - print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) - print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out) + print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out) elif isinstance(u, uri.MDMFVerifierURI): if show_header: print("MDMF Verifier URI:", file=out) - print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) - print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out) elif isinstance(u, uri.ImmutableDirectoryURI): # CHK-based directory @@ -623,7 +631,7 @@ class FindSharesOptions(BaseOptions): def parseArgs(self, storage_index_s, *nodedirs): from allmydata.util.encodingutil import argv_to_abspath self.si_s = storage_index_s - self.nodedirs = map(argv_to_abspath, nodedirs) + self.nodedirs = list(map(argv_to_abspath, nodedirs)) description = """ Locate all shares for the given storage index. This command looks through one @@ -666,7 +674,7 @@ def find_shares(options): class CatalogSharesOptions(BaseOptions): def parseArgs(self, *nodedirs): from allmydata.util.encodingutil import argv_to_abspath - self.nodedirs = map(argv_to_abspath, nodedirs) + self.nodedirs = list(map(argv_to_abspath, nodedirs)) if not nodedirs: raise usage.UsageError("must specify at least one node directory") @@ -753,7 +761,7 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): print("SDMF %s %d/%d %d #%d:%s %d %s" % \ (si_s, k, N, datalen, - seqnum, unicode(base32.b2a(root_hash), "utf-8"), + seqnum, str(base32.b2a(root_hash), "utf-8"), expiration, quote_output(abs_sharefile)), file=out) elif share_type == "MDMF": from allmydata.mutable.layout import MDMFSlotReadProxy @@ -782,7 +790,7 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): offsets) = verinfo print("MDMF %s %d/%d %d #%d:%s %d %s" % \ (si_s, k, N, datalen, - seqnum, unicode(base32.b2a(root_hash), "utf-8"), + seqnum, str(base32.b2a(root_hash), "utf-8"), expiration, quote_output(abs_sharefile)), file=out) else: print("UNKNOWN mutable %s" % quote_output(abs_sharefile), file=out) @@ -816,7 +824,7 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): ueb_hash = unpacked["UEB_hash"] print("CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize, - unicode(ueb_hash, "utf-8"), expiration, + str(ueb_hash, "utf-8"), expiration, quote_output(abs_sharefile)), file=out) else: @@ -990,7 +998,7 @@ def fixOptionsClass(args): class FlogtoolOptions(foolscap_cli.Options): def __init__(self): super(FlogtoolOptions, self).__init__() - self.subCommands = map(fixOptionsClass, self.subCommands) + self.subCommands = list(map(fixOptionsClass, self.subCommands)) def getSynopsis(self): return "Usage: tahoe [global-options] debug flogtool COMMAND [flogtool-options]" diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index eee8bf65b..5572a5760 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -94,6 +94,7 @@ PORTED_MODULES = [ "allmydata.scripts.common_http", "allmydata.scripts.common", "allmydata.scripts.create_node", + "allmydata.scripts.debug", "allmydata.scripts.runner", "allmydata.scripts.types_", "allmydata.stats", From c92bf72437a96d383e3cf272365b2f4f8a7cfc2e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 May 2021 09:36:43 -0400 Subject: [PATCH 319/463] Add missing comma. --- src/allmydata/util/_python3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 8190949e1..338dd9423 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -92,7 +92,7 @@ PORTED_MODULES = [ "allmydata.nodemaker", "allmydata.scripts", "allmydata.scripts.admin", - "allmydata.scripts.backupdb" + "allmydata.scripts.backupdb", "allmydata.scripts.cli", "allmydata.scripts.common_http", "allmydata.scripts.common", From f656daea7b95636f1fbac461fa184d75850b00cc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 May 2021 09:43:56 -0400 Subject: [PATCH 320/463] News file --- newsfragments/3718.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3718.minor diff --git a/newsfragments/3718.minor b/newsfragments/3718.minor new file mode 100644 index 000000000..e69de29bb From 1e0bf545baf8c1380d58a9a362d1a8a3386d685e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 May 2021 09:47:07 -0400 Subject: [PATCH 321/463] Port to Python 3. --- src/allmydata/scripts/default_nodedir.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+) diff --git a/src/allmydata/scripts/default_nodedir.py b/src/allmydata/scripts/default_nodedir.py index c38e20fa7..00924b8f9 100644 --- a/src/allmydata/scripts/default_nodedir.py +++ b/src/allmydata/scripts/default_nodedir.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import sys import six diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 338dd9423..66144d6c9 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -98,6 +98,7 @@ PORTED_MODULES = [ "allmydata.scripts.common", "allmydata.scripts.create_node", "allmydata.scripts.debug", + "allmydata.scripts.default_nodedir", "allmydata.scripts.runner", "allmydata.scripts.types_", "allmydata.stats", From bf133be19548359f503439cd78936bcb6aa9ca17 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 May 2021 09:56:13 -0400 Subject: [PATCH 322/463] Port to Python 3. --- src/allmydata/scripts/slow_operation.py | 14 +++++++++++--- src/allmydata/util/_python3.py | 1 + 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py index b4b2f8196..620edac34 100644 --- a/src/allmydata/scripts/slow_operation.py +++ b/src/allmydata/scripts/slow_operation.py @@ -1,7 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from future.utils import PY3 -from past.builtins import unicode +from future.utils import PY2, PY3 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from six import ensure_str import os, time @@ -29,7 +37,7 @@ class SlowOperationRunner(object): except UnknownAliasError as e: e.display(stderr) return 1 - path = unicode(path, "utf-8") + path = str(path, "utf-8") if path == '/': path = '' url = nodeurl + "uri/%s" % url_quote(rootcap) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 66144d6c9..bdbdfa5c0 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -100,6 +100,7 @@ PORTED_MODULES = [ "allmydata.scripts.debug", "allmydata.scripts.default_nodedir", "allmydata.scripts.runner", + "allmydata.scripts.slow_operation", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From 1b873126524912c1abc726202b8f9cca66c3be43 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 May 2021 09:58:59 -0400 Subject: [PATCH 323/463] Port to Python 3. --- src/allmydata/scripts/tahoe_add_alias.py | 21 ++++++++++++++------- src/allmydata/util/_python3.py | 1 + 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/allmydata/scripts/tahoe_add_alias.py b/src/allmydata/scripts/tahoe_add_alias.py index 19474b9e8..8476aeb28 100644 --- a/src/allmydata/scripts/tahoe_add_alias.py +++ b/src/allmydata/scripts/tahoe_add_alias.py @@ -1,7 +1,14 @@ -from __future__ import print_function +""" +Ported to Python 3. +""" from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function -from past.builtins import unicode +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os.path import codecs @@ -37,7 +44,7 @@ def add_line_to_aliasfile(aliasfile, alias, cap): def add_alias(options): nodedir = options['node-directory'] alias = options.alias - precondition(isinstance(alias, unicode), alias=alias) + precondition(isinstance(alias, str), alias=alias) cap = options.cap stdout = options.stdout stderr = options.stderr @@ -54,7 +61,7 @@ def add_alias(options): show_output(stderr, "Alias {alias} already exists!", alias=alias) return 1 aliasfile = os.path.join(nodedir, "private", "aliases") - cap = unicode(uri.from_string_dirnode(cap).to_string(), 'utf-8') + cap = str(uri.from_string_dirnode(cap).to_string(), 'utf-8') add_line_to_aliasfile(aliasfile, alias, cap) show_output(stdout, "Alias {alias} added", alias=alias) @@ -64,7 +71,7 @@ def create_alias(options): # mkdir+add_alias nodedir = options['node-directory'] alias = options.alias - precondition(isinstance(alias, unicode), alias=alias) + precondition(isinstance(alias, str), alias=alias) stdout = options.stdout stderr = options.stderr if u":" in alias: @@ -94,7 +101,7 @@ def create_alias(options): # probably check for others.. - add_line_to_aliasfile(aliasfile, alias, unicode(new_uri, "utf-8")) + add_line_to_aliasfile(aliasfile, alias, str(new_uri, "utf-8")) show_output(stdout, "Alias {alias} created", alias=alias) return 0 @@ -114,7 +121,7 @@ def show_output(fp, template, **kwargs): ``encoding`` attribute at all (eg StringIO.StringIO) by writing utf-8-encoded bytes. """ - assert isinstance(template, unicode) + assert isinstance(template, str) # On Python 3 fp has an encoding attribute under all real usage. On # Python 2, the encoding attribute is None if stdio is not a tty. The diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index bdbdfa5c0..0bb9286d7 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -101,6 +101,7 @@ PORTED_MODULES = [ "allmydata.scripts.default_nodedir", "allmydata.scripts.runner", "allmydata.scripts.slow_operation", + "allmydata.scripts.tahoe_add_alias", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From 693e98e3a795159958def17cc0752e95c923aa9f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 May 2021 10:05:44 -0400 Subject: [PATCH 324/463] Port to Python 3. --- src/allmydata/scripts/tahoe_backup.py | 24 ++++++++++++++++-------- src/allmydata/util/_python3.py | 1 + 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/allmydata/scripts/tahoe_backup.py b/src/allmydata/scripts/tahoe_backup.py index 4847f2e8c..b574f16e8 100644 --- a/src/allmydata/scripts/tahoe_backup.py +++ b/src/allmydata/scripts/tahoe_backup.py @@ -1,6 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from past.builtins import unicode +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os.path import time @@ -293,7 +301,7 @@ def collect_backup_targets(root, listdir, filter_children): yield FilenameUndecodableTarget(root, isdir=True) else: for child in filter_children(children): - assert isinstance(child, unicode), child + assert isinstance(child, str), child childpath = os.path.join(root, child) if os.path.islink(childpath): yield LinkTarget(childpath, isdir=False) @@ -498,10 +506,10 @@ class BackupProgress(object): ) def _format_elapsed(self, elapsed): - seconds = elapsed.total_seconds() - hours = int(seconds / 3600) - minutes = int(seconds / 60 % 60) - seconds = int(seconds % 60) + seconds = int(elapsed.total_seconds()) + hours = seconds // 3600 + minutes = (seconds // 60) % 60 + seconds = seconds % 60 return "{}h {}m {}s".format( hours, minutes, @@ -526,12 +534,12 @@ class BackupProgress(object): return self, { os.path.basename(create_path): create_value for (create_path, create_value) - in self._create_contents.items() + in list(self._create_contents.items()) if os.path.dirname(create_path) == dirpath }, { os.path.basename(compare_path): compare_value for (compare_path, compare_value) - in self._compare_contents.items() + in list(self._compare_contents.items()) if os.path.dirname(compare_path) == dirpath } diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 0bb9286d7..6e9dfd2e4 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -102,6 +102,7 @@ PORTED_MODULES = [ "allmydata.scripts.runner", "allmydata.scripts.slow_operation", "allmydata.scripts.tahoe_add_alias", + "allmydata.scripts.tahoe_backup", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From f73f601f67b73535e80457de8304153a8e08dbd9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 May 2021 10:35:16 -0400 Subject: [PATCH 325/463] Port to Python 3. --- src/allmydata/scripts/tahoe_check.py | 32 +++++++++++++++++++--------- src/allmydata/util/_python3.py | 2 ++ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index 82885d073..6bafe3d1a 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -1,19 +1,26 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from six import ensure_str, ensure_text + from urllib.parse import quote as url_quote import json -# Python 2 compatibility -from future.utils import PY2 -if PY2: - from future.builtins import str # noqa: F401 - from twisted.protocols.basic import LineOnlyReceiver from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError from allmydata.scripts.common_http import do_http, format_http_error -from allmydata.util.encodingutil import quote_output, quote_path +from allmydata.util.encodingutil import quote_output, quote_path, get_io_encoding class Checker(object): pass @@ -168,7 +175,9 @@ class DeepCheckOutput(LineOnlyReceiver, object): # LIT files and directories do not have a "summary" field. summary = cr.get("summary", "Healthy (LIT)") - print("%s: %s" % (quote_path(path), quote_output(summary, quotemarks=False)), file=stdout) + # When Python 2 is dropped the ensure_text()/ensure_str() will be unnecessary. + print(ensure_text(ensure_str("%s: %s") % (quote_path(path), quote_output(summary, quotemarks=False)), + encoding=get_io_encoding()), file=stdout) # always print out corrupt shares for shareloc in cr["results"].get("list-corrupt-shares", []): @@ -245,11 +254,14 @@ class DeepCheckAndRepairOutput(LineOnlyReceiver, object): if not path: path = [""] # we don't seem to have a summary available, so build one + # When Python 2 is dropped the ensure_text/ensure_str crap can be + # dropped. if was_healthy: - summary = "healthy" + summary = ensure_str("healthy") else: - summary = "not healthy" - print("%s: %s" % (quote_path(path), summary), file=stdout) + summary = ensure_str("not healthy") + print(ensure_text(ensure_str("%s: %s") % (quote_path(path), summary), + encoding=get_io_encoding()), file=stdout) # always print out corrupt shares prr = crr.get("pre-repair-results", {}) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 6e9dfd2e4..50e429979 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -16,6 +16,7 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + PORTED_INTEGRATION_TESTS = [ "integration.test_aaa_aardvark", "integration.test_servers_of_happiness", @@ -103,6 +104,7 @@ PORTED_MODULES = [ "allmydata.scripts.slow_operation", "allmydata.scripts.tahoe_add_alias", "allmydata.scripts.tahoe_backup", + "allmydata.scripts.tahoe_check", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From 073f521cc1ef22b19bf6b1bc2fc94ae6dbd16c73 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 May 2021 12:59:38 -0400 Subject: [PATCH 326/463] WIP --- src/allmydata/test/cli/test_backup.py | 6 ++++-- src/allmydata/util/encodingutil.py | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index 4cc7e865f..06da8a7f1 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -15,13 +15,14 @@ import os.path from six.moves import cStringIO as StringIO from datetime import timedelta import re +import sys from twisted.trial import unittest from twisted.python.monkey import MonkeyPatcher from allmydata.util import fileutil from allmydata.util.fileutil import abspath_expanduser_unicode -from allmydata.util.encodingutil import get_io_encoding, unicode_to_argv +from allmydata.util.encodingutil import quote_output, unicode_to_argv from allmydata.util.namespace import Namespace from allmydata.scripts import cli, backupdb from ..common_util import StallMixin @@ -372,7 +373,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): def test_exclude_options_unicode(self): nice_doc = u"nice_d\u00F8c.lyx" try: - doc_pattern_arg = u"*d\u00F8c*".encode(get_io_encoding()) + doc_pattern_arg = quote_output(u"*d\u00F8c*", sys.stdout.encoding) except UnicodeEncodeError: raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.") @@ -399,6 +400,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): fileutil.write(excl_filepath, exclusion_string) backup_options = parse(['--exclude-from', excl_filepath, 'from', 'to']) filtered = list(backup_options.filter_listdir(root_listdir)) + import pdb; pdb.set_trace() self._check_filtering(filtered, root_listdir, (u'_darcs', u'subdir'), (nice_doc, u'lib.a')) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 637374064..4cf690e65 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -276,7 +276,10 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): On Python 3, returns Unicode strings. """ precondition(isinstance(s, (bytes, unicode)), s) - encoding = encoding or io_encoding + # Since we're quoting, the assumption is this will be read by a human, and + # therefore printed, so stdout's encoding is the plausible one. io_encoding + # is now always utf-8. + encoding = encoding or getattr(sys.stdout, "encoding") or io_encoding if quote_newlines is None: quote_newlines = quotemarks From 01db93119266067fcd65bca65babe96d978685fb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 18 May 2021 13:35:34 -0400 Subject: [PATCH 327/463] A test that doesn't really increase coverage much. Ideally this would test immutable directories to test caching logic, but that's too much of a pain. Since I already have this written, going to leave in cause why not. --- src/allmydata/test/cli/test_cp.py | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py index d198a832c..7e651b737 100644 --- a/src/allmydata/test/cli/test_cp.py +++ b/src/allmydata/test/cli/test_cp.py @@ -238,6 +238,43 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): return d + @defer.inlineCallbacks + def test_cp_duplicate_directories(self): + self.basedir = "cli/Cp/cp_duplicate_directories" + self.set_up_grid(oneshare=True) + + filename = os.path.join(self.basedir, "file") + data = b"abc\xff\x00\xee" + with open(filename, "wb") as f: + f.write(data) + + yield self.do_cli("create-alias", "tahoe") + (rc, out, err) = yield self.do_cli("mkdir", "tahoe:test1") + self.assertEqual(rc, 0, (rc, err)) + dircap = out.strip() + + (rc, out, err) = yield self.do_cli("cp", filename, "tahoe:test1/file") + self.assertEqual(rc, 0, (rc, err)) + + # Now duplicate dirnode, testing duplicates on destination side: + (rc, out, err) = yield self.do_cli( + "cp", "--recursive", dircap, "tahoe:test2/") + self.assertEqual(rc, 0, (rc, err)) + (rc, out, err) = yield self.do_cli( + "cp", "--recursive", dircap, "tahoe:test3/") + self.assertEqual(rc, 0, (rc, err)) + + (rc, out, err) = yield self.do_cli("ls", "tahoe:test1") + (rc, out, err) = yield self.do_cli("ls", "tahoe:test2") + (rc, out, err) = yield self.do_cli("ls", "tahoe:test3") + + # Now copy to local directory, testing duplicates on origin side: + yield self.do_cli("cp", "--recursive", "tahoe:", self.basedir) + + for i in range(1, 4): + with open(os.path.join(self.basedir, "test%d" % (i,), "file"), "rb") as f: + self.assertEquals(f.read(), data) + def test_cp_replaces_mutable_file_contents(self): self.basedir = "cli/Cp/cp_replaces_mutable_file_contents" self.set_up_grid(oneshare=True) From bb74b7fe0c87120a677dfb7bcaa649fa3a9477a5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 21 May 2021 11:01:00 -0400 Subject: [PATCH 328/463] Note this code is buggy. --- src/allmydata/scripts/tahoe_cp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/scripts/tahoe_cp.py b/src/allmydata/scripts/tahoe_cp.py index fb86ff0ec..df4965dc7 100644 --- a/src/allmydata/scripts/tahoe_cp.py +++ b/src/allmydata/scripts/tahoe_cp.py @@ -701,6 +701,8 @@ class Copier(object): def need_to_copy_bytes(self, source, target): + # This should likley be a method call! but enabling that triggers + # additional bugs. https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3719 if source.need_to_copy_bytes: # mutable tahoe files, and local files return True From 1e9dfd2fcee068640f99996c5058220f12e8672b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 21 May 2021 11:01:17 -0400 Subject: [PATCH 329/463] Another test. --- src/allmydata/test/cli/test_cp.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py index 7e651b737..45efae949 100644 --- a/src/allmydata/test/cli/test_cp.py +++ b/src/allmydata/test/cli/test_cp.py @@ -275,6 +275,33 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): with open(os.path.join(self.basedir, "test%d" % (i,), "file"), "rb") as f: self.assertEquals(f.read(), data) + @defer.inlineCallbacks + def test_cp_immutable_file(self): + self.basedir = "cli/Cp/cp_immutable_file" + self.set_up_grid(oneshare=True) + + filename = os.path.join(self.basedir, "source_file") + data = b"abc\xff\x00\xee" + with open(filename, "wb") as f: + f.write(data) + + # Create immutable file: + yield self.do_cli("create-alias", "tahoe") + (rc, out, _) = yield self.do_cli("put", filename, "tahoe:file1") + filecap = out.strip() + self.assertEqual(rc, 0) + + # Copy it: + (rc, _, _) = yield self.do_cli("cp", "tahoe:file1", "tahoe:file2") + self.assertEqual(rc, 0) + + # Make sure resulting file is the same: + (rc, _, _) = yield self.do_cli("cp", "--recursive", "--caps-only", + "tahoe:", self.basedir) + self.assertEqual(rc, 0) + with open(os.path.join(self.basedir, "file2")) as f: + self.assertEqual(f.read().strip(), filecap) + def test_cp_replaces_mutable_file_contents(self): self.basedir = "cli/Cp/cp_replaces_mutable_file_contents" self.set_up_grid(oneshare=True) From 8ec41fff458890073a8c347b9aee102c2dd7e1ef Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 21 May 2021 11:11:48 -0400 Subject: [PATCH 330/463] News file. --- newsfragments/3714.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3714.minor diff --git a/newsfragments/3714.minor b/newsfragments/3714.minor new file mode 100644 index 000000000..e69de29bb From 9e6915ca78de3e0016175f89897142c6794f5acf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 21 May 2021 11:12:41 -0400 Subject: [PATCH 331/463] Not used. --- src/allmydata/test/cli/test_put.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/cli/test_put.py b/src/allmydata/test/cli/test_put.py index d916ae7a2..03306ab71 100644 --- a/src/allmydata/test/cli/test_put.py +++ b/src/allmydata/test/cli/test_put.py @@ -491,7 +491,6 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.basedir = "cli/Put/leading_slash" self.set_up_grid(oneshare=True) - DATA1 = b"data" * 100 fn1 = os.path.join(self.basedir, "DATA1") d = self.do_cli("create-alias", "tahoe") From 0d5344174fee90a3cdd5c3aa041171ff50111bae Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 21 May 2021 11:57:32 -0400 Subject: [PATCH 332/463] First integration test for put/get. --- integration/test_get_put.py | 51 +++++++++++++++++++++++++++++++++++++ newsfragments/3715.minor | 0 2 files changed, 51 insertions(+) create mode 100644 integration/test_get_put.py create mode 100644 newsfragments/3715.minor diff --git a/integration/test_get_put.py b/integration/test_get_put.py new file mode 100644 index 000000000..16df70d38 --- /dev/null +++ b/integration/test_get_put.py @@ -0,0 +1,51 @@ +""" +Integration tests for getting and putting files, including reading from stdin +and stdout. +""" + +from subprocess import Popen, PIPE + +import pytest + +from .util import run_in_thread, cli + +DATA = b"abc123 this is not utf-8 decodable \xff\x00\x33 \x11" +try: + DATA.decode("utf-8") +except UnicodeDecodeError: + pass # great, what we want +else: + raise ValueError("BUG, the DATA string was decoded from UTF-8") + + +@pytest.fixture(scope="session") +def get_put_alias(alice): + cli(alice, "create-alias", "getput") + + +def read_bytes(path): + with open(path, "rb") as f: + return f.read() + + +@run_in_thread +def test_put_from_stdin(alice, get_put_alias, tmpdir): + """ + It's possible to upload a file via `tahoe put`'s STDIN, and then download + it to a file. + """ + tempfile = str(tmpdir.join("file")) + p = Popen( + ["tahoe", "--node-directory", alice.node_dir, "put", "-", "getput:fromstdin"], + stdin=PIPE + ) + p.stdin.write(DATA) + p.stdin.close() + assert p.wait() == 0 + + cli(alice, "get", "getput:fromstdin", tempfile) + assert read_bytes(tempfile) == DATA + + +def test_get_from_stdin(): + pass diff --git a/newsfragments/3715.minor b/newsfragments/3715.minor new file mode 100644 index 000000000..e69de29bb From 6adda0c43a937fa35a41f3f1961680efd5f8413b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 21 May 2021 12:02:20 -0400 Subject: [PATCH 333/463] A second integration test for get/put. --- integration/test_get_put.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/integration/test_get_put.py b/integration/test_get_put.py index 16df70d38..bbdc363ea 100644 --- a/integration/test_get_put.py +++ b/integration/test_get_put.py @@ -47,5 +47,18 @@ def test_put_from_stdin(alice, get_put_alias, tmpdir): assert read_bytes(tempfile) == DATA -def test_get_from_stdin(): - pass +def test_get_to_stdout(alice, get_put_alias, tmpdir): + """ + It's possible to upload a file, and then download it to stdout. + """ + tempfile = tmpdir.join("file") + with tempfile.open("wb") as f: + f.write(DATA) + cli(alice, "put", str(tempfile), "getput:tostdout") + + p = Popen( + ["tahoe", "--node-directory", alice.node_dir, "get", "getput:tostdout", "-"], + stdout=PIPE + ) + assert p.stdout.read() == DATA + assert p.wait() == 0 From 7e8e93200f016f124d2ab8592950250302b0710b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 May 2021 10:13:08 -0400 Subject: [PATCH 334/463] Don't "fix" things on Python 3 that are already fixed. --- src/allmydata/windows/fixups.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/windows/fixups.py b/src/allmydata/windows/fixups.py index 732a792f5..e8d05a659 100644 --- a/src/allmydata/windows/fixups.py +++ b/src/allmydata/windows/fixups.py @@ -1,4 +1,6 @@ from __future__ import print_function + +from future.utils import PY3 from past.builtins import unicode # This code isn't loadable or sensible except on Windows. Importers all know @@ -122,6 +124,10 @@ def initialize(): SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX) + if PY3: + # The rest of this appears to be Python 2-specific + return + original_stderr = sys.stderr # If any exception occurs in this code, we'll probably try to print it on stderr, From 704da984ac15e0b03e907f9fa8fbdb24168285dd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 May 2021 10:13:30 -0400 Subject: [PATCH 335/463] News file. --- newsfragments/3700.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3700.minor diff --git a/newsfragments/3700.minor b/newsfragments/3700.minor new file mode 100644 index 000000000..e69de29bb From 0966ce17aa74d8a179bbe15e84893a379cf48688 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 May 2021 10:33:21 -0400 Subject: [PATCH 336/463] Try to fix on Windows. --- src/allmydata/test/cli/test_backup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index 4bd4b8bfa..8c544df26 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -12,6 +12,7 @@ if PY2: import __builtin__ as builtins else: import builtins +from six import ensure_str import os.path from six.moves import cStringIO as StringIO @@ -374,7 +375,9 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): def test_exclude_options_unicode(self): nice_doc = u"nice_d\u00F8c.lyx" try: - doc_pattern_arg = u"*d\u00F8c*".encode(get_io_encoding()) + doc_pattern_arg = u"*d\u00F8c*" + if PY2: + doc_pattern_arg = doc_pattern_arg.encode(get_io_encoding()) except UnicodeEncodeError: raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.") @@ -396,7 +399,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): self._check_filtering(filtered, root_listdir, (u'_darcs', u'subdir'), (nice_doc, u'lib.a')) # read exclude patterns from file - exclusion_string = doc_pattern_arg + b"\nlib.?" + exclusion_string = doc_pattern_arg + ensure_str("\nlib.?") excl_filepath = os.path.join(basedir, 'exclusion') fileutil.write(excl_filepath, exclusion_string) backup_options = parse(['--exclude-from', excl_filepath, 'from', 'to']) From 8ca6a72396589d0d8b1a855c3e0f7e3dbaf810cf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 May 2021 10:50:49 -0400 Subject: [PATCH 337/463] Write out the file in an encoding that matches what the reader expects. --- src/allmydata/test/cli/test_backup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index 8c544df26..9a60b0285 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -6,7 +6,7 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future.utils import PY2 +from future.utils import PY2, PY3 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import __builtin__ as builtins @@ -18,6 +18,7 @@ import os.path from six.moves import cStringIO as StringIO from datetime import timedelta import re +import locale from twisted.trial import unittest from twisted.python.monkey import MonkeyPatcher @@ -400,6 +401,10 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): (nice_doc, u'lib.a')) # read exclude patterns from file exclusion_string = doc_pattern_arg + ensure_str("\nlib.?") + if PY3: + # On Python 2 this gives some garbage encoding. Also on Python 2 we + # expect exclusion string to be bytes. + exclusion_string = exclusion_string.encode(locale.getpreferredencoding(False)) excl_filepath = os.path.join(basedir, 'exclusion') fileutil.write(excl_filepath, exclusion_string) backup_options = parse(['--exclude-from', excl_filepath, 'from', 'to']) From cf299684c29190eb47a43be58ec9e90a58be8093 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 May 2021 10:40:44 -0400 Subject: [PATCH 338/463] Unneeded. --- src/allmydata/test/cli/test_cp.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py index 45efae949..d55bc01bd 100644 --- a/src/allmydata/test/cli/test_cp.py +++ b/src/allmydata/test/cli/test_cp.py @@ -264,10 +264,6 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): "cp", "--recursive", dircap, "tahoe:test3/") self.assertEqual(rc, 0, (rc, err)) - (rc, out, err) = yield self.do_cli("ls", "tahoe:test1") - (rc, out, err) = yield self.do_cli("ls", "tahoe:test2") - (rc, out, err) = yield self.do_cli("ls", "tahoe:test3") - # Now copy to local directory, testing duplicates on origin side: yield self.do_cli("cp", "--recursive", "tahoe:", self.basedir) From 2c7a4c7e0745a53b283ca901ffa5aadbc0b5c16d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 May 2021 11:45:36 -0400 Subject: [PATCH 339/463] Try to fix Windows failure on Python 3. --- src/allmydata/test/test_runner.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 7cc89c287..a74447d27 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -17,6 +17,7 @@ from six import ensure_text import os.path, re, sys from os import linesep +import locale from eliot import ( log_call, @@ -92,6 +93,10 @@ def run_bintahoe(extra_argv, python_options=None): argv.extend(extra_argv) argv = list(unicode_to_argv(arg) for arg in argv) p = Popen(argv, stdout=PIPE, stderr=PIPE) + if PY2: + encoding = "utf-8" + else: + encoding = locale.getpreferredencoding(False) out = p.stdout.read().decode("utf-8") err = p.stderr.read().decode("utf-8") returncode = p.wait() @@ -103,7 +108,7 @@ class BinTahoe(common_util.SignalMixin, unittest.TestCase): """ The runner script receives unmangled non-ASCII values in argv. """ - tricky = u"\u2621" + tricky = u"\u00F6" out, err, returncode = run_bintahoe([tricky]) self.assertEqual(returncode, 1) self.assertIn(u"Unknown command: " + tricky, out) From 39334ebc420b0feb47f5af1f2fa92e8aa3193534 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 May 2021 14:12:40 -0400 Subject: [PATCH 340/463] Run tests on Windows with Python 3. --- .github/workflows/ci.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5ab5d5c5..875f21cc9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,7 @@ jobs: fail-fast: false matrix: os: + - windows-latest - macos-latest - ubuntu-latest python-version: @@ -26,11 +27,6 @@ jobs: - 3.7 - 3.8 - 3.9 - include: - # For now we're only doing Windows on 2.7, will be fixed in - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3701 - - os: windows-latest - python-version: 2.7 steps: # See https://github.com/actions/checkout. A fetch-depth of 0 From 19c7de7a7abecc3af9f80a26c636f971d467bf28 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 May 2021 14:13:06 -0400 Subject: [PATCH 341/463] News file. --- newsfragments/3701.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3701.minor diff --git a/newsfragments/3701.minor b/newsfragments/3701.minor new file mode 100644 index 000000000..e69de29bb From 52cd90941fdb7e1f3f8147397d866a05a5badd73 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 May 2021 14:29:22 -0400 Subject: [PATCH 342/463] Actually use the encoding. --- src/allmydata/test/test_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index a74447d27..a9ec83524 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -97,8 +97,8 @@ def run_bintahoe(extra_argv, python_options=None): encoding = "utf-8" else: encoding = locale.getpreferredencoding(False) - out = p.stdout.read().decode("utf-8") - err = p.stderr.read().decode("utf-8") + out = p.stdout.read().decode(encoding) + err = p.stderr.read().decode(encoding) returncode = p.wait() return (out, err, returncode) From 7f70315c83b28f8031eeca53f54c95213c7f1088 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 May 2021 14:49:44 -0400 Subject: [PATCH 343/463] Skip tests on Python 3. --- src/allmydata/test/test_windows.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/test/test_windows.py b/src/allmydata/test/test_windows.py index 01e4a57c1..bae56bfed 100644 --- a/src/allmydata/test/test_windows.py +++ b/src/allmydata/test/test_windows.py @@ -79,6 +79,7 @@ slow_settings = settings( ) @skipUnless(platform.isWindows(), "get_argv is Windows-only") +@skipUnless(PY2, "Not used on Python 3.") class GetArgvTests(SyncTestCase): """ Tests for ``get_argv``. @@ -172,6 +173,7 @@ class GetArgvTests(SyncTestCase): @skipUnless(platform.isWindows(), "intended for Windows-only codepaths") +@skipUnless(PY2, "Not used on Python 3.") class UnicodeOutputTests(SyncTestCase): """ Tests for writing unicode to stdout and stderr. From 33d89dbe8f66b84ed864dda23c3c560f0a422f23 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 26 May 2021 07:26:57 -0400 Subject: [PATCH 344/463] Move IRC from freenode.net to libera.chat Following some organizational developments, freenode.net's volunteer staff has quit and formed libera.chat, a new IRC network. Like many other FOSS projects, Tahoe-LAFS project too has decided to move to libera.chat for our IRC needs. --- README.rst | 2 +- newsfragments/3721.documentation | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3721.documentation diff --git a/README.rst b/README.rst index 6c8951fd4..39f2ae582 100644 --- a/README.rst +++ b/README.rst @@ -72,7 +72,7 @@ You can find the full Tahoe-LAFS documentation at our `documentation site `__. +- Chat with Tahoe-LAFS developers at #tahoe-lafs chat on irc.libera.chat or `Slack `__. - Join our `weekly conference calls `__ with core developers and interested community members. diff --git a/newsfragments/3721.documentation b/newsfragments/3721.documentation new file mode 100644 index 000000000..36ae33236 --- /dev/null +++ b/newsfragments/3721.documentation @@ -0,0 +1 @@ +Our IRC channel, #tahoe-lafs, has been moved to irc.libera.chat. From 206d416830699184ea11f50d5ac2345577afeac3 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 26 May 2021 07:48:50 -0400 Subject: [PATCH 345/463] Highlight IRC channel name, link to libera.chat website --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 39f2ae582..b76f614c9 100644 --- a/README.rst +++ b/README.rst @@ -72,7 +72,7 @@ You can find the full Tahoe-LAFS documentation at our `documentation site `__. +- Chat with Tahoe-LAFS developers at `#tahoe-lafs` channel on `libera.chat `__ IRC network or `Slack `__. - Join our `weekly conference calls `__ with core developers and interested community members. From cb61bfbe619d0074e382138ecbba3d01c9071fde Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 26 May 2021 07:51:29 -0400 Subject: [PATCH 346/463] Highlight IRC channel name correctly --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index b76f614c9..1c7a8d6ee 100644 --- a/README.rst +++ b/README.rst @@ -72,7 +72,7 @@ You can find the full Tahoe-LAFS documentation at our `documentation site `__ IRC network or `Slack `__. +- Chat with Tahoe-LAFS developers at ``#tahoe-lafs`` channel on `libera.chat `__ IRC network or `Slack `__. - Join our `weekly conference calls `__ with core developers and interested community members. From e5d636746eaa99a2515213cb1c9c156aa821842c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 26 May 2021 09:16:07 -0400 Subject: [PATCH 347/463] Can't use latest Twisted for typechecks, on Windows older Twisted requires compilation which fails, sticking to 3.7 is workaround since it has Windows wheels. --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 172887a5e..d33fe2eae 100644 --- a/tox.ini +++ b/tox.ini @@ -9,9 +9,9 @@ python = 2.7: py27-coverage,codechecks 3.6: py36-coverage - 3.7: py37-coverage + 3.7: py37-coverage,typechecks,codechecks3 3.8: py38-coverage - 3.9: py39-coverage,typechecks,codechecks3 + 3.9: py39-coverage pypy-3.7: pypy3 [pytest] From 53ab18885dabb8d4a5e8d1ab5b2130bc1d89d0ba Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 26 May 2021 09:19:12 -0400 Subject: [PATCH 348/463] Update docs to point to irc.libera.chat --- docs/running.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/running.rst b/docs/running.rst index 82b0443f9..898afa10b 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -235,7 +235,7 @@ Socialize ========= You can chat with other users of and hackers of this software on the -#tahoe-lafs IRC channel at ``irc.freenode.net``, or on the `tahoe-dev mailing +#tahoe-lafs IRC channel at ``irc.libera.chat``, or on the `tahoe-dev mailing list`_. .. _tahoe-dev mailing list: https://tahoe-lafs.org/cgi-bin/mailman/listinfo/tahoe-dev From 0a9b3a4003420b0b117f0e7b1a55a0753b4f0ad4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 26 May 2021 15:17:57 -0400 Subject: [PATCH 349/463] Don't want pdb. --- src/allmydata/test/cli/test_backup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index c54591644..e9f8087a2 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -409,7 +409,6 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): fileutil.write(excl_filepath, exclusion_string) backup_options = parse(['--exclude-from', excl_filepath, 'from', 'to']) filtered = list(backup_options.filter_listdir(root_listdir)) - import pdb; pdb.set_trace() self._check_filtering(filtered, root_listdir, (u'_darcs', u'subdir'), (nice_doc, u'lib.a')) From 741af0b0eb8b8fe3869f32154d607e1545e1a62f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 26 May 2021 15:27:16 -0400 Subject: [PATCH 350/463] Fix imports. --- src/allmydata/test/cli/test_backup.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index e9f8087a2..6ac55a9e8 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -9,9 +9,6 @@ from __future__ import unicode_literals from future.utils import PY2, PY3 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import __builtin__ as builtins -else: - import builtins from six import ensure_str import os.path @@ -25,7 +22,7 @@ from twisted.python.monkey import MonkeyPatcher from allmydata.util import fileutil from allmydata.util.fileutil import abspath_expanduser_unicode -from allmydata.util.encodingutil import quote_output, unicode_to_argv +from allmydata.util.encodingutil import get_io_encoding, unicode_to_argv from allmydata.util.namespace import Namespace from allmydata.scripts import cli, backupdb from ..common_util import StallMixin From bfca424578f49d34a2fe517761e7b11915440b8b Mon Sep 17 00:00:00 2001 From: Anxhelo Lushka Date: Thu, 27 May 2021 14:47:24 +0200 Subject: [PATCH 351/463] Fixes ticket:3677 --- newsfragments/3677.documentation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3677.documentation diff --git a/newsfragments/3677.documentation b/newsfragments/3677.documentation new file mode 100644 index 000000000..c499324db --- /dev/null +++ b/newsfragments/3677.documentation @@ -0,0 +1 @@ +Improves Tahoe-LAFS logo visibility on the README when GitHub dark/dimmed theme is being used. \ No newline at end of file From 433c80d8df00531e8b2302403056d2e9f024dffe Mon Sep 17 00:00:00 2001 From: Anxhelo Lushka Date: Thu, 27 May 2021 14:51:30 +0200 Subject: [PATCH 352/463] New image with stroking --- docs/_static/media/image2.png | Bin 4431 -> 7769 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/docs/_static/media/image2.png b/docs/_static/media/image2.png index d8704f35955f515c96923e715c7a627c5c206463..90e6b9375611347375405d182c6a029baa78cefd 100644 GIT binary patch literal 7769 zcmbVx2Ut@}*X|Box(JG*7%9>Tp%Vy1dKc*+<%A?OfnX9kNI;Py0s?~cB1O7@fOG_; zN)c&-fYOUp0g-y+`OZ1dcmL@N22{+K!V4%H13jhFvmZqv9 z0D!=xwmA(I>1q-p_=5Cx&Rx?24*-nqzdxYoLW~>$KzY^4*qmUlrwc=2T}6>-tQ|(w z+tr9)87x+&I;U1jp)^CAeb3FsF3KoX} zONmO0prDdcU>P}4sI-KnoSZONTns7;5tD^LB}AaoFiA0(xH$N)7Z*ty4s8!JR8{{= z7U>So(0{P*j#vT~?}+_BQ2#UhzX>3@R!{FA8UIliSJ!_;;0bD;Bs2bU z$bUrRjeXoP5JLw!aI)I3RQa{so*9j1cAAPHEUF&69c_d*%`tuk0e<@dtKf%(mx z+|XDryuiQB!Kfk$7&w=>m^f5KOiDyt#uzFF6O)5MC56PqVPazcg6d(>PWC?k2`Uc# zuTWCdppgXRe+x#VVD?y?E0SchlPl5z195kA-~#{SNtg=O1&bpICTS<}@BLaTDh4>L zy^{-R0B@+F4AxRpk%r1iON)q$LjP1(PY_womlHwwEc1V;6R0;*Pv$sP^BPAvOp0A2UdHjxozvrX>Kj+`ZIgzpu>GD73 z`JJ1;!w076geNVm&tKVNgt`A$%f$)&XT-pesNb0Y=R*As0Sub!uWqORu?PM|>*a_c zk^W!e;$JX4)}G*n#9?kZki7N(5_%Al{*d2k`!7=<|JBMr$NnwF|Av$5#_#4o1&eg^ zPszi$k?I_dRM2z{+-3lPeNIdDma+H5y7jrYCZCR)a_`h=@&8Ov7Sz(pK?|xa-kj<( zu-ATXbi%Dx%Xn?2{1=U-P;RM;!~Fq>OgTqc@dtE$iaBy!R<6sW4(^kx^T zwK!1zIEykt^wM-iY*5&q5*H6oF1}SP4m?-i`gOF~taf&KGM9-m^rr?N{#weo{K%Z0Xl~e)M0v%( zUv`dO<7@f7&ZuhwG<%EB&`7{eYtddig~?R$!8;8hQ)TVZ9nA_cpq$5-yZ&BE^kj9f ztNi}xUF^1CSO_SLw}9iES-qqzAFbkJlR=b1wlF^I?`Zz!;RF*1Zi^nP8v;(Rix0pdr=#7=FIc<2$A86P}%UyA|HPdVcGOVkxEvlS()#@)G461~YxM{cmvD5v$&dAzJG zA=0;Vb+~}Lj9dEpoQLk*yH;TLI&}uxNov^nRlD}$6G#(tS@UUK0B_}J6VC1m(?u%q zv?nE|S71=|uzs`YE2}*QkT37qY8}VkP$tFe}1gDMB_}7bJ16Y ztZ$`)AayFcKaQH4nXE>ps&tU-WNE$;WM4H|V`~{M6Ir=zL0YF)a&Xd8#;G&lcROj+r4d1Q|S1S!ca28QHlWTp8P z%Qh-gGpDxZ2v=i3#Joz5^bpW10&ze;dcNW0T{_)lP!^`uac-+I;9 z?s#L&;sd^-kvK;c@;#jB8`ik50Xoe!yCQp=cZK0{x~fBj(-fS zNL2L3Id;g_q zV14d7JCTbD!ML~o5r}_DM)u4z3vEfJzU*|+S?xNh`jCoY9&2DICyFAnBpmu9$3k8$%>RGItygkL)gysK-U@|gnSuC9R)(D?GwER3+#Td{cQGYybt zl*G;Q0+DEioiMlF6AK!|kfP68Nez|hr6hhi8% zCK|duk?c<+3~6;#OJb)}Cng0Q=m3kr-pIwD(|)((A-2;Jw#@fYl$4aQSE%+O`7;|; zV>YqPR^1^2KJTQl)57+B$qR*-Z2im2Z&=q6wi^=I%V6SwoPE1ID27mAR%jd!;&1Sz zAHJ`o$anA*xqtNNg8o*j(an>?wK9*R9Zl<@kq7&q9pv}d!nmWDh0IF7^o7ture587 zIbYpNn8GAl=LH_R3*JXS4$qN!Z@r^_`jmLopI_Xw)s`**z;yje9Ru7wSn7k9XlV@1 z&G~y0L^vQ1+Yhhy1M!CPHtM`vyKg~*0{N*dU6rYb=D24eIq7ffXSydxngaY*_P9UL zw>+#(=3yd}fj8z^Ryo@pA8f~qZ7M&r;~}7Q2D}D)1IP@rbS{UgeMJti0wbR~*V{kN zZna_3!WQn}9)>bF3Y@->!KNQ?WIkkMY4qL8#*Cr>?LMt7Qng3`y!d#RN~}|(9@Kvj zh#S3~lfF7q)XR=&?0Eeqa{Z981>zq*!f2{|$;LZtx1Gg@|GSmg48Xb_3g-rLR&A6w z#~Xi63#}>a4I4Sd7jOXI{C@fQhpvSScXu0x(*WJqS{4#*H+uVrYOuCbd6!;JV<8;a zrT#SdL*bEC^}O9;>-uWdM|Ij~T}U887I^mDK*xCN3By6!Vz;!-rTk+ zWLnH$W;O)jd&2t($Qe4HvraL$N=@97uOM5#dLM}8<7oMeEOhwV5>%Idj&nglo46z* zl{l2ZFaWT+$miVIO2zk=Q`hR@0q`%m2cZLg``?pxI9`DGZO%UNUp@aW?+}?U5uoUQ zuob_<@recjCrm_j(k|=OL7ay5FkQk53{JVtpjORy?l}j zf1Xxs1q5&qwGn4Qft>zZ?|6p<=m8$3&dn{kEQZ>fndSXv>2+jM>Qg3GhEIp+5zgY> z#{%QZ-#t(v5hB#4X*d5k+#9p?m}Gn+21u#hNRf6GA0O>_n-N{j3!Q4u9~ zYqSq(+oi!|9xFq+p1ntByE~s%EO|f(-idO@k6bGOhhHBuy{b2a>2LYp{JhkeySDb2 z13eX1&LZzcTWb*gYH?i7!$p@=Bbe` z>Ha%f=e*sgCeXeWnH*9;MbEup<+Fo1COHtEeuZjcvZ|wwFRvP@rfox2rM&8uUD4AG zvZF=kdY0TPYc%!$(I@k!c<(L!lTAZ9#HduLd7YXPz1Or)K79nvQj|0Bc-wB{OYQb~ zGChDxW*v5aR`n$AG@Oii`P|m%q~B(v*OA^r&vUi4(Gp(pP&`0~zLQYjUB{2vYIJ7Gm2*+@`nt)IT1~gk zElUef0+J)^7U1dbn(YkkXoaBz`G(Z^FBj3x@4Um;ZMn3A13D*HVTNVF4f-bUemsTp zMKWwVh}@x#>LO=O*z;9ZI6M9^OC)F9x`vB-q>GN{Zuqi$A~k09a*~i?3n(XSK00@S z2csSp{YZglFX)YSr@_JZ9|V@q0>xux+{^4EzKHh0tgnV)M!x2jof?gCyOc(Gy3pXF zl-YN@nMWE1cPn&(7`wChqD0cxcX`da+9j#>M!>$- zTZR(Ls_e;%$4ypAK#5Hw3pr;!7o_0D5GpqXu36&WUTR(6?lESaxJw?z_HnqtXl_0_ zAz{D}2tLtwlS;eExq=F4^c8a*$$V5@75E7BDt5xKJ5gG8@FaoGCg-w*{owkd=bkP` z3TV`o!^N89ca5L{ehc1sb_wf4d5iY!+D9!9V`&c@t4yKUvVvYsDvcxeS0x_A&>qMZ zGv0LI0NypSM5yb9ya?4EYNPC@ zbdH9mWHC0QO-CE`xPg_C`!U!X9`Bg&DW=LV<)9XT@=Y-}!D+zl#-Zc&?v)<=mMpRl zQrs6H_VwjR;a%3yp=ilrxW>k(^ng)lR*bL8M768X>R8$4gzfm#uF2O<37|pun)W|l zfVv{2ZLRUyc_Pxh6ex`5D^$H28?lRSrQmIS(JM!p4Q|N$C-g;5N?Na;k%QM{{45X4 zQ;sRy2E32A$y72j5@#YguL&PG zt`B#6=Ow|`5pLI06TSdf-ln*wHV=_Y}@imyql+ybKe7j$w! z^y^S-Y9>Q2io+x_u;G~=tciX=E`0_SCD-9)y_Dv~S;b(Tq6hHIARrA}!KkILD_lJL zL(tsT<~ebn9rWBJzX_l${ibhuzb<*a_302XrY-vQp3KE+`ecfG;-O@S!P&3vewtG? zlS%CBv`m+txjF}Te5A=@zR7rbyf-Z6bkGF=7ey9HRcS?$H#3|Qb#baUt>|XV<@?8ZKt(3@~D zc~|4^+&uZGZz1Ep&wi9zQ8K-BVpFZY=2aB7qPmbJna8sm>oTXHM%m8_-v3;vC610C|mHy3FQ@sC3JC(>rwS9;;bfljF1F4GDX{S@OiX~#V1`= ztpb2qPU!?03d1-*7I@r2VG~i}>nkR6pm?Is;$E-}B zVNVqX18NiSE4hgss0Dv|18zMoHderAPvw1xrdQ6%1nD=0(&-Smxmh+rKvDz_80+Gg z_IHJ$~w;|6cz-iQ!(n`$&GzEg6lS@DF0N;+q8~5JyLf#iBeDaScw={Kfm#$*d zXKc0RbONrdESi8^(OklT$we16SgAn>8-V>6o59ikc9@lXeReku!yKWoE_ZzEnhB6R z_xv2^n^vtLS#jPa2WDc#n@Xzg+@Dik8Ou}kHT1=~SB({W>A0?ejbNLm)(>@B0GSFz&7*A+L$WG8)X|min{kHhj-` zS>3UU%$=c~UPmiT<<3n&^ZbeyB%ynQGUj+CKZ9^>M4<;ZEkcoNUGG_!Ds_h%+}(Wi zL1MuBnomV*C(w&7OSF9;*8>oWkI{RpyMO^y&oL`+R&^uFC^IEC&|eU)oorL{r5_Sc zbYZwmL}#)bY{*oLK%FqI;?M^5aTb&mk5fJX_Wgd=T24aXFs4);>nVWcfhZz{_t*=1_j z%peNPsncCxgJ6QtaH4ZunOOpoDCl6!t>6{2ONDF7NnwWy!a0{;Y$W#i#l^}#k0w8f zS4g@6B(Gfb&AfblM2BY}i~jq_d`jm^yEzL{j)#LQYpfQ-mEk$zv^#C{jk&7!(NMfP z8*x~wyivVym*au>+!MIrg7MT4Gsvs9Q`#jBzDLw&kssvNz0S$Y?s=g*kQNNyc&W>& z^qHDUiVKigZ)g9kAnIE1RB)GJRc^~R7v7{PRv~6!rSAe1Sj~@hDfYOVv;{bRH@8-T zv*9B+o>D}1t&NrCqu_L@^~Q#DV#2^1FA^y}AJJN}Gd6Z$^)^YRHb@L_(roUg@RN7S4Lm{LV}|&+v#0Q>T*cJI*h) zl#x-y6}H-dki|%c?V|RL*|*A6Te08I+~fqpqM%n`gjeOId3q^Bhr3(@@h?s{L@#*cjB$CT?BXw zC7aht0Z#C*<<$l+4xLI=)pUoCT+qM7`%)Z0T=QrVT}Cx_Fa;1x2I7^LcgQA+alDmO zL}wvOjhW7GHw*8OfjwNmGaoQ;JnHIWU?k@0W@Vj8(eG7{bc}zxtptxLS9>fer{ioI zae;>wXbr!pPvJs3)2V*>6xyxW1=sz#G4*ushMx`fDu$os@Rn;ejUf94X{*-I2OiRf z6*ieQcKYeUO1`@vHAv6sCL;vK*{3p(x(04Dcng}a*mXq8nsK=FiK5_Iagpy%(t93K zb7o4t=fgJ?JWaSm4)*reow+a?ECpILVZc3!3kr4!4Yr{Q8_$`e+BPd#Y(w&@g^}eu z{3u{I+FhY|NX_J#3z8CHqobqaExD(UNF79y#nf@qhm;kzy)Ax!zjMB&v(T26BRX&! zuN=|?ioNAy?&15fGWm@~rTq>M3(l(V9Y{B_1w=D><3d1KpFNZd0Vtb?@ho{}%kWM% zVJ?$pDzJI+Zf4!}^oyc9gDtY%_?}{WrJwRir6tlSQYM~*8956cQ?EY_R!HD;0l2mW z?lZ;l3kDZ;WQ5Z5AwMLu%_WF>X2|INQ9LJ_BvWj{uFQFysvc}|Q(p~yU*)yX2&1DN z1tUWLU63p`YC;%C##+?iH^pcHfMX8|j_K2=47p^!lS#Gkwx*CjOQ6Y9E`1?F)6aCy^tK_Brr+t#0sPDHk#|t; zlMnl!mt^_MRtYXTQZ6r;1CD}^*p9-+FaP1b;5Ox4M(rvP*KTghY#UMlsH7hHp6reF zhBoOWitezQZrvh(bxsA%zspP;)6Bf`{G1$!4sPG0lE6E@sd(d!2Tksj+G|16$25Vp ztA$0XW2M%-ZG5RpfF{c(@350J{KET~_yNz0c!@A|(%zKzrpfR=dieT06(EavV z+t{==o>DT6+3Y}fxE7~ei zf@}~i@p@*y`QE?poHKLgy6!o3?my=&s!Z}^^4S30^vu8jz)JxDAUYlZxCGqV9{@mz zBml7I3;-w<007JZC7(@IZXbxBXo58XfR^;gE7P1?Ob~4LNDI(7!M=M7NL>vdJp|nR zClz1bSo&Fd_XFd=y~KTPUkS-NvswQQvjYyiN7-;eIW;ByM@X! zVV8ERiXm&`YS4E+b#?VVNyM78|6k+n-aUpDRUALJ42sQGGuX?OwZ=DQU%?$3Hu=4H zG#N58(AV+ zjE*<>6q}m?DusVAm9{N3`YCG`8Pmc@ z_$;)_MSgCIW3IuE=kL zaZ91#uZ^3zYTQ*M#s$yg-L;r=Ugi4MSEu66%;A*Wg(=)ggmh+WZ2T}wq}uMmwkNm$ zdHX{B{^(MS*Wf#UMkjO=Od{VR~7Y(G-}D)PVL%`xKj zrju3_8yfc80cx7GB%4xR`LjYFBQJOU7b?JpD_Nrg+!xt*;^6F1U-NXAsap714fmhqmt|J*`J^Cb) zO`;zM+WIj6MJp@ZGxu`mqLavv*_?G5_Ji6Ml;NUiMxS2adkPHMqWpK$VlZKi_e%d$3jeE@e;Gohv-b7{>~nTr;;CP*Z`CF( z|4z2}ZctV>GtLp-ADz<2daXTY9YDzd>aZSBg8S+@2B5^1b3?w-`yBkcM72h>>0A^r zNH1;bOAC&r5aOe=kv%1dWCJ^X1zpg}N5>)=^9k~6Xtvv<(-@EfWv09T+Ftd3K{-G9 zurV`&7+Kf^4xoe`%+wzoC14QmQ4$@TTfPsKpnYL$2Bl`}Qkm~>A~03;mC-7mw8cG* zlsiy829w4)j@x`DaQhD@>Ht3@o*AJmr4IkB$9$*Vd{n-9z9DQjiQ3z@`I_}e)v>h3=F z7+}0^*I~n;FK{y^8YtV|r+%gy?Ys6{i)ouxJ*!+OuOmWycxHYO=WbS#rz?R+Zxhd4 z#;`uYm#m=jSRTb5q`wGgu8C@24YwoqLbV)acIjrvl$ohYF}lJt*8U)sZs6>F=fQlA_2J1u~rtEZcEL?WbH znm>xqSJ6@#fuc95S+9BZysxWASkj>bLpCONM)9zm$7g&)SGv6$Qi1Rcp$b>RV5Zew zt&N4j^ef*nEKFGZ+1`hp^sQb1=97(DOP_aOmehOyqHj8^RYbov$h zIM#UMZ^B8>)Ja|978hHdOGl1MN4LD_=*MB)W@U8B2$0baVJfdxI-L^s7xGTXdX3~fx*zW@ zC`^b|qs%k>Fn{Kue5Ug>C&>{d#vfi?9u+o_Pcz8Tdwo}-Ta%PM`Y3PY}eFnzl?%6-}!t(EBq)fA`DW2mJD|LRLX z*d_4%O;%~L?Nt~a3D%!1pbnDfFaMr_CqB=+EHhXlnS-J^=r0t^MZT>6O!Bv`8UOV6 z1j`^*w%rZ+^x0__4YU2OTw^2wEV_PmvP{ia;NrTTA&M1!La`?2k8AQaA>bH*aYzow zeF&O}I~tl-|`Q>Yw;ft%LO zWiy`l8aEF`wG{ZuP`1JYHTq97(>K($yB#nP zh<(`KRbjW&aEUq!;~W|{vR&B0ekk?ixW6cG`}n(WnRxV(ALxm+(y=_E;%?$kUH9tf zOOtDfML9f*#gl~N{U!m%cC2kDf^2lgml0fhvSog9pVmJ#C?0-O`8AP_aCW0Rc111P(kbO~9Tl`fYx0?oF=lbUmxjlgjOSVKYI3`~5xtMmfR;h;_a z!#Pwnt^x<95{(;BU2PVFql`W5Z)+Fq!ZnLsGLag1wuG!8m>8#khqU@#5K@`bZH6X~ zFYb=t$YoY6FA(T|XX}iPpl$vKa=QozYucGwzT`&54%djeBtj)nb*a%@--EoLk6whX z9#e-VyNp+gW5f<%*=OLO-b#@QPHuH?#bEu(1Un|&%AQ$!*MRR_m(F_3AjR+-E2EsQ zxn$ukDVKp|>}HJu4KTR5wd8{RcewIK)KJz9Z~Yh1Lvn-6rJBJWSBSSw`s2lE)YU7J z_zS1|iLi;la{pyHH{4lMC5tG`#jf6}c6H!v7E_Rc>Y^vV`_C-N>5ankKs1ygvGs~eP$dj8I z76G=UZ9*4|{*yO-lMBd*5;t|UH1&N%N8B}g%@E((O5#>|X86O_5)c)6(u>6e;kF@* zlpBJt)r-Ht*Z9sH>~?;2+^kD-9#Q+z>8T|_DHF_OLJ|(J+F(uSse_5?*i{a!(g9#w@E?3iC=ZUbZGiN81XiS{Yo9+ugJ)! z{aQghnicB270NbC{d>Viqg!RFnUhFlcs*iRkixND;~`qeM9!_zJEG(J&MCwo!qt+T(IFCY~8g2;TI;icy%HVON# zt@g>ejjx?=ar9YM_zN{d5{9rtW5-rEIZ08U;ud98$|H`uPVx{jA8Q4^&NUedZHF^) zQ)Ol_L4%vEnjpGG(!FTzU4;6?kEr?aUsOj+b6!)~*--mBRv^K1n{B;l%a;p|demu-3&}(etY=BT}#jLnwBEg28@T7@xb0teFX1Sr% z<|RJ0!*C2Ymb^R7U_}6hTja3XsN+6HvPPN={9agMtzzEUI_W5~$qT+SCZkVVRra=U zYnC*Q7e(19NONPBKkvkq5r}BW?s878i7s=dmweh) zg&?ha+)0PC>8m|=UI=xe&RZf%^q3LDx}Gy!*f=S1vdyQnzaLr%dO1lyPr)&66-0EQ z;uU#tFSza{aUOa@L-&^s$x~R|NM4k*LJc7jnxcp_flkLx70eyK<}Z^}8VoeCT9c%C z3BW@*$IuN>j4MH*s3ft+WJcgMX;iAKLa1CA;z8~CiImWt=ncqx&GU4Swqtq&`4nl6 zwrHP!`cRK7AC}qe=nt46m-8vTI`>Bj>K-l*<9`9dg)+4>u12L$I@lKRXpnp9vz>qg zr#lRP_c|$Uf9RBx_!q;%RNwxwwEfWJ#mTghLqst}T43$T`uZt)Wtj8cm76zQ%&T>J zz)jfor9zbKn3OcM=>B4-P<9${#WhtpiB-#Qm>wSVqDz1B=WaQAeKjq~!(sfZC%7+| zAfNpAw?=6pJ<3@}+6`Fm8ahfDSJPKJ-hLnJM?NE5uK|k1Eo{rv9n`tZ^%Z`a! zm5QpUI6STXD?_FqM!1G?%c3k|Qq zvx>G^UU5MQ?_agu<}bi3#A^4}w5NX4wQ)vTsx`+-I)np3Gav5T?a00IGtv`{Zin1K z)+eO^h7yFzB@|&%$Kf%>5r)WlN7fwlMa9%%8v>-|x-oxUtdn~XP1q1-q9(&5$}7QO zG9pr2eb8fGCj6%GAlO{k38dIOH880ECG9hbqI58xQB#Z@x5_jirxWl%!sxm02d%-$iTxLOK{16`A zU1kAC-AHYYok^)beeTnL7x#=XY7)Ofj3wIfi|_+_Ly6Jj&Y5AyiMK*CGKX#-kIRU! zDq%co9rB8NTzTdvto-d##7ct4L`dT#jIXOTVHI0aR~RqN`Z?4Cb(Rn8l_UnHB`n=c75Y% zCTd(ajmP;R7BuF%w6Bj2ocWA4c1~vFCp)YWlLDM@3eRk}g)lWxiT Date: Thu, 27 May 2021 15:19:04 +0200 Subject: [PATCH 353/463] Update newsfragments/3677.documentation Co-authored-by: Jean-Paul Calderone --- newsfragments/3677.documentation | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3677.documentation b/newsfragments/3677.documentation index c499324db..51730e765 100644 --- a/newsfragments/3677.documentation +++ b/newsfragments/3677.documentation @@ -1 +1 @@ -Improves Tahoe-LAFS logo visibility on the README when GitHub dark/dimmed theme is being used. \ No newline at end of file +The visibility of the Tahoe-LAFS logo has been improved for "dark" themed viewing. From d78e9d891d37b752296d60a3bc850f483850e551 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 May 2021 11:40:59 -0400 Subject: [PATCH 354/463] Passing test. --- src/allmydata/scripts/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 41695aa47..922079c92 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -362,11 +362,12 @@ class BackupOptions(FileStoreOptions): line. The file is assumed to be in the argv encoding.""" abs_filepath = argv_to_abspath(filepath) try: - exclude_file = open(abs_filepath) + exclude_file = open(abs_filepath, "rb") except Exception as e: raise BackupConfigurationError('Error opening exclude file %s. (Error: %s)' % ( quote_local_unicode_path(abs_filepath), e)) try: + for line in exclude_file: self.opt_exclude(line) finally: From 49be6906c8628b862d6ca751c0174c69be248fd9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 May 2021 13:07:56 -0400 Subject: [PATCH 355/463] Fix test. --- src/allmydata/test/cli/test_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index e01dcc4cb..472105ca1 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -15,7 +15,7 @@ from six.moves import cStringIO as StringIO from allmydata import uri from allmydata.util import base32 -from allmydata.util.encodingutil import to_bytes +from allmydata.util.encodingutil import to_bytes, quote_output_u from allmydata.mutable.publish import MutableData from allmydata.immutable import upload from allmydata.scripts import debug @@ -168,7 +168,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.uris = {} self.fileurls = {} DATA = b"data" * 100 - quoted_good = u"'g\u00F6\u00F6d'" + quoted_good = quote_output_u("g\u00F6\u00F6d") d = c0.create_dirnode() def _stash_root_and_create_file(n): From 0425b64041651ba385eadbb07dcd2d8081e82473 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 May 2021 13:26:23 -0400 Subject: [PATCH 356/463] Match current logic. --- src/allmydata/test/test_encodingutil.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index f7987d466..062c64ba1 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -379,7 +379,10 @@ class QuoteOutput(ReallyEqualMixin, unittest.TestCase): check(u"\n", u"\"\\x0a\"", quote_newlines=True) def test_quote_output_default(self): - self.test_quote_output_utf8(None) + """Default is the encoding of sys.stdout if known, otherwise utf-8.""" + encoding = getattr(sys.stdout, "encoding") or "utf-8" + self.assertEqual(quote_output(u"\u2621"), + quote_output(u"\u2621", encoding=encoding)) def win32_other(win32, other): From 8aa339127638b413a943727d2985a5bd7245487c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 May 2021 13:26:46 -0400 Subject: [PATCH 357/463] Consistent behavior. --- src/allmydata/test/common_util.py | 3 ++- src/allmydata/util/encodingutil.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 9e7f0618c..21b8817a4 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -12,6 +12,7 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 import os +import sys import time import signal from random import randrange @@ -85,7 +86,7 @@ def run_cli_native(verb, *args, **kwargs): bytes. """ nodeargs = kwargs.pop("nodeargs", []) - encoding = kwargs.pop("encoding", None) or "utf-8" + encoding = kwargs.pop("encoding", None) or getattr(sys.stdout, "encoding") or "utf-8" return_bytes = kwargs.pop("return_bytes", False) verb = maybe_unicode_to_argv(verb) args = [maybe_unicode_to_argv(a) for a in args] diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 4cf690e65..576091539 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -256,7 +256,11 @@ def quote_output_u(*args, **kwargs): result = quote_output(*args, **kwargs) if isinstance(result, unicode): return result - return result.decode(kwargs.get("encoding", None) or io_encoding) + # Since we're quoting, the assumption is this will be read by a human, and + # therefore printed, so stdout's encoding is the plausible one. io_encoding + # is now always utf-8. + return result.decode(kwargs.get("encoding", None) or + getattr(sys.stdout, "encoding") or io_encoding) def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): @@ -287,7 +291,7 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): def _encode(s): if isinstance(s, bytes): try: - s = s.decode('utf-8') + s = s.decode("utf-8") except UnicodeDecodeError: return b'b"%s"' % (ESCAPABLE_8BIT.sub(lambda m: _bytes_escape(m, quote_newlines), s),) From eddd9550f085d0d75fdbef3fdb26c17284ba8154 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 May 2021 13:26:56 -0400 Subject: [PATCH 358/463] Pass test. --- src/allmydata/test/cli/test_create_alias.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_create_alias.py b/src/allmydata/test/cli/test_create_alias.py index 6f6881860..99590a631 100644 --- a/src/allmydata/test/cli/test_create_alias.py +++ b/src/allmydata/test/cli/test_create_alias.py @@ -20,7 +20,7 @@ from allmydata.util import fileutil from allmydata.scripts.common import get_aliases from allmydata.scripts import cli, runner from ..no_network import GridTestMixin -from allmydata.util.encodingutil import quote_output +from allmydata.util.encodingutil import quote_output_u from .common import CLITestMixin class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): @@ -182,7 +182,7 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) self.assertEqual(len(err), 0, err) - self.failUnlessIn(u"Alias %s created" % ensure_text(quote_output(etudes_arg)), out) + self.failUnlessIn(u"Alias %s created" % (quote_output_u(etudes_arg),), out) aliases = get_aliases(self.get_clientdir()) self.failUnless(aliases[u"\u00E9tudes"].startswith(b"URI:DIR2:")) From 8be4acd7ec27861d015cfd33e64723180a38937b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 May 2021 13:54:27 -0400 Subject: [PATCH 359/463] Faking sys.stdout doesn't go well since we lookup encoding attribute. --- src/allmydata/test/cli/test_status.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/cli/test_status.py b/src/allmydata/test/cli/test_status.py index 4488299b2..58818ed41 100644 --- a/src/allmydata/test/cli/test_status.py +++ b/src/allmydata/test/cli/test_status.py @@ -130,9 +130,10 @@ class Integration(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(_check) return d - @mock.patch('sys.stdout') - def test_help(self, fake): - return self.do_cli('status', '--help') + @defer.inlineCallbacks + def test_help(self): + rc, _, _ = yield self.do_cli('status', '--help') + self.assertEqual(rc, 0) class CommandStatus(unittest.TestCase): From 69c8305ae8837a4746e681385ffd12671c10a023 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 May 2021 13:55:29 -0400 Subject: [PATCH 360/463] Flake fix. --- src/allmydata/test/cli/test_create_alias.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/cli/test_create_alias.py b/src/allmydata/test/cli/test_create_alias.py index 99590a631..176bf7576 100644 --- a/src/allmydata/test/cli/test_create_alias.py +++ b/src/allmydata/test/cli/test_create_alias.py @@ -10,7 +10,6 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from six import ensure_text from six.moves import StringIO import os.path from twisted.trial import unittest From 64f6ccd17ffb53d9a71d562fdbe93b69fed39e83 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 May 2021 09:53:28 -0400 Subject: [PATCH 361/463] Make --exclude-from behavior consistent, at the cost of a minor incompatibility. --- docs/frontends/CLI.rst | 6 +++--- newsfragments/3716.incompat | 1 + newsfragments/3716.minor | 0 src/allmydata/scripts/cli.py | 5 ++--- src/allmydata/test/cli/test_backup.py | 20 ++++++++------------ 5 files changed, 14 insertions(+), 18 deletions(-) create mode 100644 newsfragments/3716.incompat delete mode 100644 newsfragments/3716.minor diff --git a/docs/frontends/CLI.rst b/docs/frontends/CLI.rst index 0badede98..d501b7af5 100644 --- a/docs/frontends/CLI.rst +++ b/docs/frontends/CLI.rst @@ -514,10 +514,10 @@ Command Examples the pattern will be matched against any level of the directory tree; it's still impossible to specify absolute path exclusions. -``tahoe backup --exclude-from=/path/to/filename ~ work:backups`` +``tahoe backup --exclude-from-utf-8=/path/to/filename ~ work:backups`` - ``--exclude-from`` is similar to ``--exclude``, but reads exclusion - patterns from ``/path/to/filename``, one per line. + ``--exclude-from-utf-8`` is similar to ``--exclude``, but reads exclusion + patterns from a UTF-8-encoded ``/path/to/filename``, one per line. ``tahoe backup --exclude-vcs ~ work:backups`` diff --git a/newsfragments/3716.incompat b/newsfragments/3716.incompat new file mode 100644 index 000000000..aa03eea47 --- /dev/null +++ b/newsfragments/3716.incompat @@ -0,0 +1 @@ +tahoe backup's --exclude-from has been renamed to --exclude-from-utf-8, and correspondingly requires the file to be UTF-8 encoded. \ No newline at end of file diff --git a/newsfragments/3716.minor b/newsfragments/3716.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 922079c92..c58b5da41 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -357,17 +357,16 @@ class BackupOptions(FileStoreOptions): exclude = self['exclude'] exclude.add(g) - def opt_exclude_from(self, filepath): + def opt_exclude_from_utf_8(self, filepath): """Ignore file matching glob patterns listed in file, one per line. The file is assumed to be in the argv encoding.""" abs_filepath = argv_to_abspath(filepath) try: - exclude_file = open(abs_filepath, "rb") + exclude_file = open(abs_filepath, "r", encoding="utf-8") except Exception as e: raise BackupConfigurationError('Error opening exclude file %s. (Error: %s)' % ( quote_local_unicode_path(abs_filepath), e)) try: - for line in exclude_file: self.opt_exclude(line) finally: diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index 6ac55a9e8..dc5c3ff17 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -355,14 +355,14 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): exclusion_string = "_darcs\n*py\n.svn" excl_filepath = os.path.join(basedir, 'exclusion') fileutil.write(excl_filepath, exclusion_string) - backup_options = parse(['--exclude-from', excl_filepath, 'from', 'to']) + backup_options = parse(['--exclude-from-utf-8', excl_filepath, 'from', 'to']) filtered = list(backup_options.filter_listdir(subdir_listdir)) self._check_filtering(filtered, subdir_listdir, (u'another_doc.lyx', u'CVS'), (u'.svn', u'_darcs', u'run_snake_run.py')) # test BackupConfigurationError self.failUnlessRaises(cli.BackupConfigurationError, parse, - ['--exclude-from', excl_filepath + '.no', 'from', 'to']) + ['--exclude-from-utf-8', excl_filepath + '.no', 'from', 'to']) # test that an iterator works too backup_options = parse(['--exclude', '*lyx', 'from', 'to']) @@ -373,7 +373,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): def test_exclude_options_unicode(self): nice_doc = u"nice_d\u00F8c.lyx" try: - doc_pattern_arg = u"*d\u00F8c*" + doc_pattern_arg_unicode = doc_pattern_arg = u"*d\u00F8c*" if PY2: doc_pattern_arg = doc_pattern_arg.encode(get_io_encoding()) except UnicodeEncodeError: @@ -397,14 +397,10 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): self._check_filtering(filtered, root_listdir, (u'_darcs', u'subdir'), (nice_doc, u'lib.a')) # read exclude patterns from file - exclusion_string = doc_pattern_arg + ensure_str("\nlib.?") - if PY3: - # On Python 2 this gives some garbage encoding. Also on Python 2 we - # expect exclusion string to be bytes. - exclusion_string = exclusion_string.encode(locale.getpreferredencoding(False)) + exclusion_string = (doc_pattern_arg_unicode + "\nlib.?").encode("utf-8") excl_filepath = os.path.join(basedir, 'exclusion') fileutil.write(excl_filepath, exclusion_string) - backup_options = parse(['--exclude-from', excl_filepath, 'from', 'to']) + backup_options = parse(['--exclude-from-utf-8', excl_filepath, 'from', 'to']) filtered = list(backup_options.filter_listdir(root_listdir)) self._check_filtering(filtered, root_listdir, (u'_darcs', u'subdir'), (nice_doc, u'lib.a')) @@ -427,20 +423,20 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): ns = Namespace() ns.called = False original_open = open - def call_file(name, *args): + def call_file(name, *args, **kwargs): if name.endswith("excludes.dummy"): ns.called = True self.failUnlessEqual(name, abspath_expanduser_unicode(exclude_file)) return StringIO() else: - return original_open(name, *args) + return original_open(name, *args, **kwargs) if PY2: from allmydata.scripts import cli as module_to_patch else: import builtins as module_to_patch patcher = MonkeyPatcher((module_to_patch, 'open', call_file)) - patcher.runWithPatches(parse_options, basedir, "backup", ['--exclude-from', unicode_to_argv(exclude_file), 'from', 'to']) + patcher.runWithPatches(parse_options, basedir, "backup", ['--exclude-from-utf-8', unicode_to_argv(exclude_file), 'from', 'to']) self.failUnless(ns.called) def test_ignore_symlinks(self): From b509ff69fcb77088990ff8126da6e9a8c8485374 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 May 2021 11:25:16 -0400 Subject: [PATCH 362/463] Flake fix. --- src/allmydata/test/cli/test_backup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index dc5c3ff17..df598b811 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -6,16 +6,14 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future.utils import PY2, PY3 +from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from six import ensure_str import os.path from six.moves import cStringIO as StringIO from datetime import timedelta import re -import locale from twisted.trial import unittest from twisted.python.monkey import MonkeyPatcher From bf1df54fdd462d18732b88fe36e19193b9bbda84 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 31 May 2021 08:37:02 -0400 Subject: [PATCH 363/463] Add newsfragment --- newsfragments/3726.documentation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3726.documentation diff --git a/newsfragments/3726.documentation b/newsfragments/3726.documentation new file mode 100644 index 000000000..3ff6e67ad --- /dev/null +++ b/newsfragments/3726.documentation @@ -0,0 +1 @@ +Register Tahoe-LAFS project with Libera.Chat IRC network. From 0397266c17e50c420ee505a8bdc8a33163c9f33f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 1 Jun 2021 10:51:12 -0400 Subject: [PATCH 364/463] Update newsfragment --- newsfragments/3726.documentation | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3726.documentation b/newsfragments/3726.documentation index 3ff6e67ad..fb94fff32 100644 --- a/newsfragments/3726.documentation +++ b/newsfragments/3726.documentation @@ -1 +1 @@ -Register Tahoe-LAFS project with Libera.Chat IRC network. +Tahoe-LAFS project is now registered with Libera.Chat IRC network. From b3b85b2e412639f17213b7fab09bc84b929efdba Mon Sep 17 00:00:00 2001 From: Anxhelo Lushka Date: Tue, 1 Jun 2021 19:32:10 +0200 Subject: [PATCH 365/463] Added a link to Fosshost in the README --- README.rst | 4 ++++ newsfragments/3727.minor | 0 2 files changed, 4 insertions(+) create mode 100644 newsfragments/3727.minor diff --git a/README.rst b/README.rst index 1c7a8d6ee..cb22ea4f9 100644 --- a/README.rst +++ b/README.rst @@ -93,6 +93,10 @@ As a community-driven open source project, Tahoe-LAFS welcomes contributions of Before authoring or reviewing a patch, please familiarize yourself with the `Coding Standard `__ and the `Contributor Code of Conduct `__. +🤝 Supporters +-------------- + +We would like to thank `Fosshost `__ for supporting us with hosting services. If your project open source project needs help, you can apply for support. ❓ FAQ ------ diff --git a/newsfragments/3727.minor b/newsfragments/3727.minor new file mode 100644 index 000000000..e69de29bb From e594022c60d0f4a190449d42f28cdaf4da29e238 Mon Sep 17 00:00:00 2001 From: Anxhelo Lushka Date: Tue, 1 Jun 2021 19:50:21 +0200 Subject: [PATCH 366/463] Small typo fix --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index cb22ea4f9..2cc6e38eb 100644 --- a/README.rst +++ b/README.rst @@ -96,7 +96,7 @@ Before authoring or reviewing a patch, please familiarize yourself with the `Cod 🤝 Supporters -------------- -We would like to thank `Fosshost `__ for supporting us with hosting services. If your project open source project needs help, you can apply for support. +We would like to thank `Fosshost `__ for supporting us with hosting services. If your open source project needs help, you can apply for their support. ❓ FAQ ------ From a0744ffa8c834e65153f0e692cd9b47efc4a7450 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Jun 2021 09:54:57 -0400 Subject: [PATCH 367/463] Don't shadow builtin. --- src/allmydata/scripts/slow_operation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py index 620edac34..3c23fb533 100644 --- a/src/allmydata/scripts/slow_operation.py +++ b/src/allmydata/scripts/slow_operation.py @@ -66,10 +66,10 @@ class SlowOperationRunner(object): def wait_for_results(self): last = 0 - for next in self.poll_times(): - delay = next - last + for next_item in self.poll_times(): + delay = next_item - last time.sleep(delay) - last = next + last = next_item if self.poll(): return 0 From 464b7055c2e4359206961fec6d54a5a3294f9efd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Jun 2021 10:12:05 -0400 Subject: [PATCH 368/463] Port to Python 3. --- src/allmydata/scripts/tahoe_cp.py | 42 ++++++++++++++++++------------- src/allmydata/util/_python3.py | 1 + 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/src/allmydata/scripts/tahoe_cp.py b/src/allmydata/scripts/tahoe_cp.py index df4965dc7..aae03291f 100644 --- a/src/allmydata/scripts/tahoe_cp.py +++ b/src/allmydata/scripts/tahoe_cp.py @@ -1,6 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from past.builtins import unicode +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os.path from urllib.parse import quote as url_quote @@ -162,7 +170,7 @@ class LocalDirectoryTarget(object): self.children[n] = LocalFileTarget(pn) def get_child_target(self, name): - precondition(isinstance(name, unicode), name) + precondition(isinstance(name, str), name) precondition(len(name), name) # don't want "" if self.children is None: self.populate(recurse=False) @@ -175,7 +183,7 @@ class LocalDirectoryTarget(object): return child def put_file(self, name, inf): - precondition(isinstance(name, unicode), name) + precondition(isinstance(name, str), name) pathname = os.path.join(self.pathname, name) fileutil.put_file(pathname, inf) @@ -258,7 +266,7 @@ class TahoeDirectorySource(object): nodetype, d = parsed assert nodetype == "dirnode" self.mutable = d.get("mutable", False) # older nodes don't provide it - self.children_d = dict( [(unicode(name),value) + self.children_d = dict( [(str(name),value) for (name,value) in d["children"].items()] ) self.children = None @@ -268,7 +276,7 @@ class TahoeDirectorySource(object): self.writecap = to_bytes(d.get("rw_uri")) self.readcap = to_bytes(d.get("ro_uri")) self.mutable = d.get("mutable", False) # older nodes don't provide it - self.children_d = dict( [(unicode(name),value) + self.children_d = dict( [(str(name),value) for (name,value) in d["children"].items()] ) self.children = None @@ -338,7 +346,7 @@ class TahoeDirectoryTarget(object): self.writecap = to_bytes(d.get("rw_uri")) self.readcap = to_bytes(d.get("ro_uri")) self.mutable = d.get("mutable", False) # older nodes don't provide it - self.children_d = dict( [(unicode(name),value) + self.children_d = dict( [(str(name),value) for (name,value) in d["children"].items()] ) self.children = None @@ -355,7 +363,7 @@ class TahoeDirectoryTarget(object): nodetype, d = parsed assert nodetype == "dirnode" self.mutable = d.get("mutable", False) # older nodes don't provide it - self.children_d = dict( [(unicode(name),value) + self.children_d = dict( [(str(name),value) for (name,value) in d["children"].items()] ) self.children = None @@ -411,7 +419,7 @@ class TahoeDirectoryTarget(object): def get_child_target(self, name): # return a new target for a named subdirectory of this dir - precondition(isinstance(name, unicode), name) + precondition(isinstance(name, str), name) if self.children is None: self.populate(recurse=False) if name in self.children: @@ -424,7 +432,7 @@ class TahoeDirectoryTarget(object): return child def put_file(self, name, inf): - precondition(isinstance(name, unicode), name) + precondition(isinstance(name, str), name) url = self.nodeurl + "uri" if not seekable(inf): inf = inf.read() @@ -444,7 +452,7 @@ class TahoeDirectoryTarget(object): self.new_children[name] = filecap def put_uri(self, name, filecap): - precondition(isinstance(name, unicode), name) + precondition(isinstance(name, str), name) self.new_children[name] = filecap def set_children(self): @@ -453,7 +461,7 @@ class TahoeDirectoryTarget(object): url = (self.nodeurl + "uri/" + url_quote(self.writecap) + "?t=set_children") set_data = {} - for (name, filecap) in self.new_children.items(): + for (name, filecap) in list(self.new_children.items()): # it just so happens that ?t=set_children will accept both file # read-caps and write-caps as ['rw_uri'], and will handle either # correctly. So don't bother trying to figure out whether the one @@ -597,7 +605,7 @@ class Copier(object): # and get_source_info. def get_target_info(self, destination_spec): - precondition(isinstance(destination_spec, unicode), destination_spec) + precondition(isinstance(destination_spec, str), destination_spec) rootcap, path_utf8 = get_alias(self.aliases, destination_spec, None) path = path_utf8.decode("utf-8") if rootcap == DefaultAliasMarker: @@ -644,7 +652,7 @@ class Copier(object): """ This turns an argv string into a (Local|Tahoe)(File|Directory)Source. """ - precondition(isinstance(source_spec, unicode), source_spec) + precondition(isinstance(source_spec, str), source_spec) rootcap, path_utf8 = get_alias(self.aliases, source_spec, None) path = path_utf8.decode("utf-8") # any trailing slash is removed in abspath_expanduser_unicode(), so @@ -752,7 +760,7 @@ class Copier(object): # target name collisions are an error collisions = [] - for target, sources in targetmap.items(): + for target, sources in list(targetmap.items()): target_names = {} for source in sources: name = source.basename() @@ -821,7 +829,7 @@ class Copier(object): def assign_targets(self, targetmap, source, target): # copy everything in the source into the target precondition(isinstance(source, DirectorySources), source) - for name, child in source.children.items(): + for name, child in list(source.children.items()): if isinstance(child, DirectorySources): # we will need a target directory for this one subtarget = target.get_child_target(name) @@ -837,7 +845,7 @@ class Copier(object): files_copied = 0 targets_finished = 0 - for target, sources in targetmap.items(): + for target, sources in list(targetmap.items()): _assert(isinstance(target, DirectoryTargets), target) for source in sources: _assert(isinstance(source, FileSources), source) @@ -857,7 +865,7 @@ class Copier(object): def copy_file_into_dir(self, source, name, target): precondition(isinstance(source, FileSources), source) precondition(isinstance(target, DirectoryTargets), target) - precondition(isinstance(name, unicode), name) + precondition(isinstance(name, str), name) if self.need_to_copy_bytes(source, target): # if the target is a local directory, this will just write the # bytes to disk. If it is a tahoe directory, it will upload the diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 50e429979..48c73f60f 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -105,6 +105,7 @@ PORTED_MODULES = [ "allmydata.scripts.tahoe_add_alias", "allmydata.scripts.tahoe_backup", "allmydata.scripts.tahoe_check", + "allmydata.scripts.tahoe_cp", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From e9cf5d5ef0b4cbbbb16dc3fce5466eebff832b4c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Jun 2021 10:12:22 -0400 Subject: [PATCH 369/463] News file. --- newsfragments/3728.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3728.minor diff --git a/newsfragments/3728.minor b/newsfragments/3728.minor new file mode 100644 index 000000000..e69de29bb From 9d5f1ad73544a5bfcf693db78f72d11731af4b34 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Jun 2021 10:14:59 -0400 Subject: [PATCH 370/463] Port to Python 3. --- src/allmydata/scripts/tahoe_get.py | 10 +++++++++- src/allmydata/util/_python3.py | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/tahoe_get.py b/src/allmydata/scripts/tahoe_get.py index 150b0457b..39f1686ce 100644 --- a/src/allmydata/scripts/tahoe_get.py +++ b/src/allmydata/scripts/tahoe_get.py @@ -1,6 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from future.utils import PY3 +from future.utils import PY2, PY3 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from urllib.parse import quote as url_quote from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 48c73f60f..17bcf0107 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -106,6 +106,7 @@ PORTED_MODULES = [ "allmydata.scripts.tahoe_backup", "allmydata.scripts.tahoe_check", "allmydata.scripts.tahoe_cp", + "allmydata.scripts.tahoe_get", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From 61fdea9043c7e5fba6cc80c62a9ea957c9f305d7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Jun 2021 10:16:17 -0400 Subject: [PATCH 371/463] Port to Python 3. --- src/allmydata/scripts/tahoe_invite.py | 10 ++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 11 insertions(+) diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py index 5be0fa871..09d4cbd59 100644 --- a/src/allmydata/scripts/tahoe_invite.py +++ b/src/allmydata/scripts/tahoe_invite.py @@ -1,5 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + try: from allmydata.scripts.types_ import SubCommands except ImportError: diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 17bcf0107..42a050b0a 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -107,6 +107,7 @@ PORTED_MODULES = [ "allmydata.scripts.tahoe_check", "allmydata.scripts.tahoe_cp", "allmydata.scripts.tahoe_get", + "allmydata.scripts.tahoe_invite", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From 9804a44c502ce2e364c480416720fbb065972973 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Jun 2021 11:19:45 -0400 Subject: [PATCH 372/463] Port to Python 3. --- src/allmydata/scripts/tahoe_ls.py | 45 +++++++++++++++++++------------ src/allmydata/util/_python3.py | 1 + 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/allmydata/scripts/tahoe_ls.py b/src/allmydata/scripts/tahoe_ls.py index b3e09b699..1ddfa555a 100644 --- a/src/allmydata/scripts/tahoe_ls.py +++ b/src/allmydata/scripts/tahoe_ls.py @@ -1,6 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from past.builtins import unicode +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from six import ensure_text, ensure_str import time @@ -28,7 +37,7 @@ def list(options): e.display(stderr) return 1 - path = unicode(path, "utf-8") + path = str(path, "utf-8") url = nodeurl + "uri/%s" % url_quote(rootcap) if path: # move where.endswith check here? @@ -50,7 +59,7 @@ def list(options): if options['json']: # The webapi server should always output printable ASCII. if is_printable_ascii(data): - data = unicode(data, "ascii") + data = str(data, "ascii") print(data, file=stdout) return 0 else: @@ -87,7 +96,7 @@ def list(options): for name in childnames: child = children[name] - name = unicode(name) + name = str(name) childtype = child[0] # See webapi.txt for a discussion of the meanings of unix local @@ -147,23 +156,18 @@ def list(options): if not options["classify"]: classify = "" - encoding_error = False - try: - line.append(unicode_to_output(name) + classify) - except UnicodeEncodeError: - encoding_error = True - line.append(quote_output(name) + classify) + line.append(name + classify) if options["uri"]: - line.append(ensure_str(uri)) + line.append(ensure_text(uri)) if options["readonly-uri"]: - line.append(quote_output(ensure_str(ro_uri) or "-", quotemarks=False)) + line.append(quote_output(ensure_text(ro_uri) or "-", quotemarks=False)) - rows.append((encoding_error, line)) + rows.append(line) max_widths = [] left_justifys = [] - for (encoding_error, row) in rows: + for row in rows: for i,cell in enumerate(row): while len(max_widths) <= i: max_widths.append(0) @@ -185,12 +189,19 @@ def list(options): fmt = " ".join(fmt_pieces) rc = 0 - for (encoding_error, row) in rows: + for row in rows: + row = (fmt % tuple(row)).rstrip() + encoding_error = False + try: + row = unicode_to_output(row) + except UnicodeEncodeError: + encoding_error = True + row = quote_output(row) if encoding_error: - print((fmt % tuple(row)).rstrip(), file=stderr) + print(row, file=stderr) rc = 1 else: - print((fmt % tuple(row)).rstrip(), file=stdout) + print(row, file=stdout) if rc == 1: print("\nThis listing included files whose names could not be converted to the terminal" \ diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 42a050b0a..936879271 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -108,6 +108,7 @@ PORTED_MODULES = [ "allmydata.scripts.tahoe_cp", "allmydata.scripts.tahoe_get", "allmydata.scripts.tahoe_invite", + "allmydata.scripts.tahoe_ls", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From defe16f91233eaac08d82bf899ff9b1754c550ad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Jun 2021 11:30:19 -0400 Subject: [PATCH 373/463] Port to Python 3. --- src/allmydata/scripts/tahoe_manifest.py | 21 ++++++++++++++++----- src/allmydata/util/_python3.py | 1 + 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/allmydata/scripts/tahoe_manifest.py b/src/allmydata/scripts/tahoe_manifest.py index 966583244..b55075eef 100644 --- a/src/allmydata/scripts/tahoe_manifest.py +++ b/src/allmydata/scripts/tahoe_manifest.py @@ -1,7 +1,16 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from future.utils import PY3 -from past.builtins import unicode +from future.utils import PY2, PY3 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from six import ensure_str from urllib.parse import quote as url_quote import json @@ -37,7 +46,7 @@ class ManifestStreamer(LineOnlyReceiver, object): except UnknownAliasError as e: e.display(stderr) return 1 - path = unicode(path, "utf-8") + path = str(path, "utf-8") if path == '/': path = '' url = nodeurl + "uri/%s" % url_quote(rootcap) @@ -96,8 +105,10 @@ class ManifestStreamer(LineOnlyReceiver, object): if vc: print(quote_output(vc, quotemarks=False), file=stdout) else: - print("%s %s" % (quote_output(d["cap"], quotemarks=False), - quote_path(d["path"], quotemarks=False)), file=stdout) + # ensure_str() only necessary for Python 2. + print(ensure_str("%s %s") % ( + quote_output(d["cap"], quotemarks=False), + quote_path(d["path"], quotemarks=False)), file=stdout) def manifest(options): return ManifestStreamer().run(options) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 936879271..aa18b69ba 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -109,6 +109,7 @@ PORTED_MODULES = [ "allmydata.scripts.tahoe_get", "allmydata.scripts.tahoe_invite", "allmydata.scripts.tahoe_ls", + "allmydata.scripts.tahoe_manifest", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From a910ebcc8a75120dbfd682c3822080ebf85a550e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Jun 2021 11:31:32 -0400 Subject: [PATCH 374/463] Fix flakes. --- src/allmydata/scripts/cli.py | 2 +- src/allmydata/scripts/tahoe_ls.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index c58b5da41..55975b8c5 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -515,7 +515,7 @@ def list_aliases(options): def list_(options): from allmydata.scripts import tahoe_ls - rc = tahoe_ls.list(options) + rc = tahoe_ls.ls(options) return rc def get(options): diff --git a/src/allmydata/scripts/tahoe_ls.py b/src/allmydata/scripts/tahoe_ls.py index 1ddfa555a..5a7136d77 100644 --- a/src/allmydata/scripts/tahoe_ls.py +++ b/src/allmydata/scripts/tahoe_ls.py @@ -10,7 +10,7 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from six import ensure_text, ensure_str +from six import ensure_text import time from urllib.parse import quote as url_quote @@ -20,7 +20,7 @@ from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ from allmydata.scripts.common_http import do_http, format_http_error from allmydata.util.encodingutil import unicode_to_output, quote_output, is_printable_ascii, to_bytes -def list(options): +def ls(options): nodeurl = options['node-url'] aliases = options.aliases where = options.where From aef8705b6f9edaed1dd4f4cdb893566f78431b4e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Jun 2021 09:26:18 -0400 Subject: [PATCH 375/463] News file. --- newsfragments/3729.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3729.minor diff --git a/newsfragments/3729.minor b/newsfragments/3729.minor new file mode 100644 index 000000000..e69de29bb From 53482dd8acaab2d1c5e0970537c502770c93d1f0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Jun 2021 09:26:24 -0400 Subject: [PATCH 376/463] Port to Python 3. --- src/allmydata/scripts/tahoe_mkdir.py | 12 ++++++++++-- src/allmydata/util/_python3.py | 1 + 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/tahoe_mkdir.py b/src/allmydata/scripts/tahoe_mkdir.py index 54e8ebe46..85fe12554 100644 --- a/src/allmydata/scripts/tahoe_mkdir.py +++ b/src/allmydata/scripts/tahoe_mkdir.py @@ -1,6 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from past.builtins import unicode +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from urllib.parse import quote as url_quote from allmydata.scripts.common_http import do_http, check_http_error @@ -37,7 +45,7 @@ def mkdir(options): return 0 # create a new directory at the given location - path = unicode(path, "utf-8") + path = str(path, "utf-8") if path.endswith("/"): path = path[:-1] # path must be "/".join([s.encode("utf-8") for s in segments]) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index aa18b69ba..6979efdd3 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -110,6 +110,7 @@ PORTED_MODULES = [ "allmydata.scripts.tahoe_invite", "allmydata.scripts.tahoe_ls", "allmydata.scripts.tahoe_manifest", + "allmydata.scripts.tahoe_mkdir", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From 58d6f9f6cfb9b70f10ed161059eb62c13304ac20 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Jun 2021 09:28:10 -0400 Subject: [PATCH 377/463] Port to Python 3. --- src/allmydata/scripts/tahoe_mv.py | 14 +++++++++++--- src/allmydata/util/_python3.py | 1 + 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/allmydata/scripts/tahoe_mv.py b/src/allmydata/scripts/tahoe_mv.py index 84f83edcd..d921047a8 100644 --- a/src/allmydata/scripts/tahoe_mv.py +++ b/src/allmydata/scripts/tahoe_mv.py @@ -1,6 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from past.builtins import unicode +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re from urllib.parse import quote as url_quote @@ -27,7 +35,7 @@ def mv(options, mode="move"): except UnknownAliasError as e: e.display(stderr) return 1 - from_path = unicode(from_path, "utf-8") + from_path = str(from_path, "utf-8") from_url = nodeurl + "uri/%s" % url_quote(rootcap) if from_path: from_url += "/" + escape_path(from_path) @@ -47,7 +55,7 @@ def mv(options, mode="move"): e.display(stderr) return 1 to_url = nodeurl + "uri/%s" % url_quote(rootcap) - path = unicode(path, "utf-8") + path = str(path, "utf-8") if path: to_url += "/" + escape_path(path) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 6979efdd3..829cae829 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -111,6 +111,7 @@ PORTED_MODULES = [ "allmydata.scripts.tahoe_ls", "allmydata.scripts.tahoe_manifest", "allmydata.scripts.tahoe_mkdir", + "allmydata.scripts.tahoe_mv", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From cf68f55039cc4407dcf24a0a49d6c8b81d8792e4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Jun 2021 09:34:42 -0400 Subject: [PATCH 378/463] Port to Python 3. --- src/allmydata/scripts/tahoe_put.py | 11 +++++++++-- src/allmydata/util/_python3.py | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/tahoe_put.py b/src/allmydata/scripts/tahoe_put.py index 8db705d01..1ea45e8ea 100644 --- a/src/allmydata/scripts/tahoe_put.py +++ b/src/allmydata/scripts/tahoe_put.py @@ -1,7 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function from future.utils import PY2 -from past.builtins import unicode +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from io import BytesIO from urllib.parse import quote as url_quote @@ -56,7 +63,7 @@ def put(options): except UnknownAliasError as e: e.display(stderr) return 1 - path = unicode(path, "utf-8") + path = str(path, "utf-8") if path.startswith("/"): suggestion = to_file.replace(u"/", u"", 1) print("Error: The remote filename must not start with a slash", file=stderr) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 829cae829..030b7efd5 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -112,6 +112,7 @@ PORTED_MODULES = [ "allmydata.scripts.tahoe_manifest", "allmydata.scripts.tahoe_mkdir", "allmydata.scripts.tahoe_mv", + "allmydata.scripts.tahoe_put", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From 77676c16487094f52ea38b58699aebcbae8a1f23 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Jun 2021 09:36:22 -0400 Subject: [PATCH 379/463] Port to Python 3. --- src/allmydata/scripts/tahoe_run.py | 10 ++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 11 insertions(+) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index bc4ba27d1..218d8d458 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -1,5 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + __all__ = [ "RunOptions", "run", diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 030b7efd5..b37b548ae 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -113,6 +113,7 @@ PORTED_MODULES = [ "allmydata.scripts.tahoe_mkdir", "allmydata.scripts.tahoe_mv", "allmydata.scripts.tahoe_put", + "allmydata.scripts.tahoe_run", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From cb1e591c6e937f3ff9c85b120ce05ec225e9c507 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Jun 2021 09:37:59 -0400 Subject: [PATCH 380/463] Port to Python 3. --- src/allmydata/scripts/tahoe_status.py | 10 +++++++++- src/allmydata/util/_python3.py | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/tahoe_status.py b/src/allmydata/scripts/tahoe_status.py index ff746901b..a723274f0 100644 --- a/src/allmydata/scripts/tahoe_status.py +++ b/src/allmydata/scripts/tahoe_status.py @@ -1,6 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from future.builtins import chr +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os from urllib.parse import urlencode, quote as url_quote diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index b37b548ae..808d7a414 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -114,6 +114,7 @@ PORTED_MODULES = [ "allmydata.scripts.tahoe_mv", "allmydata.scripts.tahoe_put", "allmydata.scripts.tahoe_run", + "allmydata.scripts.tahoe_status", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From 279d6b6542205010e30070ec0da24f5fe9fce793 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Jun 2021 09:39:13 -0400 Subject: [PATCH 381/463] Port to Python 3. --- src/allmydata/scripts/tahoe_unlink.py | 10 ++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 11 insertions(+) diff --git a/src/allmydata/scripts/tahoe_unlink.py b/src/allmydata/scripts/tahoe_unlink.py index 1ec92c69e..5bdebb960 100644 --- a/src/allmydata/scripts/tahoe_unlink.py +++ b/src/allmydata/scripts/tahoe_unlink.py @@ -1,5 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from urllib.parse import quote as url_quote from allmydata.scripts.common_http import do_http, format_http_success, format_http_error from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 808d7a414..bb586f286 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -115,6 +115,7 @@ PORTED_MODULES = [ "allmydata.scripts.tahoe_put", "allmydata.scripts.tahoe_run", "allmydata.scripts.tahoe_status", + "allmydata.scripts.tahoe_unlink", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From 293cea6fd25e1f811e84a64230bf2e376ce60ef2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Jun 2021 09:40:11 -0400 Subject: [PATCH 382/463] Port to Python 3. --- src/allmydata/scripts/tahoe_webopen.py | 14 ++++++++++++-- src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/tahoe_webopen.py b/src/allmydata/scripts/tahoe_webopen.py index 0292e0d40..dbec31e87 100644 --- a/src/allmydata/scripts/tahoe_webopen.py +++ b/src/allmydata/scripts/tahoe_webopen.py @@ -1,4 +1,14 @@ -from past.builtins import unicode +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from urllib.parse import quote as url_quote @@ -18,7 +28,7 @@ def webopen(options, opener=None): except UnknownAliasError as e: e.display(stderr) return 1 - path = unicode(path, "utf-8") + path = str(path, "utf-8") if path == '/': path = '' url = nodeurl + "uri/%s" % url_quote(rootcap) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index bb586f286..5e06c2c01 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -116,6 +116,7 @@ PORTED_MODULES = [ "allmydata.scripts.tahoe_run", "allmydata.scripts.tahoe_status", "allmydata.scripts.tahoe_unlink", + "allmydata.scripts.tahoe_webopen", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From f48cf88f3503df9304425bab0b5429141007aba9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Jun 2021 09:45:29 -0400 Subject: [PATCH 383/463] Fix flake. --- src/allmydata/scripts/tahoe_status.py | 4 ++-- src/allmydata/test/cli/test_status.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/allmydata/scripts/tahoe_status.py b/src/allmydata/scripts/tahoe_status.py index a723274f0..75bc775f8 100644 --- a/src/allmydata/scripts/tahoe_status.py +++ b/src/allmydata/scripts/tahoe_status.py @@ -61,7 +61,7 @@ def _get_json_for_cap(options, cap): 'uri/%s?t=json' % url_quote(cap), ) -def pretty_progress(percent, size=10, ascii=False): +def pretty_progress(percent, size=10, output_ascii=False): """ Displays a unicode or ascii based progress bar of a certain length. Should we just depend on a library instead? @@ -72,7 +72,7 @@ def pretty_progress(percent, size=10, ascii=False): curr = int(percent / 100.0 * size) part = (percent / (100.0 / size)) - curr - if ascii: + if output_ascii: part = int(part * 4) part = '.oO%'[part] block_chr = '#' diff --git a/src/allmydata/test/cli/test_status.py b/src/allmydata/test/cli/test_status.py index 58818ed41..724661211 100644 --- a/src/allmydata/test/cli/test_status.py +++ b/src/allmydata/test/cli/test_status.py @@ -60,30 +60,30 @@ class FakeStatus(object): class ProgressBar(unittest.TestCase): def test_ascii0(self): - prog = pretty_progress(80.0, size=10, ascii=True) + prog = pretty_progress(80.0, size=10, output_ascii=True) self.assertEqual('########. ', prog) def test_ascii1(self): - prog = pretty_progress(10.0, size=10, ascii=True) + prog = pretty_progress(10.0, size=10, output_ascii=True) self.assertEqual('#. ', prog) def test_ascii2(self): - prog = pretty_progress(13.0, size=10, ascii=True) + prog = pretty_progress(13.0, size=10, output_ascii=True) self.assertEqual('#o ', prog) def test_ascii3(self): - prog = pretty_progress(90.0, size=10, ascii=True) + prog = pretty_progress(90.0, size=10, output_ascii=True) self.assertEqual('#########.', prog) def test_unicode0(self): self.assertEqual( - pretty_progress(82.0, size=10, ascii=False), + pretty_progress(82.0, size=10, output_ascii=False), u'\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u258e ', ) def test_unicode1(self): self.assertEqual( - pretty_progress(100.0, size=10, ascii=False), + pretty_progress(100.0, size=10, output_ascii=False), u'\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588', ) From b0013f00037603036cb2b2a425cf3abebf45f010 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 7 Jun 2021 15:30:46 -0400 Subject: [PATCH 384/463] Add newsfragment --- newsfragments/3730.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3730.minor diff --git a/newsfragments/3730.minor b/newsfragments/3730.minor new file mode 100644 index 000000000..e69de29bb From a3c221bab930d03e91a77251b76860c570427914 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 7 Jun 2021 15:31:12 -0400 Subject: [PATCH 385/463] Rename towncrier configuration file --- towncrier.pyproject.toml => towncrier.toml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename towncrier.pyproject.toml => towncrier.toml (100%) diff --git a/towncrier.pyproject.toml b/towncrier.toml similarity index 100% rename from towncrier.pyproject.toml rename to towncrier.toml From af919cd5bdacfe598a88b6998e2d7c365ed2569c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 7 Jun 2021 15:35:12 -0400 Subject: [PATCH 386/463] Update towncrier in `news` and `draftnews` tox envs Since `towncrier.toml` is Towncrier's default configuration file, `--config towncrier.toml` is not strictly necessary. But being explict about the configuration file seems to be a good idea. --- tox.ini | 53 ++++++++--------------------------------------------- 1 file changed, 8 insertions(+), 45 deletions(-) diff --git a/tox.ini b/tox.ini index d33fe2eae..e74959a2f 100644 --- a/tox.ini +++ b/tox.ini @@ -169,59 +169,22 @@ passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH # see comment in [testenv] about "certifi" whitelist_externals = mv deps = - certifi - towncrier >= 19.2 + certifi + towncrier==21.3.0 commands = - # With pip >= 10 the existence of pyproject.toml (which we are - # required to have to configure towncrier) triggers a "build - # isolation" mode which prevents anything from working. Avoid - # triggering that pip behavior by keeping the towncrier configuration - # somewhere else and only bringing it in when it's actually needed - # (after pip is done). - # - # Some discussion is available at - # https://github.com/pypa/pip/issues/5696 - # - # towncrier post 19.2 (unreleased as of this writing) adds a --config - # option that can be used instead of this file shuffling. - mv towncrier.pyproject.toml pyproject.toml - - # towncrier 19.2 + works with python2.7 - python -m towncrier --draft - - # put it back - mv pyproject.toml towncrier.pyproject.toml + python -m towncrier --draft --config towncrier.toml [testenv:news] passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH # see comment in [testenv] about "certifi" whitelist_externals = mv deps = - certifi - towncrier >= 19.2 + certifi + towncrier==21.3.0 commands = - # With pip >= 10 the existence of pyproject.toml (which we are - # required to have to configure towncrier) triggers a "build - # isolation" mode which prevents anything from working. Avoid - # triggering that pip behavior by keeping the towncrier configuration - # somewhere else and only bringing it in when it's actually needed - # (after pip is done). - # - # Some discussion is available at - # https://github.com/pypa/pip/issues/5696 - # - # towncrier post 19.2 (unreleased as of this writing) adds a --config - # option that can be used instead of this file shuffling. - mv towncrier.pyproject.toml pyproject.toml - - # towncrier 19.2 + works with python2.7 - python -m towncrier --yes - - # put it back - mv pyproject.toml towncrier.pyproject.toml - - # commit the changes - git commit -m "update NEWS.txt for release" + python -m towncrier --yes --config towncrier.toml + # commit the changes + git commit -m "update NEWS.txt for release" [testenv:deprecations] commands = From 9678d2ff790b799273d943e8187b6e22523a58b7 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 7 Jun 2021 15:52:20 -0400 Subject: [PATCH 387/463] Update towncrier config file in `codechecks` tox env --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index e74959a2f..272e902fe 100644 --- a/tox.ini +++ b/tox.ini @@ -136,8 +136,8 @@ commands = # If towncrier.check fails, you forgot to add a towncrier news # fragment explaining the change in this branch. Create one at # `newsfragments/.` with some text for the news - # file. See towncrier.pyproject.toml for legal values. - python -m towncrier.check --config towncrier.pyproject.toml + # file. See towncrier.toml for legal values. + python -m towncrier.check --config towncrier.toml [testenv:codechecks3] From c5d716804272d172fb8856d29e02ecd8b332af52 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 7 Jun 2021 17:29:46 -0400 Subject: [PATCH 388/463] Add newsfragment --- newsfragments/3731.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3731.minor diff --git a/newsfragments/3731.minor b/newsfragments/3731.minor new file mode 100644 index 000000000..e69de29bb From d9d01869c3ea30da616cbe646d57aaf1198bf2c0 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 7 Jun 2021 17:30:59 -0400 Subject: [PATCH 389/463] Pass $HOME to git in tox `news` environment This is needed on macOS, or `git commit` would fail. Also, $HOME has no place in `codechecks` Tox env, since we do not invoke git there, unlike the (now removed) comment indicated. --- tox.ini | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 272e902fe..1920b62c7 100644 --- a/tox.ini +++ b/tox.ini @@ -111,8 +111,6 @@ commands = [testenv:codechecks] basepython = python2.7 -# On macOS, git inside of towncrier needs $HOME. -passenv = HOME whitelist_externals = /bin/mv setenv = @@ -175,7 +173,8 @@ commands = python -m towncrier --draft --config towncrier.toml [testenv:news] -passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH +# On macOS, git invoked from Tox needs $HOME. +passenv = HOME * TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH # see comment in [testenv] about "certifi" whitelist_externals = mv deps = From 232b0d44eee50cb341d20bb491ef1080656cbc64 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 7 Jun 2021 17:35:25 -0400 Subject: [PATCH 390/463] Add newsfragment --- newsfragments/3732.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3732.minor diff --git a/newsfragments/3732.minor b/newsfragments/3732.minor new file mode 100644 index 000000000..e69de29bb From 3008a85dde921e38befcb3a66ff93eba5faa213c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 7 Jun 2021 17:36:14 -0400 Subject: [PATCH 391/463] Whitelist git in tox `news` environment --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 272e902fe..b13415dad 100644 --- a/tox.ini +++ b/tox.ini @@ -177,7 +177,9 @@ commands = [testenv:news] passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH # see comment in [testenv] about "certifi" -whitelist_externals = mv +whitelist_externals = + mv + git deps = certifi towncrier==21.3.0 From 6363162b9df609cacce038f1c8c88d97100a1f9d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 7 Jun 2021 17:37:11 -0400 Subject: [PATCH 392/463] Comment about certify at the appropriate place --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index b13415dad..e3d35d718 100644 --- a/tox.ini +++ b/tox.ini @@ -166,9 +166,9 @@ commands = mypy src [testenv:draftnews] passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH -# see comment in [testenv] about "certifi" whitelist_externals = mv deps = + # see comment in [testenv] about "certifi" certifi towncrier==21.3.0 commands = @@ -176,11 +176,11 @@ commands = [testenv:news] passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH -# see comment in [testenv] about "certifi" whitelist_externals = mv git deps = + # see comment in [testenv] about "certifi" certifi towncrier==21.3.0 commands = From 92eb702f6be2d1cc02d53d776a6318578d455e32 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 8 Jun 2021 14:36:20 -0400 Subject: [PATCH 393/463] Add newsfragment --- newsfragments/3733.installation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3733.installation diff --git a/newsfragments/3733.installation b/newsfragments/3733.installation new file mode 100644 index 000000000..c1cac649b --- /dev/null +++ b/newsfragments/3733.installation @@ -0,0 +1 @@ +Use netifaces 0.11.0 wheel package from PyPI.org if you use 64-bit Python 2.7 on Windows. VCPython27 downloads are no longer available at Microsoft's website, which has made building Python 2.7 wheel packages of Python libraries with C extensions (such as netifaces) on Windows difficult. From 197b9051749303896e18ce5cb5792d207a666350 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 8 Jun 2021 14:36:59 -0400 Subject: [PATCH 394/463] Update netifaces dependency --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index df917eb40..22988de73 100644 --- a/setup.py +++ b/setup.py @@ -128,7 +128,7 @@ install_requires = [ "future >= 0.18.2", # Discover local network configuration - "netifaces", + "netifaces >= 0.11.0", # Utility code: "pyutil >= 3.3.0", From 6c578df6a9307a7d444f718d0fd8a384e4436017 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 8 Jun 2021 14:48:12 -0400 Subject: [PATCH 395/463] CI: remove special handling for Python 2.7 + Windows --- .github/workflows/ci.yml | 41 ---------------------------------------- 1 file changed, 41 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 875f21cc9..92a0b9bfc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,33 +37,10 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - if: ${{ matrix.os != 'windows-latest' }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - # See note below about need for using 32-bit Python 2.7 on - # Windows. - - name: Set up Python ${{ matrix.python-version }} [Windows x64] - if: ${{ ( matrix.os == 'windows-latest' ) && ( matrix.python-version != '2.7' ) }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - architecture: 'x64' - - # We use netifaces, which does not ship a 64-bit wheel for the - # Python 2.7 + Windows combination, but it ships a 32-bit wheel. - # Since MS has removed vcpython27 compiler downloads from their - # usual download site, building a netifaces wheel locally is not - # an option anymore. So let us just test with 32-bit Python on - # Windows. - - name: Set up Python ${{ matrix.python-version }} [Windows x86] - if: ${{ ( matrix.os == 'windows-latest' ) && ( matrix.python-version == '2.7' ) }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - architecture: 'x86' - # To use pip caching with GitHub Actions in an OS-independent # manner, we need `pip cache dir` command, which became # available since pip v20.1+. At the time of writing this, @@ -203,19 +180,10 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - if: ${{ matrix.os != 'windows-latest' }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - # See this step under coverage job. - - name: Set up Python ${{ matrix.python-version }} [Windows x86] - if: ${{ matrix.os == 'windows-latest' }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - architecture: 'x86' - - name: Get pip cache directory id: pip-cache run: | @@ -272,19 +240,10 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - if: ${{ matrix.os != 'windows-latest' }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - # See this step under coverage job. - - name: Set up Python ${{ matrix.python-version }} [Windows x86] - if: ${{ matrix.os == 'windows-latest' }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - architecture: 'x86' - - name: Get pip cache directory id: pip-cache run: | From ccfd897914edd7e1a2d4daec58f720d6aa811002 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 8 Jun 2021 14:54:19 -0400 Subject: [PATCH 396/463] Do not rock the NixOS boat --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 22988de73..df917eb40 100644 --- a/setup.py +++ b/setup.py @@ -128,7 +128,7 @@ install_requires = [ "future >= 0.18.2", # Discover local network configuration - "netifaces >= 0.11.0", + "netifaces", # Utility code: "pyutil >= 3.3.0", From 6fdda6163c528d654b0a0b9178cd742080b4ed61 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Jun 2021 10:47:17 -0400 Subject: [PATCH 397/463] Run Python 3 integration tests on more than just Linux. --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 875f21cc9..15904c7b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,11 +175,11 @@ jobs: os: - macos-latest - windows-latest + - ubuntu-latest python-version: - 2.7 - include: - - os: ubuntu-latest - python-version: 3.6 + - 3.6 + - 3.9 steps: From 051f1e4c9e3c6666d6c0106df76cf51a6404471d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Jun 2021 10:47:39 -0400 Subject: [PATCH 398/463] News file. --- newsfragments/3722.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3722.minor diff --git a/newsfragments/3722.minor b/newsfragments/3722.minor new file mode 100644 index 000000000..e69de29bb From 490d3ca29d7334b3d1cdc95c5f70456bbef91984 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Jun 2021 11:03:01 -0400 Subject: [PATCH 399/463] Work with latest mypy. --- newsfragments/3734.minor | 0 tox.ini | 6 +++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3734.minor diff --git a/newsfragments/3734.minor b/newsfragments/3734.minor new file mode 100644 index 000000000..e69de29bb diff --git a/tox.ini b/tox.ini index 272e902fe..bf688f882 100644 --- a/tox.ini +++ b/tox.ini @@ -155,7 +155,11 @@ basepython = python3 skip_install = True deps = mypy - git+https://github.com/Shoobx/mypy-zope + mypy-zope + types-mock + types-six + types-PyYAML + types-pkg_resources git+https://github.com/warner/foolscap # Twisted 21.2.0 introduces some type hints which we are not yet # compatible with. From 702f249f998ce9167dc43e87521f5290574084b7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Jun 2021 11:22:11 -0400 Subject: [PATCH 400/463] Use the correct py.test. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index bf688f882..b86abe15b 100644 --- a/tox.ini +++ b/tox.ini @@ -104,7 +104,7 @@ setenv = commands = python --version # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' - python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration} + py.test --timeout=1800 --coverage -v {posargs:integration} coverage combine coverage report From 2214e7a59fbf4f7f9bbb726447d170b4342b2ca3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Jun 2021 11:52:30 -0400 Subject: [PATCH 401/463] macOS has issues on Python 3, don't want to fix them in this PR. --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15904c7b3..d5b5f581d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,13 +173,15 @@ jobs: fail-fast: false matrix: os: - - macos-latest - windows-latest - ubuntu-latest python-version: - 2.7 - 3.6 - 3.9 + include: + - os: macos-latest + python-version: 2.7 steps: From ca793f9ff1df0cbc898b93df676aaf7e2f006d4a Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 13 Jun 2021 22:38:07 -0600 Subject: [PATCH 402/463] bz2 -> gz --- docs/INSTALL.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/INSTALL.rst b/docs/INSTALL.rst index 253cbe5ae..71add0cf3 100644 --- a/docs/INSTALL.rst +++ b/docs/INSTALL.rst @@ -181,8 +181,8 @@ following instructions with the local filename. New python executable in ~/venv/bin/python2.7 Installing setuptools, pip, wheel...done. - % venv/bin/pip install https://tahoe-lafs.org/downloads/tahoe-lafs-1.15.1.tar.bz2 - Collecting https://tahoe-lafs.org/downloads/tahoe-lafs-1.15.1.tar.bz2 + % venv/bin/pip install https://tahoe-lafs.org/downloads/tahoe-lafs-1.15.1.tar.gz + Collecting https://tahoe-lafs.org/downloads/tahoe-lafs-1.15.1.tar.gz ... Installing collected packages: ... Successfully installed ... @@ -208,17 +208,17 @@ those developers authorized to sign a Tahoe release: :code: Signatures are made available beside the release. So for example, a release -like ``https://tahoe-lafs.org/downloads/tahoe-lafs-1.16.0.tar.bz2`` might -have signatures ``tahoe-lafs-1.16.0.tar.bz2.meejah.asc`` and -``tahoe-lafs-1.16.0.tar.bz2.warner.asc``. +like ``https://tahoe-lafs.org/downloads/tahoe-lafs-1.16.0.tar.gz`` might +have signatures ``tahoe-lafs-1.16.0.tar.gz.meejah.asc`` and +``tahoe-lafs-1.16.0.tar.gz.warner.asc``. To verify the signatures using GnuPG:: - % gpg --verify tahoe-lafs-1.16.0.tar.bz2.meejah.asc tahoe-lafs-1.16.0.tar.bz2 + % gpg --verify tahoe-lafs-1.16.0.tar.gz.meejah.asc tahoe-lafs-1.16.0.tar.gz gpg: Signature made XXX gpg: using RSA key 9D5A2BD5688ECB889DEBCD3FC2602803128069A7 gpg: Good signature from "meejah " [full] - % gpg --verify tahoe-lafs-1.16.0.tar.bz2.warner.asc tahoe-lafs-1.16.0.tar.bz2 + % gpg --verify tahoe-lafs-1.16.0.tar.gz.warner.asc tahoe-lafs-1.16.0.tar.gz gpg: Signature made XXX gpg: using RSA key 967EFE06699872411A77DF36D43B4C9C73225AAF gpg: Good signature from "Brian Warner " [full] From 7a63c95d68dc7506b297c0cad9c6f959ce7e580c Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 13 Jun 2021 22:38:22 -0600 Subject: [PATCH 403/463] news --- newsfragments/3735.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3735.minor diff --git a/newsfragments/3735.minor b/newsfragments/3735.minor new file mode 100644 index 000000000..e69de29bb From b69902091a8a2923d61d67995748ee67f4b3a82a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Jun 2021 10:55:43 -0400 Subject: [PATCH 404/463] News file. --- newsfragments/3702.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3702.minor diff --git a/newsfragments/3702.minor b/newsfragments/3702.minor new file mode 100644 index 000000000..e69de29bb From 744f9bab4ecfbfdae2084eba2f47292ef9f4a52f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Jun 2021 10:55:49 -0400 Subject: [PATCH 405/463] Works fine on Python 3. --- misc/coding_tools/check-debugging.py | 9 +++++++-- misc/coding_tools/check-umids.py | 8 +++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/misc/coding_tools/check-debugging.py b/misc/coding_tools/check-debugging.py index f2ba6528e..b920f5634 100755 --- a/misc/coding_tools/check-debugging.py +++ b/misc/coding_tools/check-debugging.py @@ -1,13 +1,18 @@ #! /usr/bin/python -# ./check-debugging.py src +""" +Checks for defer.setDebugging(). + +Runs on Python 3. + +Usage: ./check-debugging.py src +""" from __future__ import print_function import sys, re, os ok = True -umids = {} for starting_point in sys.argv[1:]: for root, dirs, files in os.walk(starting_point): diff --git a/misc/coding_tools/check-umids.py b/misc/coding_tools/check-umids.py index c06b795fe..345610f3e 100644 --- a/misc/coding_tools/check-umids.py +++ b/misc/coding_tools/check-umids.py @@ -1,4 +1,10 @@ -#! /usr/bin/python +#! /usr/bin/python3 + +""" +Ensure UMIDS are unique. + +This runs on Python 3. +""" # ./check-umids.py src From 28220d8a167d60b645d34dbf95c0a8885aeaee30 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Jun 2021 11:24:10 -0400 Subject: [PATCH 406/463] Switch to pylint instead of custom AST parsing that wouldn't work on Python 3. --- misc/coding_tools/check-miscaptures.py | 186 ------------------------- tox.ini | 37 +++-- 2 files changed, 23 insertions(+), 200 deletions(-) delete mode 100644 misc/coding_tools/check-miscaptures.py diff --git a/misc/coding_tools/check-miscaptures.py b/misc/coding_tools/check-miscaptures.py deleted file mode 100644 index 81e76f891..000000000 --- a/misc/coding_tools/check-miscaptures.py +++ /dev/null @@ -1,186 +0,0 @@ -#! /usr/bin/python - -from __future__ import print_function - -import os, sys, compiler -from compiler.ast import Node, For, While, ListComp, AssName, Name, Lambda, Function - - -def check_source(source): - return check_thing(compiler.parse, source) - -def check_file(path): - return check_thing(compiler.parseFile, path) - -def check_thing(parser, thing): - try: - ast = parser(thing) - except SyntaxError as e: - return e - else: - results = [] - check_ast(ast, results) - return results - -def check_ast(ast, results): - """Check a node outside a loop.""" - if isinstance(ast, (For, While, ListComp)): - check_loop(ast, results) - else: - for child in ast.getChildNodes(): - if isinstance(ast, Node): - check_ast(child, results) - -def check_loop(ast, results): - """Check a particular outer loop.""" - - # List comprehensions have a poorly designed AST of the form - # ListComp(exprNode, [ListCompFor(...), ...]), in which the - # result expression is outside the ListCompFor node even though - # it is logically inside the loop(s). - # There may be multiple ListCompFor nodes (in cases such as - # [lambda: (a,b) for a in ... for b in ...] - # ), and that case they are not nested in the AST. But these - # warts (nonobviously) happen not to matter for our analysis. - - assigned = {} # maps name to lineno of topmost assignment - nested = set() - collect_assigned_and_nested(ast, assigned, nested) - - # For each nested function... - for funcnode in nested: - # Check for captured variables in this function. - captured = set() - collect_captured(funcnode, assigned, captured, False) - for name in captured: - # We want to report the outermost capturing function - # (since that is where the workaround will need to be - # added), and the topmost assignment to the variable. - # Just one report per capturing function per variable - # will do. - results.append(make_result(funcnode, name, assigned[name])) - - # Check each node in the function body in case it - # contains another 'for' loop. - childnodes = funcnode.getChildNodes()[len(funcnode.defaults):] - for child in childnodes: - check_ast(child, results) - -def collect_assigned_and_nested(ast, assigned, nested): - """ - Collect the names assigned in this loop, not including names - assigned in nested functions. Also collect the nodes of functions - that are nested one level deep. - """ - if isinstance(ast, AssName): - if ast.name not in assigned or assigned[ast.name] > ast.lineno: - assigned[ast.name] = ast.lineno - else: - childnodes = ast.getChildNodes() - if isinstance(ast, (Lambda, Function)): - nested.add(ast) - - # The default argument expressions are "outside" the - # function, even though they are children of the - # Lambda or Function node. - childnodes = childnodes[:len(ast.defaults)] - - for child in childnodes: - if isinstance(ast, Node): - collect_assigned_and_nested(child, assigned, nested) - -def collect_captured(ast, assigned, captured, in_function_yet): - """Collect any captured variables that are also in assigned.""" - if isinstance(ast, Name): - if ast.name in assigned: - captured.add(ast.name) - else: - childnodes = ast.getChildNodes() - if isinstance(ast, (Lambda, Function)): - # Formal parameters of the function are excluded from - # captures we care about in subnodes of the function body. - new_assigned = assigned.copy() - remove_argnames(ast.argnames, new_assigned) - - if len(new_assigned) > 0: - for child in childnodes[len(ast.defaults):]: - collect_captured(child, new_assigned, captured, True) - - # The default argument expressions are "outside" *this* - # function, even though they are children of the Lambda or - # Function node. - if not in_function_yet: - return - childnodes = childnodes[:len(ast.defaults)] - - for child in childnodes: - if isinstance(ast, Node): - collect_captured(child, assigned, captured, True) - - -def remove_argnames(names, fromset): - for element in names: - if element in fromset: - del fromset[element] - elif isinstance(element, (tuple, list)): - remove_argnames(element, fromset) - - -def make_result(funcnode, var_name, var_lineno): - if hasattr(funcnode, 'name'): - func_name = 'function %r' % (funcnode.name,) - else: - func_name = '' - return (funcnode.lineno, func_name, var_name, var_lineno) - -def report(out, path, results): - for r in results: - print(path + (":%r %s captures %r assigned at line %d" % r), file=out) - -def check(sources, out): - class Counts(object): - n = 0 - processed_files = 0 - suspect_files = 0 - error_files = 0 - counts = Counts() - - def _process(path): - results = check_file(path) - if isinstance(results, SyntaxError): - print(path + (" NOT ANALYSED due to syntax error: %s" % results), file=out) - counts.error_files += 1 - else: - report(out, path, results) - counts.n += len(results) - counts.processed_files += 1 - if len(results) > 0: - counts.suspect_files += 1 - - for source in sources: - print("Checking %s..." % (source,), file=out) - if os.path.isfile(source): - _process(source) - else: - for (dirpath, dirnames, filenames) in os.walk(source): - for fn in filenames: - (basename, ext) = os.path.splitext(fn) - if ext == '.py': - _process(os.path.join(dirpath, fn)) - - print("%d suspiciously captured variables in %d out of %d file(s)." - % (counts.n, counts.suspect_files, counts.processed_files), file=out) - if counts.error_files > 0: - print("%d file(s) not processed due to syntax errors." - % (counts.error_files,), file=out) - return counts.n - - -sources = ['src'] -if len(sys.argv) > 1: - sources = sys.argv[1:] -if check(sources, sys.stderr) > 0: - sys.exit(1) - - -# TODO: self-tests diff --git a/tox.ini b/tox.ini index b86abe15b..3559a1077 100644 --- a/tox.ini +++ b/tox.ini @@ -109,12 +109,10 @@ commands = coverage report +# Once 2.7 is dropped, this can be removed. It just does flake8 with Python 2 +# since that can give different results than flake8 on Python 3. [testenv:codechecks] basepython = python2.7 -# On macOS, git inside of towncrier needs $HOME. -passenv = HOME -whitelist_externals = - /bin/mv setenv = # Workaround an error when towncrier is run under the VCS hook, # https://stackoverflow.com/a/4027726/624787: @@ -128,26 +126,37 @@ setenv = DEFAULT_FILES=src integration static misc setup.py commands = flake8 {posargs:{env:DEFAULT_FILES}} - python misc/coding_tools/check-umids.py {posargs:{env:DEFAULT_FILES}} - python misc/coding_tools/check-debugging.py {posargs:{env:DEFAULT_FILES}} - python misc/coding_tools/find-trailing-spaces.py -r {posargs:{env:DEFAULT_FILES}} - python misc/coding_tools/check-miscaptures.py {posargs:{env:DEFAULT_FILES}} - - # If towncrier.check fails, you forgot to add a towncrier news - # fragment explaining the change in this branch. Create one at - # `newsfragments/.` with some text for the news - # file. See towncrier.toml for legal values. - python -m towncrier.check --config towncrier.toml [testenv:codechecks3] basepython = python3 +deps = + # Newer versions of PyLint have buggy configuration + # (https://github.com/PyCQA/pylint/issues/4574), so stick to old version + # for now. + pylint < 2.5 +# On macOS, git inside of towncrier needs $HOME. +passenv = HOME +whitelist_externals = + /bin/mv setenv = # If no positional arguments are given, try to run the checks on the # entire codebase, including various pieces of supporting code. DEFAULT_FILES=src integration static misc setup.py commands = flake8 {posargs:{env:DEFAULT_FILES}} + python misc/coding_tools/check-umids.py {posargs:{env:DEFAULT_FILES}} + python misc/coding_tools/check-debugging.py {posargs:{env:DEFAULT_FILES}} + python misc/coding_tools/find-trailing-spaces.py -r {posargs:{env:DEFAULT_FILES}} + # PyLint has other useful checks, might want to enable them: + # http://pylint.pycqa.org/en/latest/technical_reference/features.html + pylint --disable=all --enable=cell-var-from-loop {posargs:{env:DEFAULT_FILES}} + + # If towncrier.check fails, you forgot to add a towncrier news + # fragment explaining the change in this branch. Create one at + # `newsfragments/.` with some text for the news + # file. See towncrier.toml for legal values. + python -m towncrier.check --config towncrier.toml [testenv:typechecks] From aa809669dbba1c65eb4cd40a52888010aa611f3c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Jun 2021 11:48:24 -0400 Subject: [PATCH 407/463] Add a couple more tests for more complete coverage. --- src/allmydata/test/test_auth.py | 35 ++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_auth.py b/src/allmydata/test/test_auth.py index 4922ca5f3..d5198d326 100644 --- a/src/allmydata/test/test_auth.py +++ b/src/allmydata/test/test_auth.py @@ -39,8 +39,10 @@ dBSD8940XU3YW+oeq8e+p3yQ2GinHfeJ3BYQyNQLuMAJ """) DUMMY_ACCOUNTS = u"""\ -alice password URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111 +alice herpassword URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111 bob sekrit URI:DIR2:bbbbbbbbbbbbbbbbbbbbbbbbbb:2222222222222222222222222222222222222222222222222222 + +# dennis password URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111 carol {key} URI:DIR2:cccccccccccccccccccccccccc:3333333333333333333333333333333333333333333333333333 """.format(key=str(DUMMY_KEY.public().toString("openssh"), "ascii")).encode("ascii") @@ -54,7 +56,7 @@ class AccountFileCheckerKeyTests(unittest.TestCase): abspath = abspath_expanduser_unicode(str(self.account_file.path)) self.checker = auth.AccountFileChecker(None, abspath) - def test_unknown_user(self): + def test_unknown_user_ssh(self): """ AccountFileChecker.requestAvatarId returns a Deferred that fires with UnauthorizedLogin if called with an SSHPrivateKey object with a @@ -65,6 +67,19 @@ class AccountFileCheckerKeyTests(unittest.TestCase): avatarId = self.checker.requestAvatarId(key_credentials) return self.assertFailure(avatarId, error.UnauthorizedLogin) + def test_unknown_user_password(self): + """ + AccountFileChecker.requestAvatarId returns a Deferred that fires with + UnauthorizedLogin if called with an SSHPrivateKey object with a + username not present in the account file. + + We use a commented out user, so we're also checking that comments are + skipped. + """ + key_credentials = credentials.UsernamePassword(b"dennis", b"password") + d = self.checker.requestAvatarId(key_credentials) + return self.assertFailure(d, error.UnauthorizedLogin) + def test_password_auth_user_with_ssh_key(self): """ AccountFileChecker.requestAvatarId returns a Deferred that fires with @@ -81,7 +96,21 @@ class AccountFileCheckerKeyTests(unittest.TestCase): AccountFileChecker.requestAvatarId returns a Deferred that fires with the user if the correct password is given. """ - key_credentials = credentials.UsernamePassword(b"alice", b"password") + key_credentials = credentials.UsernamePassword(b"alice", b"herpassword") + d = self.checker.requestAvatarId(key_credentials) + def authenticated(avatarId): + self.assertEqual( + (b"alice", + b"URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111"), + (avatarId.username, avatarId.rootcap)) + return d + + def test_password_auth_user_with_correct_hashed_password(self): + """ + AccountFileChecker.requestAvatarId returns a Deferred that fires with + the user if the correct password is given in hashed form. + """ + key_credentials = credentials.UsernameHashedPassword(b"alice", b"herpassword") d = self.checker.requestAvatarId(key_credentials) def authenticated(avatarId): self.assertEqual( From 6e3279317645f65483c1a4e6347eb178621e34fc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Jun 2021 11:51:40 -0400 Subject: [PATCH 408/463] News file. --- newsfragments/3736.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3736.minor diff --git a/newsfragments/3736.minor b/newsfragments/3736.minor new file mode 100644 index 000000000..e69de29bb From 2447d09fc0f7088492d9b1d53deca46e73818b5e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Jun 2021 11:58:48 -0400 Subject: [PATCH 409/463] Port to Python 3. --- src/allmydata/frontends/auth.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+) diff --git a/src/allmydata/frontends/auth.py b/src/allmydata/frontends/auth.py index 376cdebe4..b61062334 100644 --- a/src/allmydata/frontends/auth.py +++ b/src/allmydata/frontends/auth.py @@ -1,3 +1,15 @@ +""" +Authentication for frontends. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from zope.interface import implementer from twisted.internet import defer from twisted.cred import error, checkers, credentials diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 5e06c2c01..8025582f2 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -52,6 +52,7 @@ PORTED_MODULES = [ "allmydata.deep_stats", "allmydata.dirnode", "allmydata.frontends", + "allmydata.frontends.auth", "allmydata.frontends.sftpd", "allmydata.hashtree", "allmydata.history", From 473654ccb72b7f04cc1774339ef2c1826db3775a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Jun 2021 13:26:50 -0400 Subject: [PATCH 410/463] Test demonstrating the problem. --- src/allmydata/test/cli/test_cp.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py index d55bc01bd..fff50f331 100644 --- a/src/allmydata/test/cli/test_cp.py +++ b/src/allmydata/test/cli/test_cp.py @@ -55,6 +55,11 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("get", "tahoe:" + artonwall_arg)) d.addCallback(lambda rc_out_err: self.assertEqual(rc_out_err[1], DATA1)) + # Version where destination filename is explicitly Unicode too. + d.addCallback(lambda res: self.do_cli("cp", fn1, "tahoe:" + artonwall_arg + "-2")) + d.addCallback(lambda res: self.do_cli("get", "tahoe:" + artonwall_arg + "-2")) + d.addCallback(lambda rc_out_err: self.assertEqual(rc_out_err[1], DATA1)) + d.addCallback(lambda res: self.do_cli("cp", fn2, "tahoe:")) d.addCallback(lambda res: self.do_cli("get", "tahoe:Metallica")) @@ -74,7 +79,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 0) if PY2: out = out.decode(get_io_encoding()) - self.failUnlessReallyEqual(out, u"Metallica\n\u00C4rtonwall\n") + self.failUnlessReallyEqual(out, u"Metallica\n\u00C4rtonwall\n\u00C4rtonwall-2\n") self.assertEqual(len(err), 0, err) d.addCallback(_check) From d74ef0798d16581827f3e6740e4246655197330e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Jun 2021 13:46:22 -0400 Subject: [PATCH 411/463] Fix quoting to work on Python 2. --- src/allmydata/scripts/common_http.py | 6 ++++-- src/allmydata/web/common.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py index bf47f2dd6..cbc6e2969 100644 --- a/src/allmydata/scripts/common_http.py +++ b/src/allmydata/scripts/common_http.py @@ -97,8 +97,10 @@ def format_http_success(resp): return "%s %s" % (resp.status, quote_output(resp.reason, quotemarks=False)) def format_http_error(msg, resp): - return "%s: %s %s\n%s" % (msg, resp.status, quote_output(resp.reason, quotemarks=False), - quote_output(resp.read(), quotemarks=False)) + return quote_output( + "%s: %s %s\n%s" % (msg, resp.status, str(resp.reason, "utf-8"), + str(resp.read(), "utf-8")), + quotemarks=False) def check_http_error(resp, stderr): if resp.status < 200 or resp.status >= 300: diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index bb84a2e70..bf89044a3 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -90,6 +90,7 @@ from allmydata.util.time_format import ( ) from allmydata.util.encodingutil import ( quote_output, + quote_output_u, to_bytes, ) from allmydata.util import abbreviate @@ -324,7 +325,7 @@ def humanize_exception(exc): return ("There was already a child by that name, and you asked me " "to not replace it.", http.CONFLICT) if isinstance(exc, NoSuchChildError): - quoted_name = quote_output(exc.args[0], encoding="utf-8", quotemarks=False) + quoted_name = quote_output_u(exc.args[0], quotemarks=False) return ("No such child: %s" % quoted_name, http.NOT_FOUND) if isinstance(exc, NotEnoughSharesError): t = ("NotEnoughSharesError: This indicates that some " From 23a71db38d7fb3f77a923911b5c6c8f4a00e022e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Jun 2021 13:46:57 -0400 Subject: [PATCH 412/463] News file. --- newsfragments/3738.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3738.bugfix diff --git a/newsfragments/3738.bugfix b/newsfragments/3738.bugfix new file mode 100644 index 000000000..6a4bc1cd9 --- /dev/null +++ b/newsfragments/3738.bugfix @@ -0,0 +1 @@ +Fix regression where uploading files with non-ASCII names failed. \ No newline at end of file From 3461665ccfd04164ffbcd9aabfea650d02e9ecfb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Jun 2021 13:56:34 -0400 Subject: [PATCH 413/463] Get rid of annoying warning. --- newsfragments/3739.bugfix | 1 + src/allmydata/scripts/runner.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3739.bugfix diff --git a/newsfragments/3739.bugfix b/newsfragments/3739.bugfix new file mode 100644 index 000000000..875941cf8 --- /dev/null +++ b/newsfragments/3739.bugfix @@ -0,0 +1 @@ +Fixed annoying UnicodeWarning message on Python 2 when running CLI tools. \ No newline at end of file diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 43a3b8807..7f7f88bf6 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -204,7 +204,8 @@ def _setup_coverage(reactor): """ # can we put this _setup_coverage call after we hit # argument-parsing? - if '--coverage' not in sys.argv: + # ensure_str() only necessary on Python 2. + if six.ensure_str('--coverage') not in sys.argv: return sys.argv.remove('--coverage') From e8308043e3181b65756b80c926c653a78c1950be Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Jun 2021 14:03:58 -0400 Subject: [PATCH 414/463] Method that doesn't break on Python 3. --- src/allmydata/scripts/common_http.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py index cbc6e2969..a842f93ae 100644 --- a/src/allmydata/scripts/common_http.py +++ b/src/allmydata/scripts/common_http.py @@ -97,9 +97,10 @@ def format_http_success(resp): return "%s %s" % (resp.status, quote_output(resp.reason, quotemarks=False)) def format_http_error(msg, resp): + # ensure_text() shouldn't be necessary when Python 2 is dropped. return quote_output( - "%s: %s %s\n%s" % (msg, resp.status, str(resp.reason, "utf-8"), - str(resp.read(), "utf-8")), + "%s: %s %s\n%s" % (msg, resp.status, six.ensure_text(resp.reason), + six.ensure_text(resp.read())), quotemarks=False) def check_http_error(resp, stderr): From f778d25c44c7d2feb59332a43f89ceeafd7a05c7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Jun 2021 14:05:49 -0400 Subject: [PATCH 415/463] Apply same fix to success path. --- src/allmydata/scripts/common_http.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py index a842f93ae..95099a2eb 100644 --- a/src/allmydata/scripts/common_http.py +++ b/src/allmydata/scripts/common_http.py @@ -94,7 +94,10 @@ def do_http(method, url, body=b""): def format_http_success(resp): - return "%s %s" % (resp.status, quote_output(resp.reason, quotemarks=False)) + # ensure_text() shouldn't be necessary when Python 2 is dropped. + return quote_output( + "%s %s" % (resp.status, six.ensure_text(resp.reason)), + quotemarks=False) def format_http_error(msg, resp): # ensure_text() shouldn't be necessary when Python 2 is dropped. From 3f98349cd9f3000ddbf71ff97ca72e7c9ed14f41 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Jun 2021 11:53:25 -0400 Subject: [PATCH 416/463] Limit Pickle protocol to version supported by Python 2. --- misc/operations_helpers/cpu-watcher.tac | 4 +++- newsfragments/3741.minor | 0 src/allmydata/storage/crawler.py | 4 +++- src/allmydata/storage/expirer.py | 8 ++++++-- 4 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 newsfragments/3741.minor diff --git a/misc/operations_helpers/cpu-watcher.tac b/misc/operations_helpers/cpu-watcher.tac index 795b9c444..c50b51c61 100644 --- a/misc/operations_helpers/cpu-watcher.tac +++ b/misc/operations_helpers/cpu-watcher.tac @@ -201,7 +201,9 @@ class CPUWatcher(service.MultiService, resource.Resource, Referenceable): log.msg("error reading process %s (%s), ignoring" % (pid, name)) log.err() try: - pickle.dump(self.history, open("history.pickle.tmp", "wb")) + # Newer protocols won't work in Python 2; when it is dropped, + # protocol v4 can be used (added in Python 3.4). + pickle.dump(self.history, open("history.pickle.tmp", "wb"), protocol=2) os.rename("history.pickle.tmp", "history.pickle") except: pass diff --git a/newsfragments/3741.minor b/newsfragments/3741.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index f13f7cb99..bd4f4f432 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -252,7 +252,9 @@ class ShareCrawler(service.MultiService): self.state["last-complete-prefix"] = last_complete_prefix tmpfile = self.statefile + ".tmp" with open(tmpfile, "wb") as f: - pickle.dump(self.state, f) + # Newer protocols won't work in Python 2; when it is dropped, + # protocol v4 can be used (added in Python 3.4). + pickle.dump(self.state, f, protocol=2) fileutil.move_into_place(tmpfile, self.statefile) def startService(self): diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index ffe2bf774..7c6cd8218 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -95,7 +95,9 @@ class LeaseCheckingCrawler(ShareCrawler): if not os.path.exists(self.historyfile): history = {} # cyclenum -> dict with open(self.historyfile, "wb") as f: - pickle.dump(history, f) + # Newer protocols won't work in Python 2; when it is dropped, + # protocol v4 can be used (added in Python 3.4). + pickle.dump(history, f, protocol=2) def create_empty_cycle_dict(self): recovered = self.create_empty_recovered_dict() @@ -319,7 +321,9 @@ class LeaseCheckingCrawler(ShareCrawler): oldcycles = sorted(history.keys()) del history[oldcycles[0]] with open(self.historyfile, "wb") as f: - pickle.dump(history, f) + # Newer protocols won't work in Python 2; when it is dropped, + # protocol v4 can be used (added in Python 3.4). + pickle.dump(history, f, protocol=2) def get_state(self): """In addition to the crawler state described in From f65d6c14c9e3fe6c85efb73e6dd8cab18d32980e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 25 Jun 2021 11:03:47 -0400 Subject: [PATCH 417/463] Rename related newsfragment The newsfragment text related to ticket #3681 is in odds with the changess made in ticket #3733. We do not want the old text to confuse people. Renaming it to .minor should do it, since minor newsfragments are excluded from final NEWS file. --- newsfragments/{3681.installation => 3681.minor} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{3681.installation => 3681.minor} (100%) diff --git a/newsfragments/3681.installation b/newsfragments/3681.minor similarity index 100% rename from newsfragments/3681.installation rename to newsfragments/3681.minor From a2186f3c989a9027cca9f5c279b18c4289d48e80 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 25 Jun 2021 11:07:11 -0400 Subject: [PATCH 418/463] Add an explanation within newsfragment Why are we changing an unrelated newsfragment? --- newsfragments/3681.minor | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/newsfragments/3681.minor b/newsfragments/3681.minor index a697e6c60..bc84b6b8f 100644 --- a/newsfragments/3681.minor +++ b/newsfragments/3681.minor @@ -1,3 +1,8 @@ +(The below text is no longer valid: netifaces has released a 64-bit +Python 2.7 wheel for Windows. Ticket #3733 made the switch in CI. We +should be able to test and run Tahoe-LAFS without needing vcpython27 +now.) + Tahoe-LAFS CI now runs tests only on 32-bit Windows. Microsoft has removed vcpython27 compiler downloads from their site, and Tahoe-LAFS needs vcpython27 to build and install netifaces on 64-bit Windows. From 0e08d0e080a80c88e14a1b9412e811af307e7a35 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 2 Jul 2021 10:09:47 -0700 Subject: [PATCH 419/463] News file --- newsfragments/3686.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3686.minor diff --git a/newsfragments/3686.minor b/newsfragments/3686.minor new file mode 100644 index 000000000..e69de29bb From 72d5bedf14dbcfdc90d5eb50351577daf31fdbac Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 2 Jul 2021 10:10:26 -0700 Subject: [PATCH 420/463] Depend on newer foolscap. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index df917eb40..1b64c9b09 100644 --- a/setup.py +++ b/setup.py @@ -54,9 +54,9 @@ install_requires = [ # * foolscap >= 0.12.5 has ConnectionInfo and ReconnectionInfo # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs # * foolscap 0.13.2 drops i2p support completely - # * foolscap >= 20.4 is necessary for Python 3 + # * foolscap >= 21.7 is necessary for Python 3 with i2p support. "foolscap == 0.13.1 ; python_version < '3.0'", - "foolscap >= 20.4.0 ; python_version > '3.0'", + "foolscap >= 21.7.0 ; python_version > '3.0'", # * cryptography 2.6 introduced some ed25519 APIs we rely on. Note that # Twisted[conch] also depends on cryptography and Twisted[tls] From 2494980a69b1c63456fe3aff953e1e2e0157cdff Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 5 Jul 2021 06:36:29 -0400 Subject: [PATCH 421/463] Remove stray `*` from tox passenv value --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 96fa68962..922abb56a 100644 --- a/tox.ini +++ b/tox.ini @@ -189,7 +189,7 @@ commands = [testenv:news] # On macOS, git invoked from Tox needs $HOME. -passenv = HOME * TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH +passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH HOME whitelist_externals = mv git From 7ebe654e5519843c6a819330fcc8f61aedbf5f01 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 5 Jul 2021 07:21:33 -0400 Subject: [PATCH 422/463] Remove unnecessary `whitelist_externals` from Tox configuration --- newsfragments/3745.minor | 0 tox.ini | 4 ---- 2 files changed, 4 deletions(-) create mode 100644 newsfragments/3745.minor diff --git a/newsfragments/3745.minor b/newsfragments/3745.minor new file mode 100644 index 000000000..e69de29bb diff --git a/tox.ini b/tox.ini index 7a7427299..4f6756c1c 100644 --- a/tox.ini +++ b/tox.ini @@ -137,8 +137,6 @@ deps = pylint < 2.5 # On macOS, git inside of towncrier needs $HOME. passenv = HOME -whitelist_externals = - /bin/mv setenv = # If no positional arguments are given, try to run the checks on the # entire codebase, including various pieces of supporting code. @@ -179,7 +177,6 @@ commands = mypy src [testenv:draftnews] passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH -whitelist_externals = mv deps = # see comment in [testenv] about "certifi" certifi @@ -190,7 +187,6 @@ commands = [testenv:news] passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH whitelist_externals = - mv git deps = # see comment in [testenv] about "certifi" From d2fdd92b8b91ef49fda4b568706913886babe116 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 5 Jul 2021 08:37:49 -0400 Subject: [PATCH 423/463] Use older Tor on macOS + Python 2.7 --- .github/workflows/ci.yml | 15 +++++++++++++-- newsfragments/3744.minor | 0 2 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 newsfragments/3744.minor diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5b5f581d..b1c1108f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,9 +189,20 @@ jobs: if: matrix.os == 'ubuntu-latest' run: sudo apt install tor - - name: Install Tor [macOS] - if: matrix.os == 'macos-latest' + - name: Install Tor [macOS, ${{ matrix.python-version }} ] + if: ${{ ( matrix.os == 'macos-latest' ) && ( matrix.python-version != '2.7' ) }} run: brew install tor + + # TODO: See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3744. + # We have to use an older version of Tor for running integration + # tests on macOS using Python 2.7. We should stop doing this + # when Python 2.7 is dropped. + - name: Install Tor [macOS, ${{ matrix.python-version }} ] + if: ${{ ( matrix.os == 'macos-latest' ) && ( matrix.python-version == '2.7' ) }} + run: | + brew extract --version 0.4.5.8 tor homebrew/cask + brew install tor@0.4.5.8 + brew link --overwrite tor@0.4.5.8 - name: Install Tor [Windows] if: matrix.os == 'windows-latest' diff --git a/newsfragments/3744.minor b/newsfragments/3744.minor new file mode 100644 index 000000000..e69de29bb From e0b4cf057e94bf33f3a2f68f1084d8cfd4a3e21a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Jul 2021 16:03:24 -0700 Subject: [PATCH 424/463] Fix logging of bytes in Foolscap on Python 3. --- newsfragments/3708.minor | 0 src/allmydata/util/jsonbytes.py | 12 ++++++------ src/allmydata/util/log.py | 11 ++++++++--- 3 files changed, 14 insertions(+), 9 deletions(-) create mode 100644 newsfragments/3708.minor diff --git a/newsfragments/3708.minor b/newsfragments/3708.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index f6143f4d1..08e0cb68e 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -28,8 +28,8 @@ if PY2: codecs.register_error("backslashreplace_tahoe_py2", backslashreplace_py2) -def _bytes_to_unicode(any_bytes, obj): - """Create a function that recursively converts bytes to unicode. +def bytes_to_unicode(any_bytes, obj): + """Convert bytes to unicode. :param any_bytes: If True, also support non-UTF-8-encoded bytes. :param obj: Object to de-byte-ify. @@ -63,11 +63,11 @@ class UTF8BytesJSONEncoder(json.JSONEncoder): """ def encode(self, o, **kwargs): return json.JSONEncoder.encode( - self, _bytes_to_unicode(False, o), **kwargs) + self, bytes_to_unicode(False, o), **kwargs) def iterencode(self, o, **kwargs): return json.JSONEncoder.iterencode( - self, _bytes_to_unicode(False, o), **kwargs) + self, bytes_to_unicode(False, o), **kwargs) class AnyBytesJSONEncoder(json.JSONEncoder): @@ -79,11 +79,11 @@ class AnyBytesJSONEncoder(json.JSONEncoder): """ def encode(self, o, **kwargs): return json.JSONEncoder.encode( - self, _bytes_to_unicode(True, o), **kwargs) + self, bytes_to_unicode(True, o), **kwargs) def iterencode(self, o, **kwargs): return json.JSONEncoder.iterencode( - self, _bytes_to_unicode(True, o), **kwargs) + self, bytes_to_unicode(True, o), **kwargs) def dumps(obj, *args, **kwargs): diff --git a/src/allmydata/util/log.py b/src/allmydata/util/log.py index 509deb6a4..3e80a7fde 100644 --- a/src/allmydata/util/log.py +++ b/src/allmydata/util/log.py @@ -18,6 +18,9 @@ from pyutil import nummedobj from foolscap.logging import log from twisted.python import log as tw_log +from .jsonbytes import bytes_to_unicode + + NOISY = log.NOISY # 10 OPERATIONAL = log.OPERATIONAL # 20 UNUSUAL = log.UNUSUAL # 23 @@ -28,7 +31,8 @@ SCARY = log.SCARY # 35 BAD = log.BAD # 40 -msg = log.msg +def msg(*args, **kwargs): + return log.msg(*args, **bytes_to_unicode(True, kwargs)) # If log.err() happens during a unit test, the unit test should fail. We # accomplish this by sending it to twisted.log too. When a WEIRD/SCARY/BAD @@ -39,7 +43,7 @@ def err(failure=None, _why=None, **kwargs): tw_log.err(failure, _why, **kwargs) if 'level' not in kwargs: kwargs['level'] = log.UNUSUAL - return log.err(failure, _why, **kwargs) + return log.err(failure, _why, **bytes_to_unicode(True, kwargs)) class LogMixin(object): """ I remember a msg id and a facility and pass them to log.msg() """ @@ -57,7 +61,8 @@ class LogMixin(object): if pmsgid is None: pmsgid = self._grandparentmsgid kwargs = {ensure_str(k): v for (k, v) in kwargs.items()} - msgid = log.msg(msg, facility=facility, parent=pmsgid, *args, **kwargs) + msgid = log.msg(msg, facility=facility, parent=pmsgid, *args, + **bytes_to_unicode(True, kwargs)) if self._parentmsgid is None: self._parentmsgid = msgid return msgid From 6475b4433688492c5e4fdbd2a4cf1a7e1733d734 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Jul 2021 16:17:26 -0700 Subject: [PATCH 425/463] Get rid of unnecessary version constraints. --- newsfragments/3390.minor | 0 newsfragments/3404.minor | 0 newsfragments/3723.minor | 0 setup.py | 7 +++---- tox.ini | 3 +-- 5 files changed, 4 insertions(+), 6 deletions(-) create mode 100644 newsfragments/3390.minor create mode 100644 newsfragments/3404.minor create mode 100644 newsfragments/3723.minor diff --git a/newsfragments/3390.minor b/newsfragments/3390.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3404.minor b/newsfragments/3404.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3723.minor b/newsfragments/3723.minor new file mode 100644 index 000000000..e69de29bb diff --git a/setup.py b/setup.py index 1b64c9b09..3433e93f4 100644 --- a/setup.py +++ b/setup.py @@ -114,12 +114,11 @@ install_requires = [ # Pyrsistent 0.17.0 (which we use by way of Eliot) has dropped # Python 2 entirely; stick to the version known to work for us. - # XXX: drop this bound: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3404 - "pyrsistent < 0.17.0", + "pyrsistent < 0.17.0 ; python_version < '3.0'", + "pyrsistent ; python_version > '3.0'", # A great way to define types of values. - # XXX: drop the upper bound: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3390 - "attrs >= 18.2.0, < 20", + "attrs >= 18.2.0", # WebSocket library for twisted and asyncio "autobahn >= 19.5.2", diff --git a/tox.ini b/tox.ini index 7a7427299..c3bddeb43 100644 --- a/tox.ini +++ b/tox.ini @@ -259,8 +259,7 @@ deps = # PyInstaller 4.0 drops Python 2 support. When we finish porting to # Python 3 we can reconsider this constraint. pyinstaller < 4.0 - # 2021.5.13 broke on Windows. See https://github.com/erocarrera/pefile/issues/318 - pefile < 2021.5.13 ; platform_system == "Windows" + pefile ; platform_system == "Windows" # Setting PYTHONHASHSEED to a known value assists with reproducible builds. # See https://pyinstaller.readthedocs.io/en/stable/advanced-topics.html#creating-a-reproducible-build setenv=PYTHONHASHSEED=1 From af49dcb89e736adc1470f0a6be0edd6ae5431a8c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Jul 2021 16:35:06 -0700 Subject: [PATCH 426/463] On Python 2 we don't need to convert byte keys to unicode. --- src/allmydata/util/log.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/log.py b/src/allmydata/util/log.py index 3e80a7fde..b442d30bb 100644 --- a/src/allmydata/util/log.py +++ b/src/allmydata/util/log.py @@ -18,7 +18,14 @@ from pyutil import nummedobj from foolscap.logging import log from twisted.python import log as tw_log -from .jsonbytes import bytes_to_unicode +if PY2: + def bytes_to_unicode(ign, obj): + return obj +else: + # We want to convert bytes keys to Unicode, otherwise JSON serialization + # inside foolscap will fail (for details see + # https://github.com/warner/foolscap/issues/88) + from .jsonbytes import bytes_to_unicode NOISY = log.NOISY # 10 From a612dcff5a7f4abe846e646d6a2bf75465ea0645 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Jul 2021 09:15:30 -0700 Subject: [PATCH 427/463] Run macOS integration tests for Python 3. --- .github/workflows/ci.yml | 4 +--- newsfragments/3746.minor | 0 2 files changed, 1 insertion(+), 3 deletions(-) create mode 100644 newsfragments/3746.minor diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c1ead786..db6343910 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,13 +152,11 @@ jobs: os: - windows-latest - ubuntu-latest + - macos-latest python-version: - 2.7 - 3.6 - 3.9 - include: - - os: macos-latest - python-version: 2.7 steps: diff --git a/newsfragments/3746.minor b/newsfragments/3746.minor new file mode 100644 index 000000000..e69de29bb From 0acf2483eeddfd6484f403ead8c7c255084045f4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Jul 2021 10:04:55 -0700 Subject: [PATCH 428/463] Might need older Tor for Python 3 as well. --- .github/workflows/ci.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db6343910..8c5d43e83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,17 +163,12 @@ jobs: - name: Install Tor [Ubuntu] if: matrix.os == 'ubuntu-latest' run: sudo apt install tor - - - name: Install Tor [macOS, ${{ matrix.python-version }} ] - if: ${{ ( matrix.os == 'macos-latest' ) && ( matrix.python-version != '2.7' ) }} - run: brew install tor # TODO: See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3744. # We have to use an older version of Tor for running integration - # tests on macOS using Python 2.7. We should stop doing this - # when Python 2.7 is dropped. + # tests on macOS. - name: Install Tor [macOS, ${{ matrix.python-version }} ] - if: ${{ ( matrix.os == 'macos-latest' ) && ( matrix.python-version == '2.7' ) }} + if: ${{ matrix.os == 'macos-latest' }} run: | brew extract --version 0.4.5.8 tor homebrew/cask brew install tor@0.4.5.8 From d51f1f1c5aaf7ffbb3169e5de4a320cfb94dba2b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 7 Jul 2021 14:47:39 -0700 Subject: [PATCH 429/463] Try to get faster CI by omitting older Python versions on macOS. --- .github/workflows/ci.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c5d43e83..e95d2ee88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,6 @@ jobs: matrix: os: - windows-latest - - macos-latest - ubuntu-latest python-version: - 2.7 @@ -27,6 +26,12 @@ jobs: - 3.7 - 3.8 - 3.9 + include: + # On macOS don't bother with 3.6-3.8, just to get faster builds. + - os: macos-latest + python-version: 2.7 + - os: macos-latest + python-version: 3.9 steps: # See https://github.com/actions/checkout. A fetch-depth of 0 @@ -152,11 +157,16 @@ jobs: os: - windows-latest - ubuntu-latest - - macos-latest python-version: - 2.7 - 3.6 - 3.9 + include: + # On macOS don't bother with 3.6, just to get faster builds. + - os: macos-latest + python-version: 2.7 + - os: macos-latest + python-version: 3.9 steps: From d56c09bd952d2f5ac38f394b4d0232ea7c9b6d28 Mon Sep 17 00:00:00 2001 From: Yash Nayani Date: Thu, 8 Jul 2021 12:46:00 +0530 Subject: [PATCH 430/463] Installation Guide Revamp --- docs/INSTALL.rst | 343 ------------------ .../install-on-desert-island.rst} | 6 +- docs/Installation/install-on-linux.rst | 75 ++++ docs/Installation/install-on-windows.rst | 45 +++ docs/Installation/install-tahoe.rst | 68 ++++ docs/OS-X.rst | 23 -- docs/{about.rst => about-tahoe.rst} | 2 +- docs/index.rst | 10 +- docs/release-checklist.rst | 2 +- docs/running.rst | 2 +- docs/windows.rst | 83 ----- newsfragments/3747.documentation | 1 + 12 files changed, 200 insertions(+), 460 deletions(-) delete mode 100644 docs/INSTALL.rst rename docs/{desert-island.rst => Installation/install-on-desert-island.rst} (97%) create mode 100644 docs/Installation/install-on-linux.rst create mode 100644 docs/Installation/install-on-windows.rst create mode 100644 docs/Installation/install-tahoe.rst delete mode 100644 docs/OS-X.rst rename docs/{about.rst => about-tahoe.rst} (98%) delete mode 100644 docs/windows.rst create mode 100644 newsfragments/3747.documentation diff --git a/docs/INSTALL.rst b/docs/INSTALL.rst deleted file mode 100644 index 71add0cf3..000000000 --- a/docs/INSTALL.rst +++ /dev/null @@ -1,343 +0,0 @@ -.. -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- - -.. - note: if you aren't reading the rendered form of these docs at - http://tahoe-lafs.readthedocs.io/en/latest/ , then be aware that any - ":doc:" links refer to other files in this docs/ directory - -********************* -Installing Tahoe-LAFS -********************* - -Welcome to `the Tahoe-LAFS project`_, a secure, decentralized, fault-tolerant -storage system. See :doc:`about` for an overview of the architecture and -security properties of the system. - -This procedure should work on Windows, Mac, illumos (previously OpenSolaris), -and too many flavors of Linux and of BSD to list. - -.. _the Tahoe-LAFS project: https://tahoe-lafs.org - -First: In Case Of Trouble -========================= - -In some cases these instructions may fail due to peculiarities of your -platform. - -If the following instructions don't Just Work without any further effort on -your part, then please write to `the tahoe-dev mailing list`_ where friendly -hackers will help you out. - -.. _the tahoe-dev mailing list: https://tahoe-lafs.org/cgi-bin/mailman/listinfo/tahoe-dev - -Pre-Packaged Versions -===================== - -You may not need to build Tahoe at all. - -If you are on Windows, please see :doc:`windows` for platform-specific -instructions. - -If you are on a Mac, you can either follow these instructions, or use the -pre-packaged bundle described in :doc:`OS-X`. - -Many Linux distributions include Tahoe-LAFS packages. Debian and Ubuntu users -can ``apt-get install tahoe-lafs``. See `OSPackages`_ for other -platforms. - -.. _OSPackages: https://tahoe-lafs.org/trac/tahoe-lafs/wiki/OSPackages - - -Preliminaries -============= - -If you don't use a pre-packaged copy of Tahoe, you can build it yourself. -You'll need Python2.7, pip, and virtualenv. -Tahoe-LAFS depends on some libraries which require a C compiler to build. -However, for many platforms, PyPI hosts already-built packages of libraries. - -If there is no already-built package for your platform, -you will need a C compiler, -the Python development headers, -and some libraries (libffi-dev and libssl-dev). - -On a modern Debian/Ubuntu-derived distribution, this command will get you -everything you need:: - - apt-get install build-essential python-dev libffi-dev libssl-dev libyaml-dev python-virtualenv - -On OS-X, install pip and virtualenv as described below. If you want to -compile the dependencies yourself, you'll also need to install -Xcode and its command-line tools. - -**Note** that Tahoe-LAFS depends on `openssl 1.1.1c` or greater. - -Python 2.7 ----------- - -Check if you already have an adequate version of Python installed by running -``python -V``. The latest version of Python v2.7 is recommended, which is -2.7.11 as of this writing. Python v2.6.x and v3 do not work. On Windows, we -recommend the use of native Python v2.7, not Cygwin Python. If you don't have -one of these versions of Python installed, `download`_ and install the latest -version of Python v2.7. Make sure that the path to the installation directory -has no spaces in it (e.g. on Windows, do not install Python in the "Program -Files" directory):: - - % python --version - Python 2.7.11 - -.. _download: https://www.python.org/downloads/ - -pip ---- - -Many Python installations already include ``pip``, but in case yours does -not, get it with the `pip install instructions`_:: - - % pip --version - pip 10.0.1 from ... (python 2.7) - -.. _pip install instructions: https://pip.pypa.io/en/stable/installing/ - -virtualenv ----------- - -If you do not have an OS-provided copy of ``virtualenv``, install it with the -instructions from the `virtualenv documentation`_:: - - - % virtualenv --version - 15.1.0 - -.. _virtualenv documentation: https://virtualenv.pypa.io/en/latest/installation.html - -C compiler and libraries ------------------------- - -Except on OS-X, where the Tahoe project hosts pre-compiled wheels for all -dependencies, you will need several C libraries installed before you can -build. You will also need the Python development headers, and a C compiler -(your python installation should know how to find these). - -On Debian/Ubuntu-derived systems, the necessary packages are ``python-dev``, -``libffi-dev``, and ``libssl-dev``, and can be installed with ``apt-get``. On -RPM-based system (like Fedora) these may be named ``python-devel``, etc, -instead, and cam be installed with ``yum`` or ``rpm``. - -**Note** that Tahoe-LAFS depends on `openssl 1.1.1c` or greater. - - -Install the Latest Tahoe-LAFS Release -===================================== - -We recommend creating a fresh virtualenv for your Tahoe-LAFS install, to -isolate it from any python packages that are already installed (and to -isolate the rest of your system from Tahoe's dependencies). - -This example uses a virtualenv named ``venv``, but you can call it anything -you like. Many people prefer to keep all their virtualenvs in one place, like -``~/.local/venvs/`` or ``~/venvs/``. - -It's usually a good idea to upgrade the virtualenv's ``pip`` and -``setuptools`` to their latest versions, with ``venv/bin/pip install -U pip -setuptools``. Many operating systems have an older version of ``virtualenv``, -which then includes older versions of pip and setuptools. Upgrading is easy, -and only affects the virtualenv: not the rest of your computer. - -Then use the virtualenv's ``pip`` to install the latest Tahoe-LAFS release -from PyPI with ``venv/bin/pip install tahoe-lafs``. After installation, run -``venv/bin/tahoe --version`` to confirm the install was successful:: - - % virtualenv venv - New python executable in ~/venv/bin/python2.7 - Installing setuptools, pip, wheel...done. - - % venv/bin/pip install -U pip setuptools - Downloading/unpacking pip from https://pypi.python.org/... - ... - Successfully installed pip setuptools - - % venv/bin/pip install tahoe-lafs - Collecting tahoe-lafs - ... - Installing collected packages: ... - Successfully installed ... - - % venv/bin/tahoe --version - tahoe-lafs: 1.15.1 - foolscap: ... - - % - -Install From a Source Tarball ------------------------------ - -You can also install directly from the source tarball URL. To verify -signatures, first see verifying_signatures_ and replace the URL in the -following instructions with the local filename. - - % virtualenv venv - New python executable in ~/venv/bin/python2.7 - Installing setuptools, pip, wheel...done. - - % venv/bin/pip install https://tahoe-lafs.org/downloads/tahoe-lafs-1.15.1.tar.gz - Collecting https://tahoe-lafs.org/downloads/tahoe-lafs-1.15.1.tar.gz - ... - Installing collected packages: ... - Successfully installed ... - - % venv/bin/tahoe --version - tahoe-lafs: 1.15.1 - ... - -.. _verifying_signatures: - -Verifying Signatures --------------------- - -First download the source tarball and then any signatures. There are several -developers who are able to produce signatures for a release. A release may -have multiple signatures. All should be valid and you should confirm at least -one of them (ideally, confirm all). - -This statement, signed by the existing Tahoe release-signing key, attests to -those developers authorized to sign a Tahoe release: - -.. include:: developer-release-signatures - :code: - -Signatures are made available beside the release. So for example, a release -like ``https://tahoe-lafs.org/downloads/tahoe-lafs-1.16.0.tar.gz`` might -have signatures ``tahoe-lafs-1.16.0.tar.gz.meejah.asc`` and -``tahoe-lafs-1.16.0.tar.gz.warner.asc``. - -To verify the signatures using GnuPG:: - - % gpg --verify tahoe-lafs-1.16.0.tar.gz.meejah.asc tahoe-lafs-1.16.0.tar.gz - gpg: Signature made XXX - gpg: using RSA key 9D5A2BD5688ECB889DEBCD3FC2602803128069A7 - gpg: Good signature from "meejah " [full] - % gpg --verify tahoe-lafs-1.16.0.tar.gz.warner.asc tahoe-lafs-1.16.0.tar.gz - gpg: Signature made XXX - gpg: using RSA key 967EFE06699872411A77DF36D43B4C9C73225AAF - gpg: Good signature from "Brian Warner " [full] - - - -Extras ------- - -Tahoe-LAFS provides some functionality only when explicitly requested at installation time. -It does this using the "extras" feature of setuptools. -You can request these extra features when running the ``pip install`` command like this:: - - % venv/bin/pip install tahoe-lafs[tor] - -This example enables support for listening and connecting using Tor. -The Tahoe-LAFS documentation for specific features which require an explicit install-time step will mention the "extra" that must be requested. - -Hacking On Tahoe-LAFS ---------------------- - -To modify the Tahoe source code, you should get a git checkout, and install -with the ``--editable`` flag. You should also use the ``[test]`` extra to get -the additional libraries needed to run the unit tests:: - - % git clone https://github.com/tahoe-lafs/tahoe-lafs.git - - % cd tahoe-lafs - - % virtualenv venv - - % venv/bin/pip install --editable .[test] - Obtaining file::~/tahoe-lafs - ... - Successfully installed ... - - % venv/bin/tahoe --version - tahoe-lafs: 1.15.1 - ... - -This way, you won't have to re-run the ``pip install`` step each time you -modify the source code. - -Running the ``tahoe`` executable -================================ - -The rest of the Tahoe-LAFS documentation assumes that you can run the -``tahoe`` executable that you just created. You have four basic options: - -* Use the full path each time (e.g. ``~/venv/bin/tahoe``). -* "`Activate`_" the virtualenv with ``. venv/bin/activate``, to get a - subshell with a ``$PATH`` that includes the ``venv/bin/`` directory, then - you can just run ``tahoe``. -* Change your ``$PATH`` to include the ``venv/bin/`` directory, so you can - just run ``tahoe``. -* Symlink from ``~/bin/tahoe`` to the ``tahoe`` executable. Since ``~/bin`` - is typically in your ``$PATH`` (at least if it exists when you log in), - this will let you just run ``tahoe``. - -You might also find the `pipsi`_ tool convenient: ``pipsi install -tahoe-lafs`` will create a new virtualenv, install tahoe into it, then -symlink just the executable (into ``~/.local/bin/tahoe``). Then either add -``~/.local/bin/`` to your ``$PATH``, or make one last symlink into -``~/bin/tahoe``. - -.. _Activate: https://virtualenv.pypa.io/en/latest/userguide.html#activate-script -.. _pipsi: https://pypi.python.org/pypi/pipsi/0.9 - -Running the Self-Tests -====================== - -To run the self-tests from a source tree, you'll need ``tox`` installed. On a -Debian/Ubuntu system, use ``apt-get install tox``. You can also install it -into your tahoe-specific virtualenv with ``pip install tox``. - -Then just run ``tox``. This will create a new fresh virtualenv, install Tahoe -(from the source tree, including any changes you have made) and all its -dependencies (including testing-only dependencies) into the virtualenv, then -run the unit tests. This ensures that the tests are repeatable and match the -results of other users, unaffected by any other Python packages installed on -your machine. On a modern computer this will take 5-10 minutes, and should -result in a "all tests passed" mesage:: - - % tox - GLOB sdist-make: ~/tahoe-lafs/setup.py - py27 recreate: ~/tahoe-lafs/.tox/py27 - py27 inst: ~/tahoe-lafs/.tox/dist/tahoe-lafs-1.15.1.zip - py27 runtests: commands[0] | tahoe --version - py27 runtests: commands[1] | trial --rterrors allmydata - allmydata.test.test_auth - AccountFileCheckerKeyTests - test_authenticated ... [OK] - test_missing_signature ... [OK] - ... - Ran 1186 tests in 423.179s - - PASSED (skips=7, expectedFailures=3, successes=1176) - __________________________ summary ___________________________________ - py27: commands succeeded - congratulations :) - -Common Problems -=============== - -If you see an error like ``fatal error: Python.h: No such file or directory`` -while compiling the dependencies, you need the Python development headers. If -you are on a Debian or Ubuntu system, you can install them with ``sudo -apt-get install python-dev``. On RedHat/Fedora, install ``python-devel``. - -Similar errors about ``openssl/crypto.h`` indicate that you are missing the -OpenSSL development headers (``libssl-dev``). Likewise ``ffi.h`` means you -need ``libffi-dev``. - -**Note** that Tahoe-LAFS depends on `openssl 1.1.1c` or greater. - - -Using Tahoe-LAFS -================ - -Now you are ready to deploy a decentralized filesystem. You will use the -``tahoe`` executable to create, configure, and launch your Tahoe-LAFS nodes. -See :doc:`running` for instructions on how to do that. diff --git a/docs/desert-island.rst b/docs/Installation/install-on-desert-island.rst similarity index 97% rename from docs/desert-island.rst rename to docs/Installation/install-on-desert-island.rst index 33db243d6..fbc59ddbd 100644 --- a/docs/desert-island.rst +++ b/docs/Installation/install-on-desert-island.rst @@ -1,6 +1,6 @@ -****************************************** -How To Build Tahoe-LAFS On A Desert Island -****************************************** +*************************************** +Building Tahoe-LAFS On A Desert Island +*************************************** (or an airplane, or anywhere else without internet connectivity) diff --git a/docs/Installation/install-on-linux.rst b/docs/Installation/install-on-linux.rst new file mode 100644 index 000000000..79fa1c066 --- /dev/null +++ b/docs/Installation/install-on-linux.rst @@ -0,0 +1,75 @@ +**************************** +Building Tahoe-LAFS on Linux +**************************** + +Tahoe-LAFS has made packages available for installing on many linux and BSD distributions. +Debian and Ubuntu users can use ``apt-get install tahoe-lafs``. +If you are working on a Linux distribution which does not have Tahoe-LAFS or are looking to hack on the source code, you can build Tahoe-LAFS yourself: + +Prerequisites +============= + +Make sure the following are installed: + +* **Python 3's latest version**: Check for the version by running ``python --version``. +* **pip**: Most python installations already include ``pip``. However, if your installation does not, see `pip installation `_. +* **virtualenv**: Use ``pip`` to install virtualenv:: + + pip install --user virtualenv + +* **C compiler and libraries**: + + * ``python-dev``: Python development headers. + * ``libffi-dev``: Foreign Functions Interface library. + * ``libssl-dev``: SSL library, Tahoe-LAFS needs OpenSSL version 1.1.1c or greater. + + .. note:: + If you are working on Debian or Ubuntu, you can install the necessary libraries using ``apt-get``:: + + apt-get install python-dev libffi-dev libssl-dev + + On an RPM-based system such as Fedora, you can install the necessary libraries using ``yum`` or ``rpm``. However, the packages may be named differently. + +Install the Latest Tahoe-LAFS Release +===================================== + +If you are looking to hack on the source code or run pre-release code, we recommend you install Tahoe-LAFS directly from source by creating a ``virtualenv`` instance: + +1. Clone the Tahoe-LAFS repository:: + + git clone https://github.com/tahoe-lafs/tahoe-lafs.git + +2. Move into the tahoe-lafs directory:: + + cd tahoe-lafs + +3. Create a fresh virtualenv for your Tahoe-LAFS install:: + + virtualenv venv + +.. note:: + venv is the name of the virtual environment in this example. Use any name for your environment. + +4. Upgrade ``pip`` and ``setuptools`` on the newly created virtual environment:: + + venv/bin/pip install -U pip setuptools + +5. If you'd like to modify the Tahoe source code, you need to install Tahoe-LAFS with the ``--editable`` flag with the ``test`` extra:: + + venv/bin/pip install --editable .[test] + +.. note:: + Tahoe-LAFS provides extra functionality when requested explicitly at installation using the "extras" feature of setuptools. To learn more about the extras which Tahoe supports, see Tahoe extras. + +6. Verify installation by checking for the version:: + + venv/bin/tahoe --version + +If you do not want to use the full path, i.e., ``venv/bin/tahoe`` everytime you want to run tahoe, you can activate the ``virtualenv``:: + + . venv/bin/activate + + This will generate a subshell with a ``$PATH`` that includes the ``venv/bin/`` directory. + + + diff --git a/docs/Installation/install-on-windows.rst b/docs/Installation/install-on-windows.rst new file mode 100644 index 000000000..5f836df06 --- /dev/null +++ b/docs/Installation/install-on-windows.rst @@ -0,0 +1,45 @@ +****************************** +Building Tahoe-LAFS on Windows +****************************** + +If you are looking to hack on the source code or run pre-release code, we recommend you create a virtualenv instance and install Tahoe-LAFS into that: + + +1. Make sure you have Powershell installed. See `PowerShell installation `_. + +2. Install the latest version of Python 3. Download the .exe file at the `python website `_. + +3. Open the installer by double-clicking it. Select the **Add Python to PATH** check-box, then click **Install Now**. + +4. Start PowerShell and enter the following command to verify python installation:: + + python --version + +5. Use ``pip`` to install ``virtualenv``:: + + pip install --user virtualenv + +6. Create a fresh virtualenv for your Tahoe-LAFS install using the following command:: + + virtualenv venv + + .. note:: + venv is the name of the virtual environment in this example. Use any name for your environment. + +7. Use pip to install Tahoe-LAFS in the virtualenv instance:: + + venv\Scripts\pip install tahoe-lafs + +6. Verify installation by checking for the version:: + + venv\Scripts\tahoe --version + +If you do not want to use the full path, i.e. ``venv\Scripts\tahoe`` everytime you want to run tahoe, you can: + +* Activate the virtualenv:: + + . venv\Scripts\activate + + This will generate a subshell with a ``$PATH`` that includes the ``venv\Scripts\`` directory. + +* Change your ``$PATH`` to include the ``venv\Scripts`` directory. \ No newline at end of file diff --git a/docs/Installation/install-tahoe.rst b/docs/Installation/install-tahoe.rst new file mode 100644 index 000000000..c8b0b521e --- /dev/null +++ b/docs/Installation/install-tahoe.rst @@ -0,0 +1,68 @@ +.. -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- + +.. + note: if you aren't reading the rendered form of these docs at + http://tahoe-lafs.readthedocs.io/en/latest/ , then be aware that any + ":doc:" links refer to other files in this docs/ directory + +********************* +Installing Tahoe-LAFS +********************* + +`Tahoe-LAFS`_ is a secure, decentralized, and fault-tolerant storage system. +To see an overview of the architecture and security properties, see :doc:`Welcome to Tahoe LAFS! <../about-tahoe>` + +Tahoe-LAFS can be installed and used on any of the following operating systems. + +.. _Tahoe-LAFS: https://tahoe-lafs.org + +Microsoft Windows +================= + +To install Tahoe-LAFS on Windows: + +1. Make sure you have Powershell installed. See `PowerShell installation `_. + +2. Install the latest version of Python 3. Download the .exe file at the `python website `_. + +3. Open the installer by double-clicking it. Select the **Add Python to PATH** check-box, then click **Install Now**. + +4. Start PowerShell and enter the following command to verify python installation:: + + python --version + +5. Enter the following command to install Tahoe-LAFS:: + + pip install tahoe-lafs + +6. Verify installation by checking for the version:: + + tahoe --version + +If you want to hack on Tahoe's source code, you can install Tahoe in a ``virtualenv`` on your Windows Machine. To learn more, see :doc:`install-on-windows`. + +Linux, BSD, or MacOS +==================== + +Tahoe-LAFS can be installed on MacOS, many Linux and BSD distributions. If you are using Ubuntu or Debian, run the following command to install Tahoe-LAFS:: + + apt-get install tahoe-lafs + +If you are working on MacOS or a Linux distribution which does not have Tahoe-LAFS packages, you can build it yourself: + +1. Make sure the following are installed: + + * **Python 3's latest version**: Check for the version by running ``python --version``. + * **pip**: Most python installations already include `pip`. However, if your installation does not, see `pip installation `_. + +2. Install Tahoe-LAFS using pip:: + + pip install tahoe-lafs + +3. Verify installation by checking for the version:: + + tahoe --version + +If you are looking to hack on the source code or run pre-release code, we recommend you install Tahoe-LAFS on a `virtualenv` instance. To learn more, see :doc:`install-on-linux`. + +You can always write to the `tahoe-dev mailing list `_ or chat on the `Libera.chat IRC `_ if you are not able to get Tahoe-LAFS up and running on your deployment. diff --git a/docs/OS-X.rst b/docs/OS-X.rst deleted file mode 100644 index 29ac0474f..000000000 --- a/docs/OS-X.rst +++ /dev/null @@ -1,23 +0,0 @@ -============== -OS-X Packaging -============== - -Pre-built Tahoe-LAFS ".pkg" installers for OS-X are generated with each -source-code commit. These installers offer an easy way to get Tahoe and all -its dependencies installed on your Mac. They do not yet provide a -double-clickable application: after installation, you will have a "tahoe" -command-line tool, which you can use from a shell (a Terminal window) just as -if you'd installed from source. - -Installers are available from this directory: - - https://tahoe-lafs.org/source/tahoe-lafs/tarballs/OS-X-packages/ - -Download the latest .pkg file to your computer and double-click on it. This -will install to /Applications/tahoe.app, however the app icon there is not -how you use Tahoe (launching it will get you a dialog box with a reminder to -use Terminal). ``/Applications/tahoe.app/bin/tahoe`` is the executable. The -next shell you start ought to have that directory in your $PATH (thanks to a -file in ``/etc/paths.d/``), unless your ``.profile`` overrides it. - -Tahoe-LAFS is also easy to install with pip, as described in the README. diff --git a/docs/about.rst b/docs/about-tahoe.rst similarity index 98% rename from docs/about.rst rename to docs/about-tahoe.rst index 120abb079..348e148dd 100644 --- a/docs/about.rst +++ b/docs/about-tahoe.rst @@ -127,7 +127,7 @@ For more technical detail, please see the `the doc page`_ on the Wiki. Get Started =========== -To use Tahoe-LAFS, please see :doc:`INSTALL`. +To use Tahoe-LAFS, please see :doc:`Installing Tahoe-LAFS <../Installation/install-tahoe>`. License ======= diff --git a/docs/index.rst b/docs/index.rst index 6ad09d1bf..16067597a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,8 +10,11 @@ Contents: .. toctree:: :maxdepth: 2 - about - INSTALL + about-tahoe + Installation/install-tahoe + Installation/install-on-windows + Installation/install-on-linux + Installation/install-on-desert-island running magic-wormhole-invites configuration @@ -50,10 +53,7 @@ Contents: logging stats - desert-island debian - windows - OS-X build/build-pyOpenSSL specifications/index diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 48a65ff0a..a5761c1c7 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -70,7 +70,7 @@ Create Branch and Apply Updates - commit it - update "docs/known_issues.rst" if appropriate -- update "docs/INSTALL.rst" references to the new release +- update "docs/Installation/install-tahoe.rst" references to the new release - Push the branch to github - Create a (draft) PR; this should trigger CI (note that github doesn't let you create a PR without some changes on the branch so diff --git a/docs/running.rst b/docs/running.rst index 898afa10b..a53f5d9e2 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -10,7 +10,7 @@ Introduction This is how to run a Tahoe-LAFS client or a complete Tahoe-LAFS grid. First you have to install the Tahoe-LAFS software, as documented in -:doc:`INSTALL`. +:doc:`Installing Tahoe-LAFS <../Installation/install-tahoe>`. The ``tahoe`` program in your virtualenv's ``bin`` directory is used to create, start, and stop nodes. Each node lives in a separate base diff --git a/docs/windows.rst b/docs/windows.rst deleted file mode 100644 index 1f69ac743..000000000 --- a/docs/windows.rst +++ /dev/null @@ -1,83 +0,0 @@ -Building Tahoe-LAFS on Windows -============================== - -You'll need ``python``, ``pip``, and ``virtualenv``. But you won't need a -compiler. - -Preliminaries -------------- - -1: Install Python-2.7.11 . Use the "Windows x86-64 MSI installer" at -https://www.python.org/downloads/release/python-2711/ - -2: That should install ``pip``, but if it doesn't, look at -https://pip.pypa.io/en/stable/installing/ for installation instructions. - -3: Install ``virtualenv`` with -https://virtualenv.pypa.io/en/latest/installation.html - -Installation ------------- - -1: Start a CLI shell (e.g. PowerShell) - -2: Create a new virtualenv. Everything specific to Tahoe will go into this. -You can use whatever name you like for the virtualenv, but example uses -"venv":: - - PS C:\Users\me> virtualenv venv - New python executable in C:\Users\me\venv\Scripts\python.exe - Installing setuptools, pip, wheel...done. - > - -3: Use the virtualenv's ``pip`` to install the latest release of Tahoe-LAFS -into this virtualenv:: - - PS C:\Users\me> venv\Scripts\pip install tahoe-lafs - Collecting tahoe-lafs - ... - Installing collected packages: ... - Successfully installed ... - > - -4: Verify that Tahoe was installed correctly by running ``tahoe --version``, -using the ``tahoe`` from the virtualenv's Scripts directory:: - - PS C:\Users\me> venv\Scripts\tahoe --version - tahoe-lafs: 1.11 - foolscap: ... - -Running Tahoe-LAFS ------------------- - -The rest of the documentation assumes you can run the ``tahoe`` executable -just as you did in step 4 above. If you want to type just ``tahoe`` instead -of ``venv\Scripts\tahoe``, you can either "`activate`_" the virtualenv (by -running ``venv\Scripts\activate``, or you can add the Scripts directory to -your ``%PATH%`` environment variable. - -Now use the docs in :doc:`running` to learn how to configure your first -Tahoe node. - -.. _activate: https://virtualenv.pypa.io/en/latest/userguide.html#activate-script - -Installing A Different Version ------------------------------- - -The ``pip install tahoe-lafs`` command above will install the latest release -(from PyPI). If instead, you want to install from a git checkout, then run -the following command (using pip from the virtualenv, from the root of your -git checkout):: - - $ venv\Scripts\pip install . - -If you're planning to hack on the source code, you might want to add -``--editable`` so you won't have to re-install each time you make a change. - -Dependencies ------------- - -Tahoe-LAFS depends upon several packages that use compiled C code (such as zfec). -This code must be built separately for each platform (Windows, OS-X, and different flavors of Linux). -Fortunately, this is now done by upstream packages for most platforms. -The result is that a C compiler is usually not required to install Tahoe-LAFS. diff --git a/newsfragments/3747.documentation b/newsfragments/3747.documentation new file mode 100644 index 000000000..a2559a6a0 --- /dev/null +++ b/newsfragments/3747.documentation @@ -0,0 +1 @@ +Rewriting the installation guide for Tahoe-LAFS. From 7396130c0a6cc3b9fa05765bf885a8d103915360 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 27 Jul 2021 14:20:01 -0400 Subject: [PATCH 431/463] Integration test for I2P. --- integration/test_i2p.py | 239 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 integration/test_i2p.py diff --git a/integration/test_i2p.py b/integration/test_i2p.py new file mode 100644 index 000000000..e87ba28e2 --- /dev/null +++ b/integration/test_i2p.py @@ -0,0 +1,239 @@ +""" +Integration tests for I2P support. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +import sys +from os.path import join, exists +from os import mkdir +from time import sleep + +if PY2: + def which(path): + # This will result in skipping I2P tests on Python 2. Oh well. + return None +else: + from shutil import which + +from eliot import log_call + +import pytest +import pytest_twisted + +from . import util + +from twisted.python.filepath import ( + FilePath, +) +from twisted.internet.error import ProcessExitedAlready + +from allmydata.test.common import ( + write_introducer, +) + +if which("docker") is None: + pytest.skip('Skipping I2P tests since Docker is unavailable', allow_module_level=True) + + +@pytest.fixture(scope="session") +def i2p_network(reactor, temp_dir, request): + """Fixture to start up local i2pd.""" + proto = util._MagicTextProtocol("ephemeral keys") + reactor.spawnProcess( + proto, + which("docker"), + ( + "docker", "run", "-p", "7656:7656", "purplei2p/i2pd", + # Bad URL for reseeds, so it can't talk to other routers. + "--reseed.urls", "http://localhost:1/", + ), + ) + pytest_twisted.blockon(proto.magic_seen) + + def cleanup(): + try: + proto.transport.signalProcess("INT") + util.block_with_timeout(proto.exited, reactor) + except ProcessExitedAlready: + pass + request.addfinalizer(cleanup) + + +@pytest.fixture(scope='session') +@log_call( + action_type=u"integration:i2p:introducer", + include_args=["temp_dir", "flog_gatherer"], + include_result=False, +) +def i2p_introducer(reactor, temp_dir, flog_gatherer, request): + config = ''' +[node] +nickname = introducer_i2p +web.port = 4561 +log_gatherer.furl = {log_furl} +'''.format(log_furl=flog_gatherer) + + intro_dir = join(temp_dir, 'introducer_i2p') + print("making introducer", intro_dir) + + if not exists(intro_dir): + mkdir(intro_dir) + done_proto = util._ProcessExitedProtocol() + util._tahoe_runner_optional_coverage( + done_proto, + reactor, + request, + ( + 'create-introducer', + '--listen=i2p', + intro_dir, + ), + ) + pytest_twisted.blockon(done_proto.done) + + # over-write the config file with our stuff + with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: + f.write(config) + + # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old + # "start" command. + protocol = util._MagicTextProtocol('introducer running') + transport = util._tahoe_runner_optional_coverage( + protocol, + reactor, + request, + ( + 'run', + intro_dir, + ), + ) + + def cleanup(): + try: + transport.signalProcess('TERM') + util.block_with_timeout(protocol.exited, reactor) + except ProcessExitedAlready: + pass + request.addfinalizer(cleanup) + + pytest_twisted.blockon(protocol.magic_seen) + return transport + + +@pytest.fixture(scope='session') +def i2p_introducer_furl(i2p_introducer, temp_dir): + furl_fname = join(temp_dir, 'introducer_i2p', 'private', 'introducer.furl') + while not exists(furl_fname): + print("Don't see {} yet".format(furl_fname)) + sleep(.1) + furl = open(furl_fname, 'r').read() + return furl + + +@pytest_twisted.inlineCallbacks +def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): + yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) + yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) + # ensure both nodes are connected to "a grid" by uploading + # something via carol, and retrieve it using dave. + gold_path = join(temp_dir, "gold") + with open(gold_path, "w") as f: + f.write( + "The object-capability model is a computer security model. A " + "capability describes a transferable right to perform one (or " + "more) operations on a given object." + ) + # XXX could use treq or similar to POST these to their respective + # WUIs instead ... + + proto = util._CollectOutputProtocol() + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-b', '-m', 'allmydata.scripts.runner', + '-d', join(temp_dir, 'carol'), + 'put', gold_path, + ) + ) + yield proto.done + cap = proto.output.getvalue().strip().split()[-1] + print("TEH CAP!", cap) + + proto = util._CollectOutputProtocol(capture_stderr=False) + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-b', '-m', 'allmydata.scripts.runner', + '-d', join(temp_dir, 'dave'), + 'get', cap, + ) + ) + yield proto.done + + dave_got = proto.output.getvalue().strip() + assert dave_got == open(gold_path, 'rb').read().strip() + + +@pytest_twisted.inlineCallbacks +def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, i2p_network, introducer_furl): + node_dir = FilePath(temp_dir).child(name) + web_port = "tcp:{}:interface=localhost".format(control_port + 2000) + + print("creating", node_dir.path) + node_dir.makedirs() + proto = util._DumpOutputProtocol(None) + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-b', '-m', 'allmydata.scripts.runner', + 'create-node', + '--nickname', name, + '--introducer', introducer_furl, + '--hide-ip', + '--listen', 'i2p', + node_dir.path, + ) + ) + yield proto.done + + + # Which services should this client connect to? + write_introducer(node_dir, "default", introducer_furl) + with node_dir.child('tahoe.cfg').open('w') as f: + node_config = ''' +[node] +nickname = %(name)s +web.port = %(web_port)s +web.static = public_html +log_gatherer.furl = %(log_furl)s + +[i2p] +enabled = true + +[client] +shares.needed = 1 +shares.happy = 1 +shares.total = 2 + +''' % { + 'name': name, + 'web_port': web_port, + 'log_furl': flog_gatherer, +} + node_config = node_config.encode("utf-8") + f.write(node_config) + + print("running") + yield util._run_node(reactor, node_dir.path, request, None) + print("okay, launched") From dbd2e7f9730f47184213bdd926e639e99dea0f0b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 27 Jul 2021 14:20:26 -0400 Subject: [PATCH 432/463] News file. --- newsfragments/3743.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3743.minor diff --git a/newsfragments/3743.minor b/newsfragments/3743.minor new file mode 100644 index 000000000..e69de29bb From ddca3e9ab8139518e611ff3f7dd7e84e399c1a0e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 27 Jul 2021 14:21:40 -0400 Subject: [PATCH 433/463] At this point all integration tests are expected to pass on Python 3. --- src/allmydata/util/_python3.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 8025582f2..f65e0aaf9 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -17,21 +17,6 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -PORTED_INTEGRATION_TESTS = [ - "integration.test_aaa_aardvark", - "integration.test_servers_of_happiness", - "integration.test_sftp", - "integration.test_streaming_logs", - "integration.test_tor", - "integration.test_web", -] - -PORTED_INTEGRATION_MODULES = [ - "integration", - "integration.conftest", - "integration.util", -] - # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ "allmydata", From 982ac3cc33aad62eb647738245a3b9e52882c26f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 29 Jul 2021 10:02:02 -0400 Subject: [PATCH 434/463] Timeout if i2pd never starts. --- integration/test_i2p.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index e87ba28e2..3f2133f6f 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -56,7 +56,6 @@ def i2p_network(reactor, temp_dir, request): "--reseed.urls", "http://localhost:1/", ), ) - pytest_twisted.blockon(proto.magic_seen) def cleanup(): try: @@ -66,6 +65,8 @@ def i2p_network(reactor, temp_dir, request): pass request.addfinalizer(cleanup) + util.block_with_timeout(proto.magic_seen, reactor, timeout=30) + @pytest.fixture(scope='session') @log_call( From 97522641d669662971ce29fb7678a4b6eec50d8d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Jul 2021 11:06:28 -0400 Subject: [PATCH 435/463] Skip on Windows. --- integration/test_i2p.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 3f2133f6f..4f212f5fb 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -41,6 +41,10 @@ from allmydata.test.common import ( if which("docker") is None: pytest.skip('Skipping I2P tests since Docker is unavailable', allow_module_level=True) +# Docker on Windows machines sometimes expects Windows-y Docker images, so just +# don't bother. +if sys.platform.startswith('win'): + pytest.skip('Skipping I2P tests on Windows', allow_module_level=True) @pytest.fixture(scope="session") From ce2363e3ded009bc3604a88af6445903dd45ea81 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Jul 2021 11:08:56 -0400 Subject: [PATCH 436/463] More aggressively shut down i2pd and other i2p-related processes. --- integration/test_i2p.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 4f212f5fb..1aa7a5d99 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -47,7 +47,7 @@ if sys.platform.startswith('win'): pytest.skip('Skipping I2P tests on Windows', allow_module_level=True) -@pytest.fixture(scope="session") +@pytest.fixture def i2p_network(reactor, temp_dir, request): """Fixture to start up local i2pd.""" proto = util._MagicTextProtocol("ephemeral keys") @@ -63,7 +63,7 @@ def i2p_network(reactor, temp_dir, request): def cleanup(): try: - proto.transport.signalProcess("INT") + proto.transport.signalProcess("KILL") util.block_with_timeout(proto.exited, reactor) except ProcessExitedAlready: pass @@ -72,7 +72,7 @@ def i2p_network(reactor, temp_dir, request): util.block_with_timeout(proto.magic_seen, reactor, timeout=30) -@pytest.fixture(scope='session') +@pytest.fixture @log_call( action_type=u"integration:i2p:introducer", include_args=["temp_dir", "flog_gatherer"], @@ -133,7 +133,7 @@ log_gatherer.furl = {log_furl} return transport -@pytest.fixture(scope='session') +@pytest.fixture def i2p_introducer_furl(i2p_introducer, temp_dir): furl_fname = join(temp_dir, 'introducer_i2p', 'private', 'introducer.furl') while not exists(furl_fname): From d060af641a553f9feda6124d5beecb05a7115abb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Jul 2021 11:13:57 -0400 Subject: [PATCH 437/463] Python 3 port is essentially done-get rid of relevant tests and tracking code. --- newsfragments/3751.minor | 0 nix/tahoe-lafs.nix | 4 - src/allmydata/test/test_python3.py | 122 ----------- src/allmydata/util/_python3.py | 311 ----------------------------- 4 files changed, 437 deletions(-) create mode 100644 newsfragments/3751.minor delete mode 100644 src/allmydata/test/test_python3.py delete mode 100644 src/allmydata/util/_python3.py diff --git a/newsfragments/3751.minor b/newsfragments/3751.minor new file mode 100644 index 000000000..e69de29bb diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 8005ca50b..35b29f1cc 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -72,10 +72,6 @@ python.pkgs.buildPythonPackage rec { rm src/allmydata/test/test_connections.py rm src/allmydata/test/cli/test_create.py - # Since we're deleting files, this complains they're missing. For now Nix - # is Python 2-only, anyway, so these tests don't add anything yet. - rm src/allmydata/test/test_python3.py - # Generate _version.py ourselves since we can't rely on the Python code # extracting the information from the .git directory we excluded. cat > src/allmydata/_version.py < Date: Fri, 30 Jul 2021 11:27:41 -0400 Subject: [PATCH 438/463] Choose node names that won't conflict. --- integration/test_i2p.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 1aa7a5d99..f0b06f1e2 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -145,8 +145,8 @@ def i2p_introducer_furl(i2p_introducer, temp_dir): @pytest_twisted.inlineCallbacks def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): - yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) - yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) + yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) + yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) # ensure both nodes are connected to "a grid" by uploading # something via carol, and retrieve it using dave. gold_path = join(temp_dir, "gold") @@ -165,7 +165,7 @@ def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_netw sys.executable, ( sys.executable, '-b', '-m', 'allmydata.scripts.runner', - '-d', join(temp_dir, 'carol'), + '-d', join(temp_dir, 'carol_i2p'), 'put', gold_path, ) ) @@ -179,7 +179,7 @@ def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_netw sys.executable, ( sys.executable, '-b', '-m', 'allmydata.scripts.runner', - '-d', join(temp_dir, 'dave'), + '-d', join(temp_dir, 'dave_i2p'), 'get', cap, ) ) From 90e84730e57e8cba179889c4242a34452d6a064e Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sun, 8 Aug 2021 21:49:02 +0100 Subject: [PATCH 439/463] Fixes 3757 : Refactored test_introducer in web tests to use custom base test cases Signed-off-by: fenn-cs --- newsfragments/3757.other | 0 src/allmydata/test/web/test_introducer.py | 33 ++++++++++++++--------- 2 files changed, 20 insertions(+), 13 deletions(-) create mode 100644 newsfragments/3757.other diff --git a/newsfragments/3757.other b/newsfragments/3757.other new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index 08d95bda9..34b1088e3 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -18,6 +18,12 @@ from bs4 import BeautifulSoup from twisted.trial import unittest from twisted.internet import reactor from twisted.internet import defer +from testtools.twistedsupport import succeeded + +from ..common import ( + SyncTestCase, + AsyncTestCase, +) from foolscap.api import ( fireEventually, @@ -53,6 +59,10 @@ from ..common_web import ( render, ) +from testtools.matchers import ( + Equals +) + @defer.inlineCallbacks def create_introducer_webish(reactor, port_assigner, basedir): @@ -86,11 +96,10 @@ def create_introducer_webish(reactor, port_assigner, basedir): yield fireEventually(None) intro_node.startService() - defer.returnValue((intro_node, ws)) -class IntroducerWeb(unittest.TestCase): +class IntroducerWeb(AsyncTestCase): """ Tests for web-facing functionality of an introducer node. """ @@ -102,6 +111,8 @@ class IntroducerWeb(unittest.TestCase): # Anything using Foolscap leaves some timer trash in the reactor that # we have to arrange to have cleaned up. self.addCleanup(lambda: flushEventualQueue(None)) + return super(IntroducerWeb, self).setUp() + @defer.inlineCallbacks def test_welcome(self): @@ -187,7 +198,7 @@ class IntroducerWeb(unittest.TestCase): self.assertEqual(data["announcement_summary"], {}) -class IntroducerRootTests(unittest.TestCase): +class IntroducerRootTests(SyncTestCase): """ Tests for ``IntroducerRoot``. """ @@ -222,16 +233,12 @@ class IntroducerRootTests(unittest.TestCase): 0, ) - resource = IntroducerRoot(introducer_node) - response = json.loads( - self.successResultOf( + resource = IntroducerRoot(introducer_node) + response = json.loads(succeeded( render(resource, {b"t": [b"json"]}), - ), - ) - self.assertEqual( - response, - { + )._matcher.result) + deferred = defer.succeed({ u"subscription_summary": {"arbitrary": 2}, u"announcement_summary": {"arbitrary": 1}, - }, - ) + }) + self.assertThat(deferred.result, Equals(response)) From 7ad3fa9e25975eb749345af7ea9745dfe166f7e7 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Mon, 9 Aug 2021 23:46:30 +0100 Subject: [PATCH 440/463] correct use of succeeded, remove trailing spaces and unused import Signed-off-by: fenn-cs --- src/allmydata/test/web/test_introducer.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index 34b1088e3..b1bc69f17 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -15,7 +15,6 @@ from os.path import join from bs4 import BeautifulSoup -from twisted.trial import unittest from twisted.internet import reactor from twisted.internet import defer from testtools.twistedsupport import succeeded @@ -60,7 +59,8 @@ from ..common_web import ( ) from testtools.matchers import ( - Equals + Equals, + AfterPreprocessing, ) @@ -112,7 +112,6 @@ class IntroducerWeb(AsyncTestCase): # we have to arrange to have cleaned up. self.addCleanup(lambda: flushEventualQueue(None)) return super(IntroducerWeb, self).setUp() - @defer.inlineCallbacks def test_welcome(self): @@ -233,12 +232,10 @@ class IntroducerRootTests(SyncTestCase): 0, ) - resource = IntroducerRoot(introducer_node) - response = json.loads(succeeded( - render(resource, {b"t": [b"json"]}), - )._matcher.result) - deferred = defer.succeed({ - u"subscription_summary": {"arbitrary": 2}, - u"announcement_summary": {"arbitrary": 1}, - }) - self.assertThat(deferred.result, Equals(response)) + resource = IntroducerRoot(introducer_node) + response = render(resource, {b"t": [b"json"]}) + expected = { + u"subscription_summary": {"arbitrary": 2}, + u"announcement_summary": {"arbitrary": 1}, + } + self.assertThat(response, succeeded(AfterPreprocessing(json.loads, Equals(expected))) From a6b7c07e1cbbfbc6f1aa00a33dd64e1c10159d97 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 10 Aug 2021 10:13:00 +0100 Subject: [PATCH 441/463] added missing parathensis, ran yapf Signed-off-by: fenn-cs --- src/allmydata/test/web/test_introducer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index b1bc69f17..ba0a5beb9 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -238,4 +238,6 @@ class IntroducerRootTests(SyncTestCase): u"subscription_summary": {"arbitrary": 2}, u"announcement_summary": {"arbitrary": 1}, } - self.assertThat(response, succeeded(AfterPreprocessing(json.loads, Equals(expected))) + self.assertThat( + response, + succeeded(AfterPreprocessing(json.loads, Equals(expected)))) From 78a3ca52c4b2eb0e891b4c50d7461eee46214486 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 10 Aug 2021 19:00:05 +0100 Subject: [PATCH 442/463] DUMMY COMMIT TO TRIGGER CI : added commit text to fragments Signed-off-by: fenn-cs --- newsfragments/3757.other | 1 + 1 file changed, 1 insertion(+) diff --git a/newsfragments/3757.other b/newsfragments/3757.other index e69de29bb..3d2d3f272 100644 --- a/newsfragments/3757.other +++ b/newsfragments/3757.other @@ -0,0 +1 @@ +Refactored test_introducer in web tests to use custom base test cases \ No newline at end of file From a4da8048700204f50e377a44e8116e519fcffde7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 07:33:58 -0400 Subject: [PATCH 443/463] Stop using the dockerhub-auth context for normal jobs --- .circleci/config.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1fb7558de..8aa039694 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,18 +16,8 @@ workflows: jobs: # Start with jobs testing various platforms. - # Every job that pulls a Docker image from Docker Hub needs to provide - # credentials for that pull operation to avoid being subjected to - # unauthenticated pull limits shared across all of CircleCI. Use this - # first job to define a yaml anchor that can be used to supply a - # CircleCI job context which makes Docker Hub credentials available in - # the environment. - # - # Contexts are managed in the CircleCI web interface: - # - # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - "debian-9": &DOCKERHUB_CONTEXT - context: "dockerhub-auth" + {} - "debian-10": <<: *DOCKERHUB_CONTEXT From c5fec82328231c718ce8bcfbf02c670bf3305136 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 07:40:16 -0400 Subject: [PATCH 444/463] CircleCI succeeds in pulling the docker images without this It says "Warning!" but then says it is going to use its own credentials. Great. Just what I want. --- .circleci/config.yml | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8aa039694..3e48203c0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,72 +15,49 @@ workflows: ci: jobs: # Start with jobs testing various platforms. - - - "debian-9": &DOCKERHUB_CONTEXT - {} - + - "debian-9": - "debian-10": - <<: *DOCKERHUB_CONTEXT requires: - "debian-9" - "ubuntu-20-04": - <<: *DOCKERHUB_CONTEXT - "ubuntu-18-04": - <<: *DOCKERHUB_CONTEXT requires: - "ubuntu-20-04" - "ubuntu-16-04": - <<: *DOCKERHUB_CONTEXT requires: - "ubuntu-20-04" - "fedora-29": - <<: *DOCKERHUB_CONTEXT - "fedora-28": - <<: *DOCKERHUB_CONTEXT requires: - "fedora-29" - "centos-8": - <<: *DOCKERHUB_CONTEXT - - "nixos-19-09": - <<: *DOCKERHUB_CONTEXT # Test against PyPy 2.7 - "pypy27-buster": - <<: *DOCKERHUB_CONTEXT # Just one Python 3.6 configuration while the port is in-progress. - "python36": - <<: *DOCKERHUB_CONTEXT # Other assorted tasks and configurations - "lint": - <<: *DOCKERHUB_CONTEXT - "pyinstaller": - <<: *DOCKERHUB_CONTEXT - "deprecations": - <<: *DOCKERHUB_CONTEXT - "c-locale": - <<: *DOCKERHUB_CONTEXT # Any locale other than C or UTF-8. - "another-locale": - <<: *DOCKERHUB_CONTEXT - "integration": - <<: *DOCKERHUB_CONTEXT requires: # If the unit test suite doesn't pass, don't bother running the # integration tests. - "debian-9" - "typechecks": - <<: *DOCKERHUB_CONTEXT - - "docs": - <<: *DOCKERHUB_CONTEXT images: # Build the Docker images used by the ci jobs. This makes the ci jobs From 6f36f85a87330569475eb81cb483d600107730eb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 07:41:59 -0400 Subject: [PATCH 445/463] Define the yaml anchor elsewhere, where it is still needed --- .circleci/config.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3e48203c0..7db863847 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -72,8 +72,16 @@ workflows: - "master" jobs: - - "build-image-debian-10": - <<: *DOCKERHUB_CONTEXT + # Every job that pushes a Docker image from Docker Hub needs to provide + # credentials. Use this first job to define a yaml anchor that can be + # used to supply a CircleCI job context which makes Docker Hub + # credentials available in the environment. + # + # Contexts are managed in the CircleCI web interface: + # + # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts + - "build-image-debian-10": &DOCKERHUB_CONTEXT + context: "dockerhub-auth" - "build-image-debian-9": <<: *DOCKERHUB_CONTEXT - "build-image-ubuntu-16-04": From 0e2c4ff7e65312a420df2c6b0effb939caa86c17 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 07:45:45 -0400 Subject: [PATCH 446/463] I suppose you cannot define an empty yaml map by leaving everything out --- .circleci/config.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7db863847..f2d5a29b9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,11 +16,13 @@ workflows: jobs: # Start with jobs testing various platforms. - "debian-9": + {} - "debian-10": requires: - "debian-9" - "ubuntu-20-04": + {} - "ubuntu-18-04": requires: - "ubuntu-20-04" @@ -29,26 +31,37 @@ workflows: - "ubuntu-20-04" - "fedora-29": + {} - "fedora-28": requires: - "fedora-29" - "centos-8": + {} + - "nixos-19-09": + {} # Test against PyPy 2.7 - "pypy27-buster": + {} # Just one Python 3.6 configuration while the port is in-progress. - "python36": + {} # Other assorted tasks and configurations - "lint": + {} - "pyinstaller": + {} - "deprecations": + {} - "c-locale": + {} # Any locale other than C or UTF-8. - "another-locale": + {} - "integration": requires: @@ -57,7 +70,9 @@ workflows: - "debian-9" - "typechecks": + {} - "docs": + {} images: # Build the Docker images used by the ci jobs. This makes the ci jobs From 2bb310c5110518143a6358f3cf1b8685e811421b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 09:50:23 -0400 Subject: [PATCH 447/463] Try to run codechecks3 on CI --- .circleci/config.yml | 20 +++++++++++++++++++- newsfragments/3760.minor | 0 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3760.minor diff --git a/.circleci/config.yml b/.circleci/config.yml index 1fb7558de..153b29089 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -150,7 +150,7 @@ jobs: # Since this job is never scheduled this step is never run so the # actual value here is irrelevant. - lint: + codechecks: docker: - <<: *DOCKERHUB_AUTH image: "circleci/python:2" @@ -168,6 +168,24 @@ jobs: command: | ~/.local/bin/tox -e codechecks + codechecks3: + docker: + - <<: *DOCKERHUB_AUTH + image: "circleci/python:3" + + steps: + - "checkout" + + - run: + name: "Install tox" + command: | + pip install --user tox + + - run: + name: "Static-ish code checks" + command: | + ~/.local/bin/tox -e codechecks3 + pyinstaller: docker: - <<: *DOCKERHUB_AUTH diff --git a/newsfragments/3760.minor b/newsfragments/3760.minor new file mode 100644 index 000000000..e69de29bb From 02f82e2665b7c8af1ab6a7278435bdfda837ca3d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 10:00:06 -0400 Subject: [PATCH 448/463] news fragment --- newsfragments/3759.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3759.minor diff --git a/newsfragments/3759.minor b/newsfragments/3759.minor new file mode 100644 index 000000000..e69de29bb From 97008b70b2592c94ee60c4f040a62d526c44f9c2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 10:25:42 -0400 Subject: [PATCH 449/463] Avoid renaming a job, it causes operational hassle --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8aa459583..5b123dfa8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -140,7 +140,7 @@ jobs: # Since this job is never scheduled this step is never run so the # actual value here is irrelevant. - codechecks: + lint: docker: - <<: *DOCKERHUB_AUTH image: "circleci/python:2" From a1112e4cd0dbbe1ddfb542d371203b1a0cb846ab Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 10:25:49 -0400 Subject: [PATCH 450/463] Add the new job to the workflow --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5b123dfa8..28e4c8d58 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -53,6 +53,8 @@ workflows: # Other assorted tasks and configurations - "lint": {} + - "codechecks3": + {} - "pyinstaller": {} - "deprecations": From 9a8faae28319c40e7e480a451c10072d90ca822b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Aug 2021 09:24:00 -0400 Subject: [PATCH 451/463] Remove PYTHONIOENCODING, set for towncrier, no longer run here --- tox.ini | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tox.ini b/tox.ini index 13b2b999c..9b0f71038 100644 --- a/tox.ini +++ b/tox.ini @@ -114,13 +114,6 @@ commands = [testenv:codechecks] basepython = python2.7 setenv = - # Workaround an error when towncrier is run under the VCS hook, - # https://stackoverflow.com/a/4027726/624787: - # File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/codechecks/lib/python2.7/site-packages/towncrier/check.py", line 44, in __main - # .decode(getattr(sys.stdout, "encoding", "utf8")) - # `TypeError: decode() argument 1 must be string, not None` - PYTHONIOENCODING=utf_8 - # If no positional arguments are given, try to run the checks on the # entire codebase, including various pieces of supporting code. DEFAULT_FILES=src integration static misc setup.py @@ -190,7 +183,7 @@ passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH HO whitelist_externals = git deps = - # see comment in [testenv] about "certifi" + # see comment in [testenv] about "certifi" certifi towncrier==21.3.0 commands = From a591c2c36d6f05535a21848bd89c1c7047191e23 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Aug 2021 10:23:37 -0400 Subject: [PATCH 452/463] news fragment --- newsfragments/3764.documentation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3764.documentation diff --git a/newsfragments/3764.documentation b/newsfragments/3764.documentation new file mode 100644 index 000000000..d473cd27c --- /dev/null +++ b/newsfragments/3764.documentation @@ -0,0 +1 @@ +The Great Black Swamp proposed specification now includes sample interactions to demonstrate expected usage patterns. \ No newline at end of file From 23212fc816cf5f75e1238ae1ccb9827c64fd6d67 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Aug 2021 14:01:44 -0400 Subject: [PATCH 453/463] add some example client/server interactions --- docs/proposed/http-storage-node-protocol.rst | 130 +++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index ad9dd30bc..623cd4f12 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -576,6 +576,136 @@ Just like ``GET /v1/mutable/:storage_index``. Advise the server the data read from the indicated share was corrupt. Just like the immutable version. +Sample Interactions +------------------- + +Immutable Data +~~~~~~~~~~~~~~ + +1. Create a bucket for storage index ``AAAAAAAAAAAAAAAA`` to hold two immutable shares, discovering that share ``1`` was already uploaded:: + + PUT /v1/immutable/AAAAAAAAAAAAAAAA + {"renew-secret": "efgh", "cancel-secret": "ijkl", + "share-numbers": [1, 7], "allocated-size": 48} + + 200 OK + {"already-have": [1], "allocated": [7]} + +#. Upload the content for immutable share ``7``:: + + PUT /v1/immutable/AAAAAAAAAAAAAAAA/7 + Content-Range: bytes 0-15/48 + + + 200 OK + + PUT /v1/immutable/AAAAAAAAAAAAAAAA/7 + Content-Range: bytes 16-31/48 + + + 200 OK + + PUT /v1/immutable/AAAAAAAAAAAAAAAA/7 + Content-Range: bytes 32-47/48 + + + 201 CREATED + +#. Download the content of the previously uploaded immutable share ``7``:: + + GET /v1/immutable/AAAAAAAAAAAAAAAA?share=7&offset=0&size=48 + + 200 OK + + +#. Renew the lease on all immutable shares in bucket ``AAAAAAAAAAAAAAAA``:: + + POST /v1/lease/AAAAAAAAAAAAAAAA + {"renew-secret": "efgh"} + + 204 NO CONTENT + +Mutable Data +~~~~~~~~~~~~ + +1. Create mutable share number ``3`` with ``10`` bytes of data in slot ``BBBBBBBBBBBBBBBB``:: + + POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write + { + "secrets": { + "write-enabler": "abcd", + "lease-renew": "efgh", + "lease-cancel": "ijkl" + }, + "test-write-vectors": { + 3: { + "test": [{ + "offset": 0, + "size": 1, + "operator": "eq", + "specimen": "" + }], + "write": [{ + "offset": 0, + "data": "xxxxxxxxxx" + }], + "new-length": 10 + } + }, + "read-vector": [] + } + + 200 OK + { + "success": true, + "data": [] + } + +#. Safely rewrite the contents of a known version of mutble share number ``3`` (or fail):: + + POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write + { + "secrets": { + "write-enabler": "abcd", + "lease-renew": "efgh", + "lease-cancel": "ijkl" + }, + "test-write-vectors": { + 3: { + "test": [{ + "offset": 0, + "size": , + "operator": "eq", + "specimen": "" + }], + "write": [{ + "offset": 0, + "data": "yyyyyyyyyy" + }], + "new-length": 10 + } + }, + "read-vector": [] + } + + 200 OK + { + "success": true, + "data": [] + } + +#. Download the contents of share number ``3``:: + + GET /v1/mutable/BBBBBBBBBBBBBBBB?share=3&offset=0&size=10 + + +#. Renew the lease on previously uploaded mutable share in slot ``BBBBBBBBBBBBBBBB``:: + + POST /v1/lease/BBBBBBBBBBBBBBBB + {"renew-secret": "efgh"} + + 204 NO CONTENT + .. _RFC 7469: https://tools.ietf.org/html/rfc7469#section-2.4 .. _RFC 7049: https://tools.ietf.org/html/rfc7049#section-4 From 65f71135e19efa7d56ed927d4bbd75f047898f7e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Aug 2021 14:10:50 -0400 Subject: [PATCH 454/463] Spell the endpoint correctly --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 623cd4f12..3592a1532 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -584,7 +584,7 @@ Immutable Data 1. Create a bucket for storage index ``AAAAAAAAAAAAAAAA`` to hold two immutable shares, discovering that share ``1`` was already uploaded:: - PUT /v1/immutable/AAAAAAAAAAAAAAAA + POST /v1/immutable/AAAAAAAAAAAAAAAA {"renew-secret": "efgh", "cancel-secret": "ijkl", "share-numbers": [1, 7], "allocated-size": 48} From 47dfe5d4f0c75e49b3561a92c019d7063bd87d00 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Aug 2021 14:11:08 -0400 Subject: [PATCH 455/463] fix typo --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 3592a1532..1f7df84c0 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -661,7 +661,7 @@ Mutable Data "data": [] } -#. Safely rewrite the contents of a known version of mutble share number ``3`` (or fail):: +#. Safely rewrite the contents of a known version of mutable share number ``3`` (or fail):: POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write { From 49300fc0808307913216d9a522ff83601182e236 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Aug 2021 14:58:30 -0400 Subject: [PATCH 456/463] news fragment --- newsfragments/3765.documentation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3765.documentation diff --git a/newsfragments/3765.documentation b/newsfragments/3765.documentation new file mode 100644 index 000000000..a3b59c4d6 --- /dev/null +++ b/newsfragments/3765.documentation @@ -0,0 +1 @@ +The Great Black Swamp proposed specification now includes a glossary. \ No newline at end of file From 0f78d8df2561dd2509cec991abac78d65e16069d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Aug 2021 14:58:34 -0400 Subject: [PATCH 457/463] a glossary --- docs/proposed/http-storage-node-protocol.rst | 41 ++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index ad9dd30bc..cc2d15627 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -13,6 +13,47 @@ Specifically, it should be possible to implement a Tahoe-LAFS storage server wit The Tahoe-LAFS client will also need to change but it is not expected that it will be noticably simplified by this change (though this may be the first step towards simplifying it). +Glossary +-------- + +.. glossary:: + + `Foolscap `_ + an RPC/RMI (Remote Procedure Call / Remote Method Invocation) protocol for use with Twisted + + storage server + a Tahoe-LAFS process configured to offer storage and reachable over the network for store and retrieve operations + + introducer + a Tahoe-LAFS process at a known location configured to re-publish announcements about the location of storage servers + + fURL + a self-authenticating URL-like string which can be used to locate a remote object using the Foolscap protocol + + lease + state associated with a share informing a storage server of the duration of storage desired by a client + + share + a single unit of client-provided arbitrary data to be stored by a storage server + (in practice, one of the outputs of applying ZFEC encoding to some ciphertext with some additional metadata attached) + + bucket + a group of one or more immutable shares held by a storage server and having a common storage index + + slot + a group of one or more mutable shares held by a storage server and having a common storage index + (sometimes "slot" is considered a synonym for "storage index of a slot") + + storage index + a short string which can address a slot or a bucket + (in practice, derived by hashing the encryption key associated with contents of that slot or bucket) + + write enabler + a short secret string which storage servers require to be presented before allowing mutation of any mutable share + + lease renew secret + a short secret string which storage servers required to be presented before allowing a particular lease to be renewed + Motivation ---------- From c5a03f4116bb3d83ed7e6213b784e8cc3a0a1528 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 18 Aug 2021 10:17:48 -0400 Subject: [PATCH 458/463] elaborate on ``already-have`` and ``allocated`` somewhat --- docs/proposed/http-storage-node-protocol.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 1f7df84c0..dd084c397 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -422,6 +422,15 @@ However, we decided this does not matter because: therefore no proxy servers can perform any extra logging. * Tahoe-LAFS itself does not currently log HTTP request URLs. +The response includes ``already-have`` and ``allocated`` for two reasons: + +* If an upload is interrupted and the client loses its local state that lets it know it already uploaded some shares + then this allows it to discover this fact (by inspecting ``already-have``) and only upload the missing shares (indicated by ``allocated``). + +* If an upload has completed a client may still choose to re-balance storage by moving shares between servers. + This might be because a server has become unavailable and a remaining server needs to store more shares for the upload. + It could also just be that the client's preferred servers have changed. + ``PUT /v1/immutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! From 602d4c5a9129e6312c1ed64d72dc4fb365f53ec4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 19 Aug 2021 10:26:45 -0400 Subject: [PATCH 459/463] improve the "create lease" endpoint * Simplify some language using terms from our new glossary * explicitly state the two success-case behaviors * make the error-case behavior different from the success-case behavior * link to some tickets about future work in this area --- docs/proposed/http-storage-node-protocol.rst | 24 ++++++++++++-------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 41a0a0fea..de0918b58 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -369,19 +369,26 @@ For example:: ``PUT /v1/lease/:storage_index`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Create a new lease that applies to all shares for the given storage index. +Create a new lease on the bucket addressed by ``storage_index``. The details of the lease are encoded in the request body. For example:: {"renew-secret": "abcd", "cancel-secret": "efgh"} -If there are no shares for the given ``storage_index`` -then do nothing and return ``NO CONTENT``. - If the ``renew-secret`` value matches an existing lease -then that lease will be renewed instead. +then the expiration time of that lease will be changed to 31 days after the time of this operation. +If it does not match an existing lease +then a new lease will be created with this ``renew-secret`` which expires 31 days after the time of this operation. + +In these cases the response is ``NO CONTENT`` with an empty body. + +It is possible that the storage server will have no shares for the given ``storage_index`` because: + +* no such shares have ever been uploaded. +* a previous lease expired and the storage server reclaimed the storage by deleting the shares. + +In these cases the server takes no action and returns ``NOT FOUND``. -The lease expires after 31 days. Discussion `````````` @@ -391,10 +398,9 @@ We chose to put these values into the request body to make the URL simpler. Several behaviors here are blindly copied from the Foolscap-based storage server protocol. -* There is a cancel secret but there is no API to use it to cancel a lease. +* There is a cancel secret but there is no API to use it to cancel a lease (see ticket:3768). * The lease period is hard-coded at 31 days. -* There is no way to differentiate between success and an unknown **storage index**. -* There are separate **add** and **renew** lease APIs. +* There are separate **add** and **renew** lease APIs (see ticket:3773). These are not necessarily ideal behaviors but they are adopted to avoid any *semantic* changes between the Foolscap- and HTTP-based protocols. From 03c633cda0f92daa22c0dae36c900681a2ed27b6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 19 Aug 2021 10:28:17 -0400 Subject: [PATCH 460/463] news fragment --- newsfragments/3763.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3763.minor diff --git a/newsfragments/3763.minor b/newsfragments/3763.minor new file mode 100644 index 000000000..e69de29bb From 117befd8986596f1b6903a68076aad75da4a0b7c Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Mon, 23 Aug 2021 02:34:32 +0100 Subject: [PATCH 461/463] detect all port 0 configs Signed-off-by: fenn-cs --- newsfragments/3563.minor | 1 + src/allmydata/node.py | 2 +- src/allmydata/test/test_node.py | 17 +++++++++++------ 3 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 newsfragments/3563.minor diff --git a/newsfragments/3563.minor b/newsfragments/3563.minor new file mode 100644 index 000000000..038b50995 --- /dev/null +++ b/newsfragments/3563.minor @@ -0,0 +1 @@ +detect all port 0 configs \ No newline at end of file diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 4dcb7cc76..5a6f8c66f 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -793,7 +793,7 @@ def _tub_portlocation(config, get_local_addresses_sync, allocate_tcp_port): tubport = _convert_tub_port(cfg_tubport) for port in tubport.split(","): - if port in ("0", "tcp:0"): + if port in ("0", "tcp:0", "tcp:port=0", "tcp:0:interface=127.0.0.1"): raise PortAssignmentRequired() if cfg_location is None: diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index e44fd5743..cf5fa27f3 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -6,7 +6,7 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future.utils import PY2, native_str +from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 @@ -530,6 +530,14 @@ def _stub_allocate_tcp_port(): """ return 999 +def _stub_none(): + """ + A function like ``_stub_allocate_tcp`` or ``_stub_get_local_addresses_sync`` + but that return an empty list since ``allmydata.node._tub_portlocation`` requires a + callable for paramter 1 and 2 counting from 0. + """ + return [] + class TestMissingPorts(unittest.TestCase): """ @@ -550,7 +558,7 @@ class TestMissingPorts(unittest.TestCase): ) config = config_from_string(self.basedir, "portnum", config_data) with self.assertRaises(PortAssignmentRequired): - _tub_portlocation(config, None, None) + _tub_portlocation(config, _stub_none, _stub_none) def test_listen_on_zero_with_host(self): """ @@ -563,10 +571,7 @@ class TestMissingPorts(unittest.TestCase): ) config = config_from_string(self.basedir, "portnum", config_data) with self.assertRaises(PortAssignmentRequired): - _tub_portlocation(config, None, None) - test_listen_on_zero_with_host.todo = native_str( # type: ignore - "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3563" - ) + _tub_portlocation(config, _stub_none, _stub_none) def test_parsing_tcp(self): """ From 398fe537e8da2e8d8e4eaf5e9bd145cbd32025e1 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Mon, 23 Aug 2021 02:48:24 +0100 Subject: [PATCH 462/463] added \n in newfragment file Signed-off-by: fenn-cs --- newsfragments/3563.minor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3563.minor b/newsfragments/3563.minor index 038b50995..ea08fda56 100644 --- a/newsfragments/3563.minor +++ b/newsfragments/3563.minor @@ -1 +1 @@ -detect all port 0 configs \ No newline at end of file +detect all port 0 configs From 7a88b5ddae5497f52421deb2e852971809a62b45 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 24 Aug 2021 13:05:11 +0100 Subject: [PATCH 463/463] removed unnecessary comment in minor newsfragment Signed-off-by: fenn-cs --- newsfragments/3563.minor | 1 - 1 file changed, 1 deletion(-) diff --git a/newsfragments/3563.minor b/newsfragments/3563.minor index ea08fda56..e69de29bb 100644 --- a/newsfragments/3563.minor +++ b/newsfragments/3563.minor @@ -1 +0,0 @@ -detect all port 0 configs