ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2024-12-19 13:07:56 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #576 from tahoe-lafs/3005.upstream-inline_callbacks

Browse Source 
Use upstream inline_callbacks implementation.

Fixes: ticket:3005
This commit is contained in:
Jean-Paul Calderone 2019-03-21 16:24:21 -04:00 committed by GitHub
commit f7f9cf6abc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 9 additions and 478 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
newsfragments/3005.minor Normal file
View File

8
src/allmydata/_auto_deps.py
View File

@ -100,10 +100,10 @@ install_requires = [
# Eliot is contemplating dropping Python 2 support. Stick to a version we
# know works on Python 2.7. Because we don't have support for `==`
# constraints, pin 1.6.x this way. I feel pretty safe betting that we
# won't end up stuck on Eliot 1.6.100 with a critical fix only present in
# 1.6.101. And if we do, I know how to deal with that situation.
"eliot >= 1.6.0, <= 1.6.100",
# constraints, pin 1.7.x this way. I feel pretty safe betting that we
# won't end up stuck on Eliot 1.7.100 with a critical fix only present in
# 1.7.101. And if we do, I know how to deal with that situation.
"eliot >= 1.7.0, <= 1.7.100",
# A great way to define types of values.
"attrs >= 18.2.0",

351
src/allmydata/test/test_eliotutil.py
View File

@ -9,7 +9,6 @@ from __future__ import (
division,
)
from pprint import pformat
from sys import stdout
import logging
@ -27,7 +26,6 @@ from testtools.matchers import (
AfterPreprocessing,
)
from testtools.twistedsupport import (
has_no_result,
succeeded,
failed,
)
@ -35,7 +33,6 @@ from testtools.twistedsupport import (
from eliot import (
Message,
FileDestination,
start_action,
)
from eliot.twisted import DeferredContext
from eliot.testing import (
@ -44,15 +41,12 @@ from eliot.testing import (
)
from twisted.internet.defer import (
Deferred,
succeed,
)
from twisted.internet.task import deferLater
from twisted.internet import reactor
from ..util.eliotutil import (
eliot_friendly_generator_function,
inline_callbacks,
log_call_deferred,
_parse_destination_description,
_EliotLogging,
@ -82,350 +76,7 @@ class EliotLoggedTestTests(AsyncTestCase):
def assert_logged_messages_contain_fields(testcase, logged_messages, expected_fields):
testcase.assertEqual(len(logged_messages), len(expected_fields))
actual_fields = list(
{key: msg.message[key] for key in expected if key in msg.message}
for (msg, expected)
in zip(logged_messages, expected_fields)
)
testcase.assertEqual(actual_fields, expected_fields)
def assert_logged_action_contains_messages(testcase, logger, expected_action, expected_fields):
action = assertHasAction(
testcase,
logger,
expected_action,
True,
)
assert_logged_messages_contain_fields(
testcase,
action.children,
expected_fields,
)
def assert_expected_action_tree(testcase, logger, expected_action_type, expected_type_tree):
logged_action = assertHasAction(
testcase,
logger,
expected_action_type,
True,
)
type_tree = logged_action.type_tree()
testcase.assertEqual(
{expected_action_type: expected_type_tree},
type_tree,
"Logger had messages:\n{}".format(pformat(logger.messages, indent=4)),
)
def assert_generator_logs_action_tree(testcase, generator_function, logger, expected_action_type, expected_type_tree):
list(eliot_friendly_generator_function(generator_function)())
assert_expected_action_tree(
testcase,
logger,
expected_action_type,
expected_type_tree,
)
class EliotFriendlyGeneratorFunctionTests(SyncTestCase):
# Get our custom assertion failure messages *and* the standard ones.
longMessage = True
def test_yield_none(self):
@eliot_friendly_generator_function
def g():
Message.log(message_type=u"hello")
yield
Message.log(message_type=u"goodbye")
with start_action(action_type=u"the-action"):
list(g())
assert_expected_action_tree(
self,
self.eliot_logger,
u"the-action",
[u"hello", u"yielded", u"goodbye"],
)
def test_yield_value(self):
expected = object()
@eliot_friendly_generator_function
def g():
Message.log(message_type=u"hello")
yield expected
Message.log(message_type=u"goodbye")
with start_action(action_type=u"the-action"):
self.assertEqual([expected], list(g()))
assert_expected_action_tree(
self,
self.eliot_logger,
u"the-action",
[u"hello", u"yielded", u"goodbye"],
)
def test_yield_inside_another_action(self):
@eliot_friendly_generator_function
def g():
Message.log(message_type=u"a")
with start_action(action_type=u"confounding-factor"):
Message.log(message_type=u"b")
yield None
Message.log(message_type=u"c")
Message.log(message_type=u"d")
with start_action(action_type=u"the-action"):
list(g())
assert_expected_action_tree(
self,
self.eliot_logger,
u"the-action",
[u"a",
{u"confounding-factor": [u"b", u"yielded", u"c"]},
u"d",
],
)
def test_yield_inside_nested_actions(self):
@eliot_friendly_generator_function
def g():
Message.log(message_type=u"a")
with start_action(action_type=u"confounding-factor"):
Message.log(message_type=u"b")
yield None
with start_action(action_type=u"double-confounding-factor"):
yield None
Message.log(message_type=u"c")
Message.log(message_type=u"d")
Message.log(message_type=u"e")
with start_action(action_type=u"the-action"):
list(g())
assert_expected_action_tree(
self,
self.eliot_logger,
u"the-action", [
u"a",
{u"confounding-factor": [
u"b",
u"yielded",
{u"double-confounding-factor": [
u"yielded",
u"c",
]},
u"d",
]},
u"e",
],
)
def test_generator_and_non_generator(self):
@eliot_friendly_generator_function
def g():
Message.log(message_type=u"a")
yield
with start_action(action_type=u"action-a"):
Message.log(message_type=u"b")
yield
Message.log(message_type=u"c")
Message.log(message_type=u"d")
yield
with start_action(action_type=u"the-action"):
generator = g()
next(generator)
Message.log(message_type=u"0")
next(generator)
Message.log(message_type=u"1")
next(generator)
Message.log(message_type=u"2")
self.assertRaises(StopIteration, lambda: next(generator))
assert_expected_action_tree(
self,
self.eliot_logger,
u"the-action", [
u"a",
u"yielded",
u"0",
{
u"action-a": [
u"b",
u"yielded",
u"c",
],
},
u"1",
u"d",
u"yielded",
u"2",
],
)
def test_concurrent_generators(self):
@eliot_friendly_generator_function
def g(which):
Message.log(message_type=u"{}-a".format(which))
with start_action(action_type=which):
Message.log(message_type=u"{}-b".format(which))
yield
Message.log(message_type=u"{}-c".format(which))
Message.log(message_type=u"{}-d".format(which))
gens = [g(u"1"), g(u"2")]
with start_action(action_type=u"the-action"):
while gens:
for g in gens[:]:
try:
next(g)
except StopIteration:
gens.remove(g)
assert_expected_action_tree(
self,
self.eliot_logger,
u"the-action", [
u"1-a",
{u"1": [
u"1-b",
u"yielded",
u"1-c",
]},
u"2-a",
{u"2": [
u"2-b",
u"yielded",
u"2-c",
]},
u"1-d",
u"2-d",
],
)
def test_close_generator(self):
@eliot_friendly_generator_function
def g():
Message.log(message_type=u"a")
try:
yield
Message.log(message_type=u"b")
finally:
Message.log(message_type=u"c")
with start_action(action_type=u"the-action"):
gen = g()
next(gen)
gen.close()
assert_expected_action_tree(
self,
self.eliot_logger,
u"the-action", [
u"a",
u"yielded",
u"c",
],
)
def test_nested_generators(self):
@eliot_friendly_generator_function
def g(recurse):
with start_action(action_type=u"a-recurse={}".format(recurse)):
Message.log(message_type=u"m-recurse={}".format(recurse))
if recurse:
set(g(False))
else:
yield
with start_action(action_type=u"the-action"):
set(g(True))
assert_expected_action_tree(
self,
self.eliot_logger,
u"the-action", [{
u"a-recurse=True": [
u"m-recurse=True", {
u"a-recurse=False": [
u"m-recurse=False",
u"yielded",
],
},
],
}],
)
class InlineCallbacksTests(SyncTestCase):
# Get our custom assertion failure messages *and* the standard ones.
longMessage = True
def _a_b_test(self, logger, g):
with start_action(action_type=u"the-action"):
self.assertThat(g(), succeeded(Is(None)))
assert_expected_action_tree(
self,
logger,
u"the-action", [
u"a",
u"yielded",
u"b",
],
)
def test_yield_none(self):
@inline_callbacks
def g():
Message.log(message_type=u"a")
yield
Message.log(message_type=u"b")
self._a_b_test(self.eliot_logger, g)
def test_yield_fired_deferred(self):
@inline_callbacks
def g():
Message.log(message_type=u"a")
yield succeed(None)
Message.log(message_type=u"b")
self._a_b_test(self.eliot_logger, g)
def test_yield_unfired_deferred(self):
waiting = Deferred()
@inline_callbacks
def g():
Message.log(message_type=u"a")
yield waiting
Message.log(message_type=u"b")
with start_action(action_type=u"the-action"):
d = g()
self.assertThat(waiting, has_no_result())
waiting.callback(None)
self.assertThat(d, succeeded(Is(None)))
assert_expected_action_tree(
self,
self.eliot_logger,
u"the-action", [
u"a",
u"yielded",
u"b",
],
)
class ParseDestinationDescriptionTests(SyncTestCase):
class ParseDestinationDescriptionTests(SyncTestCase):
"""
Tests for ``_parse_destination_description``.
"""

128
src/allmydata/util/eliotutil.py
View File

@ -10,8 +10,6 @@ from __future__ import (
)
__all__ = [
"use_generator_context",
"eliot_friendly_generator_function",
"inline_callbacks",
"eliot_logging_service",
"opt_eliot_destination",
@ -30,12 +28,9 @@ __all__ = [
]
from sys import (
exc_info,
stdout,
)
from functools import wraps
from contextlib import contextmanager
from weakref import WeakKeyDictionary
from logging import (
INFO,
Handler,
@ -67,7 +62,10 @@ from eliot import (
from eliot._validation import (
ValidationError,
)
from eliot.twisted import DeferredContext
from eliot.twisted import (
DeferredContext,
inline_callbacks,
)
from twisted.python.usage import (
UsageError,
@ -84,7 +82,6 @@ from twisted.logger import (
globalLogPublisher,
)
from twisted.internet.defer import (
inlineCallbacks,
maybeDeferred,
)
from twisted.application.service import Service
@ -97,123 +94,6 @@ from .fake_inotify import (
humanReadableMask,
)
class _GeneratorContext(object):
def __init__(self, execution_context):
self._execution_context = execution_context
self._contexts = WeakKeyDictionary()
self._current_generator = None
def init_stack(self, generator):
stack = list(self._execution_context._get_stack())
self._contexts[generator] = stack
def get_stack(self):
if self._current_generator is None:
# If there is no currently active generator then we have no
# special stack to supply. Let the execution context figure out a
# different answer on its own.
return None
# Otherwise, give back the action context stack we've been tracking
# for the currently active generator. It must have been previously
# initialized (it's too late to do it now)!
return self._contexts[self._current_generator]
@contextmanager
def context(self, generator):
previous_generator = self._current_generator
try:
self._current_generator = generator
yield
finally:
self._current_generator = previous_generator
from eliot._action import _context
_the_generator_context = _GeneratorContext(_context)
def use_generator_context():
_context.get_sub_context = _the_generator_context.get_stack
use_generator_context()
def eliot_friendly_generator_function(original):
"""
Decorate a generator function so that the Eliot action context is
preserved across ``yield`` expressions.
"""
@wraps(original)
def wrapper(*a, **kw):
# Keep track of whether the next value to deliver to the generator is
# a non-exception or an exception.
ok = True
# Keep track of the next value to deliver to the generator.
value_in = None
# Create the generator with a call to the generator function. This
# happens with whatever Eliot action context happens to be active,
# which is fine and correct and also irrelevant because no code in the
# generator function can run until we call send or throw on it.
gen = original(*a, **kw)
# Initialize the per-generator Eliot action context stack to the
# current action stack. This might be the main stack or, if another
# decorated generator is running, it might be the stack for that
# generator. Not our business.
_the_generator_context.init_stack(gen)
while True:
try:
# Whichever way we invoke the generator, we will do it
# with the Eliot action context stack we've saved for it.
# Then the context manager will re-save it and restore the
# "outside" stack for us.
with _the_generator_context.context(gen):
if ok:
value_out = gen.send(value_in)
else:
value_out = gen.throw(*value_in)
# We have obtained a value from the generator. In
# giving it to us, it has given up control. Note this
# fact here. Importantly, this is within the
# generator's action context so that we get a good
# indication of where the yield occurred.
#
# This might be too noisy, consider dropping it or
# making it optional.
Message.log(message_type=u"yielded")
except StopIteration:
# When the generator raises this, it is signaling
# completion. Leave the loop.
break
else:
try:
# Pass the generator's result along to whoever is
# driving. Capture the result as the next value to
# send inward.
value_in = yield value_out
except:
# Or capture the exception if that's the flavor of the
# next value.
ok = False
value_in = exc_info()
else:
ok = True
return wrapper
def inline_callbacks(original):
"""
Decorate a function like ``inlineCallbacks`` would but in a more
Eliot-friendly way. Use it just like ``inlineCallbacks`` but where you
want Eliot action contexts to Do The Right Thing inside the decorated
function.
"""
return inlineCallbacks(
eliot_friendly_generator_function(original)
)
def validateInstanceOf(t):
"""
Return an Eliot validator that requires values to be instances of ``t``.