From 9bace08dfe795ca48b5fadd369371b4f29de1a2b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 20 Mar 2019 12:08:15 -0400 Subject: [PATCH 1/4] news fragment --- newsfragments/3005.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3005.minor diff --git a/newsfragments/3005.minor b/newsfragments/3005.minor new file mode 100644 index 000000000..e69de29bb From e5e08e8410de8e3df2674263238393453e8a6db8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 20 Mar 2019 08:57:39 -0400 Subject: [PATCH 2/4] Update pin to 1.7.x --- src/allmydata/_auto_deps.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/_auto_deps.py b/src/allmydata/_auto_deps.py index 78fa672fe..38b98e59c 100644 --- a/src/allmydata/_auto_deps.py +++ b/src/allmydata/_auto_deps.py @@ -100,10 +100,10 @@ install_requires = [ # Eliot is contemplating dropping Python 2 support. Stick to a version we # know works on Python 2.7. Because we don't have support for `==` - # constraints, pin 1.6.x this way. I feel pretty safe betting that we - # won't end up stuck on Eliot 1.6.100 with a critical fix only present in - # 1.6.101. And if we do, I know how to deal with that situation. - "eliot >= 1.6.0, <= 1.6.100", + # constraints, pin 1.7.x this way. I feel pretty safe betting that we + # won't end up stuck on Eliot 1.7.100 with a critical fix only present in + # 1.7.101. And if we do, I know how to deal with that situation. + "eliot >= 1.7.0, <= 1.7.100", # A great way to define types of values. "attrs >= 18.2.0", From e2d242d2990ed8b447d3c67cc5651770f9ed9a14 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 20 Mar 2019 08:59:17 -0400 Subject: [PATCH 3/4] remove our implementation of inline_callbacks re-publish Eliot's --- src/allmydata/util/eliotutil.py | 128 +------------------------------- 1 file changed, 4 insertions(+), 124 deletions(-) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index 535f4913d..7b36a27d1 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -10,8 +10,6 @@ from __future__ import ( ) __all__ = [ - "use_generator_context", - "eliot_friendly_generator_function", "inline_callbacks", "eliot_logging_service", "opt_eliot_destination", @@ -30,12 +28,9 @@ __all__ = [ ] from sys import ( - exc_info, stdout, ) from functools import wraps -from contextlib import contextmanager -from weakref import WeakKeyDictionary from logging import ( INFO, Handler, @@ -67,7 +62,10 @@ from eliot import ( from eliot._validation import ( ValidationError, ) -from eliot.twisted import DeferredContext +from eliot.twisted import ( + DeferredContext, + inline_callbacks, +) from twisted.python.usage import ( UsageError, @@ -84,7 +82,6 @@ from twisted.logger import ( globalLogPublisher, ) from twisted.internet.defer import ( - inlineCallbacks, maybeDeferred, ) from twisted.application.service import Service @@ -97,123 +94,6 @@ from .fake_inotify import ( humanReadableMask, ) -class _GeneratorContext(object): - def __init__(self, execution_context): - self._execution_context = execution_context - self._contexts = WeakKeyDictionary() - self._current_generator = None - - def init_stack(self, generator): - stack = list(self._execution_context._get_stack()) - self._contexts[generator] = stack - - def get_stack(self): - if self._current_generator is None: - # If there is no currently active generator then we have no - # special stack to supply. Let the execution context figure out a - # different answer on its own. - return None - # Otherwise, give back the action context stack we've been tracking - # for the currently active generator. It must have been previously - # initialized (it's too late to do it now)! - return self._contexts[self._current_generator] - - @contextmanager - def context(self, generator): - previous_generator = self._current_generator - try: - self._current_generator = generator - yield - finally: - self._current_generator = previous_generator - - -from eliot._action import _context -_the_generator_context = _GeneratorContext(_context) - - -def use_generator_context(): - _context.get_sub_context = _the_generator_context.get_stack -use_generator_context() - - -def eliot_friendly_generator_function(original): - """ - Decorate a generator function so that the Eliot action context is - preserved across ``yield`` expressions. - """ - @wraps(original) - def wrapper(*a, **kw): - # Keep track of whether the next value to deliver to the generator is - # a non-exception or an exception. - ok = True - - # Keep track of the next value to deliver to the generator. - value_in = None - - # Create the generator with a call to the generator function. This - # happens with whatever Eliot action context happens to be active, - # which is fine and correct and also irrelevant because no code in the - # generator function can run until we call send or throw on it. - gen = original(*a, **kw) - - # Initialize the per-generator Eliot action context stack to the - # current action stack. This might be the main stack or, if another - # decorated generator is running, it might be the stack for that - # generator. Not our business. - _the_generator_context.init_stack(gen) - while True: - try: - # Whichever way we invoke the generator, we will do it - # with the Eliot action context stack we've saved for it. - # Then the context manager will re-save it and restore the - # "outside" stack for us. - with _the_generator_context.context(gen): - if ok: - value_out = gen.send(value_in) - else: - value_out = gen.throw(*value_in) - # We have obtained a value from the generator. In - # giving it to us, it has given up control. Note this - # fact here. Importantly, this is within the - # generator's action context so that we get a good - # indication of where the yield occurred. - # - # This might be too noisy, consider dropping it or - # making it optional. - Message.log(message_type=u"yielded") - except StopIteration: - # When the generator raises this, it is signaling - # completion. Leave the loop. - break - else: - try: - # Pass the generator's result along to whoever is - # driving. Capture the result as the next value to - # send inward. - value_in = yield value_out - except: - # Or capture the exception if that's the flavor of the - # next value. - ok = False - value_in = exc_info() - else: - ok = True - - return wrapper - - -def inline_callbacks(original): - """ - Decorate a function like ``inlineCallbacks`` would but in a more - Eliot-friendly way. Use it just like ``inlineCallbacks`` but where you - want Eliot action contexts to Do The Right Thing inside the decorated - function. - """ - return inlineCallbacks( - eliot_friendly_generator_function(original) - ) - def validateInstanceOf(t): """ Return an Eliot validator that requires values to be instances of ``t``. From 5bdb37b78685a3e901405256bdfe7fd32cd0913d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 20 Mar 2019 09:55:00 -0400 Subject: [PATCH 4/4] Remove tests for removed code --- src/allmydata/test/test_eliotutil.py | 351 +-------------------------- 1 file changed, 1 insertion(+), 350 deletions(-) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 199403038..b382b7289 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -9,7 +9,6 @@ from __future__ import ( division, ) -from pprint import pformat from sys import stdout import logging @@ -27,7 +26,6 @@ from testtools.matchers import ( AfterPreprocessing, ) from testtools.twistedsupport import ( - has_no_result, succeeded, failed, ) @@ -35,7 +33,6 @@ from testtools.twistedsupport import ( from eliot import ( Message, FileDestination, - start_action, ) from eliot.twisted import DeferredContext from eliot.testing import ( @@ -44,15 +41,12 @@ from eliot.testing import ( ) from twisted.internet.defer import ( - Deferred, succeed, ) from twisted.internet.task import deferLater from twisted.internet import reactor from ..util.eliotutil import ( - eliot_friendly_generator_function, - inline_callbacks, log_call_deferred, _parse_destination_description, _EliotLogging, @@ -82,350 +76,7 @@ class EliotLoggedTestTests(AsyncTestCase): -def assert_logged_messages_contain_fields(testcase, logged_messages, expected_fields): - testcase.assertEqual(len(logged_messages), len(expected_fields)) - actual_fields = list( - {key: msg.message[key] for key in expected if key in msg.message} - for (msg, expected) - in zip(logged_messages, expected_fields) - ) - testcase.assertEqual(actual_fields, expected_fields) - - -def assert_logged_action_contains_messages(testcase, logger, expected_action, expected_fields): - action = assertHasAction( - testcase, - logger, - expected_action, - True, - ) - assert_logged_messages_contain_fields( - testcase, - action.children, - expected_fields, - ) - -def assert_expected_action_tree(testcase, logger, expected_action_type, expected_type_tree): - logged_action = assertHasAction( - testcase, - logger, - expected_action_type, - True, - ) - type_tree = logged_action.type_tree() - testcase.assertEqual( - {expected_action_type: expected_type_tree}, - type_tree, - "Logger had messages:\n{}".format(pformat(logger.messages, indent=4)), - ) - -def assert_generator_logs_action_tree(testcase, generator_function, logger, expected_action_type, expected_type_tree): - list(eliot_friendly_generator_function(generator_function)()) - assert_expected_action_tree( - testcase, - logger, - expected_action_type, - expected_type_tree, - ) - - -class EliotFriendlyGeneratorFunctionTests(SyncTestCase): - # Get our custom assertion failure messages *and* the standard ones. - longMessage = True - - def test_yield_none(self): - @eliot_friendly_generator_function - def g(): - Message.log(message_type=u"hello") - yield - Message.log(message_type=u"goodbye") - - with start_action(action_type=u"the-action"): - list(g()) - - assert_expected_action_tree( - self, - self.eliot_logger, - u"the-action", - [u"hello", u"yielded", u"goodbye"], - ) - - def test_yield_value(self): - expected = object() - - @eliot_friendly_generator_function - def g(): - Message.log(message_type=u"hello") - yield expected - Message.log(message_type=u"goodbye") - - with start_action(action_type=u"the-action"): - self.assertEqual([expected], list(g())) - - assert_expected_action_tree( - self, - self.eliot_logger, - u"the-action", - [u"hello", u"yielded", u"goodbye"], - ) - - def test_yield_inside_another_action(self): - @eliot_friendly_generator_function - def g(): - Message.log(message_type=u"a") - with start_action(action_type=u"confounding-factor"): - Message.log(message_type=u"b") - yield None - Message.log(message_type=u"c") - Message.log(message_type=u"d") - - with start_action(action_type=u"the-action"): - list(g()) - - assert_expected_action_tree( - self, - self.eliot_logger, - u"the-action", - [u"a", - {u"confounding-factor": [u"b", u"yielded", u"c"]}, - u"d", - ], - ) - - def test_yield_inside_nested_actions(self): - @eliot_friendly_generator_function - def g(): - Message.log(message_type=u"a") - with start_action(action_type=u"confounding-factor"): - Message.log(message_type=u"b") - yield None - with start_action(action_type=u"double-confounding-factor"): - yield None - Message.log(message_type=u"c") - Message.log(message_type=u"d") - Message.log(message_type=u"e") - - with start_action(action_type=u"the-action"): - list(g()) - - assert_expected_action_tree( - self, - self.eliot_logger, - u"the-action", [ - u"a", - {u"confounding-factor": [ - u"b", - u"yielded", - {u"double-confounding-factor": [ - u"yielded", - u"c", - ]}, - u"d", - ]}, - u"e", - ], - ) - - def test_generator_and_non_generator(self): - @eliot_friendly_generator_function - def g(): - Message.log(message_type=u"a") - yield - with start_action(action_type=u"action-a"): - Message.log(message_type=u"b") - yield - Message.log(message_type=u"c") - - Message.log(message_type=u"d") - yield - - with start_action(action_type=u"the-action"): - generator = g() - next(generator) - Message.log(message_type=u"0") - next(generator) - Message.log(message_type=u"1") - next(generator) - Message.log(message_type=u"2") - self.assertRaises(StopIteration, lambda: next(generator)) - - assert_expected_action_tree( - self, - self.eliot_logger, - u"the-action", [ - u"a", - u"yielded", - u"0", - { - u"action-a": [ - u"b", - u"yielded", - u"c", - ], - }, - u"1", - u"d", - u"yielded", - u"2", - ], - ) - - def test_concurrent_generators(self): - @eliot_friendly_generator_function - def g(which): - Message.log(message_type=u"{}-a".format(which)) - with start_action(action_type=which): - Message.log(message_type=u"{}-b".format(which)) - yield - Message.log(message_type=u"{}-c".format(which)) - Message.log(message_type=u"{}-d".format(which)) - - gens = [g(u"1"), g(u"2")] - with start_action(action_type=u"the-action"): - while gens: - for g in gens[:]: - try: - next(g) - except StopIteration: - gens.remove(g) - - assert_expected_action_tree( - self, - self.eliot_logger, - u"the-action", [ - u"1-a", - {u"1": [ - u"1-b", - u"yielded", - u"1-c", - ]}, - u"2-a", - {u"2": [ - u"2-b", - u"yielded", - u"2-c", - ]}, - u"1-d", - u"2-d", - ], - ) - - def test_close_generator(self): - @eliot_friendly_generator_function - def g(): - Message.log(message_type=u"a") - try: - yield - Message.log(message_type=u"b") - finally: - Message.log(message_type=u"c") - - - with start_action(action_type=u"the-action"): - gen = g() - next(gen) - gen.close() - - assert_expected_action_tree( - self, - self.eliot_logger, - u"the-action", [ - u"a", - u"yielded", - u"c", - ], - ) - - def test_nested_generators(self): - @eliot_friendly_generator_function - def g(recurse): - with start_action(action_type=u"a-recurse={}".format(recurse)): - Message.log(message_type=u"m-recurse={}".format(recurse)) - if recurse: - set(g(False)) - else: - yield - - with start_action(action_type=u"the-action"): - set(g(True)) - - assert_expected_action_tree( - self, - self.eliot_logger, - u"the-action", [{ - u"a-recurse=True": [ - u"m-recurse=True", { - u"a-recurse=False": [ - u"m-recurse=False", - u"yielded", - ], - }, - ], - }], - ) - - -class InlineCallbacksTests(SyncTestCase): - # Get our custom assertion failure messages *and* the standard ones. - longMessage = True - - def _a_b_test(self, logger, g): - with start_action(action_type=u"the-action"): - self.assertThat(g(), succeeded(Is(None))) - assert_expected_action_tree( - self, - logger, - u"the-action", [ - u"a", - u"yielded", - u"b", - ], - ) - - def test_yield_none(self): - @inline_callbacks - def g(): - Message.log(message_type=u"a") - yield - Message.log(message_type=u"b") - - self._a_b_test(self.eliot_logger, g) - - def test_yield_fired_deferred(self): - @inline_callbacks - def g(): - Message.log(message_type=u"a") - yield succeed(None) - Message.log(message_type=u"b") - - self._a_b_test(self.eliot_logger, g) - - def test_yield_unfired_deferred(self): - waiting = Deferred() - - @inline_callbacks - def g(): - Message.log(message_type=u"a") - yield waiting - Message.log(message_type=u"b") - - with start_action(action_type=u"the-action"): - d = g() - self.assertThat(waiting, has_no_result()) - waiting.callback(None) - self.assertThat(d, succeeded(Is(None))) - assert_expected_action_tree( - self, - self.eliot_logger, - u"the-action", [ - u"a", - u"yielded", - u"b", - ], - ) - - -class ParseDestinationDescriptionTests(SyncTestCase): +class ParseDestinationDescriptionTests(SyncTestCase): """ Tests for ``_parse_destination_description``. """