From 64961232e6addec4abb27835d1b4efd0cdb7bb10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jesper=20Br=C3=A4nn?= Date: Wed, 23 Jun 2021 20:08:24 +0200 Subject: [PATCH] Make it possible to set sequence options (#4210) * Make it possible to set sequence options Attempts to fix #3015 through looking at whether or not the option is of the type Sequence[str]. Treat all deferred options as potentially Sequence options, by making the deferred dict values a list. * Add full test coverage to optmanager again * Document how to set sequence options * minor improvements * update changelog Co-authored-by: Maximilian Hils --- CHANGELOG.md | 1 + mitmproxy/addons/core.py | 2 ++ mitmproxy/optmanager.py | 23 ++++++++++++++--------- mitmproxy/tools/cmdline.py | 2 ++ test/mitmproxy/test_optmanager.py | 14 +++++++++++++- 5 files changed, 32 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ffbae534..19685bd07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,7 @@ Mitmproxy has a completely new proxy core, fixing many longstanding issues: (@mhils) * `json()` method for HTTP Request and Response instances will return decoded JSON body. (@rbdixon) * Support for HTTP/2 Push Promises has been dropped. (@mhils) +* Make it possible to set sequence options from the command line (@Yopi) * --- TODO: add new PRs above this line --- * ... and various other fixes, documentation improvements, dependency version bumps, etc. diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py index 5a9c22b3d..d3e553dbc 100644 --- a/mitmproxy/addons/core.py +++ b/mitmproxy/addons/core.py @@ -56,6 +56,8 @@ class Core: strings and integers are set to None (if permitted), and sequences are emptied. Boolean values can be true, false or toggle. Multiple values are concatenated with a single space. + Sequences are set using multiple invocations to set for + the same option. """ strspec = f"{option}={value}" try: diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 190faabb9..7d4b60636 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -1,3 +1,4 @@ +import collections import contextlib import blinker import blinker._saferef @@ -91,7 +92,7 @@ class OptManager: mutation doesn't change the option state inadvertently. """ def __init__(self): - self.deferred: typing.Dict[str, str] = {} + self.deferred: typing.Dict[str, typing.List[str]] = {} self.changed = blinker.Signal() self.errored = blinker.Signal() # Options must be the last attribute here - after that, we raise an @@ -295,7 +296,7 @@ class OptManager: are added. """ vals = {} - unknown = {} + unknown: typing.Dict[str, typing.List[str]] = collections.defaultdict(list) for i in spec: parts = i.split("=", maxsplit=1) if len(parts) == 1: @@ -303,9 +304,9 @@ class OptManager: else: optname, optval = parts[0], parts[1] if optname in self._options: - vals[optname] = self.parse_setval(self._options[optname], optval) + vals[optname] = self.parse_setval(self._options[optname], optval, vals.get(optname)) else: - unknown[optname] = optval + unknown[optname].append(optval) if defer: self.deferred.update(unknown) elif unknown: @@ -318,15 +319,16 @@ class OptManager: have since been added. """ update = {} - for optname, optval in self.deferred.items(): + for optname, optvals in self.deferred.items(): if optname in self._options: - optval = self.parse_setval(self._options[optname], optval) - update[optname] = optval + for optval in optvals: + optval = self.parse_setval(self._options[optname], optval, update.get(optname)) + update[optname] = optval self.update(**update) for k in update.keys(): del self.deferred[k] - def parse_setval(self, o: _Option, optstr: typing.Optional[str]) -> typing.Any: + def parse_setval(self, o: _Option, optstr: typing.Optional[str], currentvalue: typing.Any) -> typing.Any: """ Convert a string to a value appropriate for the option type. """ @@ -357,7 +359,10 @@ class OptManager: if not optstr: return [] else: - return getattr(self, o.name) + [optstr] + if currentvalue: + return getattr(self, o.name) + currentvalue + [optstr] + else: + return getattr(self, o.name) + [optstr] raise NotImplementedError("Unsupported option type: %s", o.typespec) def make_parser(self, parser, optname, metavar=None, short=None): diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index b345a4ccf..471e02670 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -27,6 +27,8 @@ def common_options(parser, opts): Set an option. When the value is omitted, booleans are set to true, strings and integers are set to None (if permitted), and sequences are emptied. Boolean values can be true, false or toggle. + Sequences are set using multiple invocations to set for + the same option. """ ) parser.add_argument( diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index fe8b59746..96ce68739 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -71,7 +71,7 @@ def test_defaults(): def test_required_int(): o = TO() with pytest.raises(exceptions.OptionsError): - o.parse_setval(o._options["required_int"], None) + o.parse_setval(o._options["required_int"], None, None) def test_deepcopy(): @@ -439,6 +439,9 @@ def test_set(): opts.set("seqstr") assert opts.seqstr == [] + opts.set(*('seqstr=foo', 'seqstr=bar')) + assert opts.seqstr == ["foo", "bar"] + with pytest.raises(exceptions.OptionsError): opts.set("deferredoption=wobble") @@ -450,3 +453,12 @@ def test_set(): opts.process_deferred() assert "deferredoption" not in opts.deferred assert opts.deferredoption == "wobble" + + opts.set(*('deferredsequenceoption=a', 'deferredsequenceoption=b'), defer=True) + assert "deferredsequenceoption" in opts.deferred + opts.process_deferred() + assert "deferredsequenceoption" in opts.deferred + opts.add_option("deferredsequenceoption", typing.Sequence[str], [], "help") + opts.process_deferred() + assert "deferredsequenceoption" not in opts.deferred + assert opts.deferredsequenceoption == ["a", "b"]