diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9aed843c1..dd7c06957 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -66,6 +66,8 @@ jobs: fail-fast: false matrix: include: + - os: ubuntu-latest + py: '3.10.0-rc - 3.10' - os: windows-latest py: 3.9 - os: macos-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 814577805..acf76819b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Release History +## Unreleased: mitmproxy next + +* fix some responses not being decoded properly if the encoding was uppercase #4735 (@Mattwmaster58) +* Windows: Switch to Python's default asyncio event loop, which increases the number of sockets + that can be processed simultaneously. + ## 4 August 2021: mitmproxy 7.0.2 * Fix a WebSocket crash introduced in 7.0.1 (@mhils) diff --git a/examples/addons/websocket-inject-message.py b/examples/addons/websocket-inject-message.py index d6e29d3de..5542879ea 100644 --- a/examples/addons/websocket-inject-message.py +++ b/examples/addons/websocket-inject-message.py @@ -13,9 +13,9 @@ from mitmproxy import ctx, http def websocket_message(flow: http.HTTPFlow): assert flow.websocket is not None # make type checker happy last_message = flow.websocket.messages[-1] - if b"secret" in last_message.content: + if last_message.is_text and "secret" in last_message.text: last_message.drop() - ctx.master.commands.call("inject.websocket", flow, last_message.from_client, "ssssssh") + ctx.master.commands.call("inject.websocket", flow, last_message.from_client, "ssssssh".encode()) # Complex example: Schedule a periodic timer @@ -24,7 +24,7 @@ async def inject_async(flow: http.HTTPFlow): msg = "hello from mitmproxy! " assert flow.websocket is not None # make type checker happy while flow.websocket.timestamp_end is None: - ctx.master.commands.call("inject.websocket", flow, True, msg) + ctx.master.commands.call("inject.websocket", flow, True, msg.encode()) await asyncio.sleep(1) msg = msg[1:] + msg[:1] diff --git a/examples/contrib/har_dump.py b/examples/contrib/har_dump.py index c932154f9..14ecad701 100644 --- a/examples/contrib/har_dump.py +++ b/examples/contrib/har_dump.py @@ -154,7 +154,7 @@ def response(flow: mitmproxy.http.HTTPFlow): } if flow.server_conn.connected: - entry["serverIPAddress"] = str(flow.server_conn.ip_address[0]) + entry["serverIPAddress"] = str(flow.server_conn.peername[0]) HAR["log"]["entries"].append(entry) diff --git a/mitmproxy/__init__.py b/mitmproxy/__init__.py index 9deef9605..e69de29bb 100644 --- a/mitmproxy/__init__.py +++ b/mitmproxy/__init__.py @@ -1,9 +0,0 @@ -import asyncio -import sys - -if sys.platform == 'win32': - # workaround for - # https://github.com/tornadoweb/tornado/issues/2751 - # https://www.tornadoweb.org/en/stable/index.html#installation - # (copied multiple times in the codebase, please remove all occurrences) - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) diff --git a/mitmproxy/io/io.py b/mitmproxy/io/io.py index cfa3fe965..1ac6a8ad3 100644 --- a/mitmproxy/io/io.py +++ b/mitmproxy/io/io.py @@ -46,7 +46,7 @@ class FlowReader: if mdata["type"] not in FLOW_TYPES: raise exceptions.FlowReadException("Unknown flow type: {}".format(mdata["type"])) yield FLOW_TYPES[mdata["type"]].from_state(mdata) - except (ValueError, TypeError) as e: + except (ValueError, TypeError, IndexError) as e: if str(e) == "not a tnetstring: empty file": return # Error is due to EOF raise exceptions.FlowReadException("Invalid data format.") diff --git a/mitmproxy/net/encoding.py b/mitmproxy/net/encoding.py index 9ed2163a4..854931e60 100644 --- a/mitmproxy/net/encoding.py +++ b/mitmproxy/net/encoding.py @@ -52,6 +52,7 @@ def decode( """ if encoded is None: return None + encoding = encoding.lower() global _cache cached = ( @@ -67,7 +68,7 @@ def decode( decoded = custom_decode[encoding](encoded) except KeyError: decoded = codecs.decode(encoded, encoding, errors) # type: ignore - if encoding in ("gzip", "deflate", "br", "zstd"): + if encoding in ("gzip", "deflate", "deflateraw", "br", "zstd"): _cache = CachedDecode(encoded, encoding, errors, decoded) return decoded except TypeError: @@ -108,6 +109,7 @@ def encode(decoded: Union[None, str, bytes], encoding, errors='strict') -> Union """ if decoded is None: return None + encoding = encoding.lower() global _cache cached = ( @@ -123,7 +125,7 @@ def encode(decoded: Union[None, str, bytes], encoding, errors='strict') -> Union encoded = custom_encode[encoding](decoded) except KeyError: encoded = codecs.encode(decoded, encoding, errors) # type: ignore - if encoding in ("gzip", "deflate", "br", "zstd"): + if encoding in ("gzip", "deflate", "deflateraw", "br", "zstd"): _cache = CachedDecode(encoded, encoding, errors, decoded) return encoded except TypeError: @@ -216,7 +218,7 @@ custom_decode = { "identity": identity, "gzip": decode_gzip, "deflate": decode_deflate, - "deflateRaw": decode_deflate, + "deflateraw": decode_deflate, "br": decode_brotli, "zstd": decode_zstd, } @@ -225,7 +227,7 @@ custom_encode = { "identity": identity, "gzip": encode_gzip, "deflate": encode_deflate, - "deflateRaw": encode_deflate, + "deflateraw": encode_deflate, "br": encode_brotli, "zstd": encode_zstd, } diff --git a/mitmproxy/net/tls.py b/mitmproxy/net/tls.py index 469ecce14..f6a2eedaa 100644 --- a/mitmproxy/net/tls.py +++ b/mitmproxy/net/tls.py @@ -289,7 +289,7 @@ class ClientHello: ) @property - def cipher_suites(self): + def cipher_suites(self) -> List[int]: return self._client_hello.cipher_suites.cipher_suites @property diff --git a/mitmproxy/proxy/events.py b/mitmproxy/proxy/events.py index de7025da7..7cbf26661 100644 --- a/mitmproxy/proxy/events.py +++ b/mitmproxy/proxy/events.py @@ -72,9 +72,11 @@ class CommandCompleted(Event): return super().__new__(cls) def __init_subclass__(cls, **kwargs): - command_cls = cls.__annotations__["command"] + command_cls = cls.__annotations__.get("command", None) valid_command_subclass = ( - issubclass(command_cls, commands.Command) and command_cls is not commands.Command + isinstance(command_cls, type) + and issubclass(command_cls, commands.Command) + and command_cls is not commands.Command ) if not valid_command_subclass: warnings.warn(f"{command_cls} needs a properly annotated command attribute.", RuntimeWarning) diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py index bb2cfd492..9e1e55820 100644 --- a/mitmproxy/proxy/layers/tls.py +++ b/mitmproxy/proxy/layers/tls.py @@ -100,7 +100,14 @@ HTTP_ALPNS = (b"h2",) + HTTP1_ALPNS @dataclass class ClientHelloData: context: context.Context + """The context object for this connection.""" + client_hello: net_tls.ClientHello + """The entire parsed TLS ClientHello.""" establish_server_tls_first: bool = False + """ + If set to `True`, pause this handshake and establish TLS with an upstream server first. + This makes it possible to process the server certificate when generating an interception certificate. + """ @dataclass @@ -386,7 +393,7 @@ class ClientTLSLayer(_TLSLayer): self.conn.sni = client_hello.sni self.conn.alpn_offers = client_hello.alpn_protocols - tls_clienthello = ClientHelloData(self.context) + tls_clienthello = ClientHelloData(self.context, client_hello) yield TlsClienthelloHook(tls_clienthello) if tls_clienthello.establish_server_tls_first and not self.context.server.tls_established: diff --git a/setup.py b/setup.py index 01b9d785c..6a1a069eb 100644 --- a/setup.py +++ b/setup.py @@ -100,7 +100,7 @@ setup( "hypothesis>=5.8,<7", "parver>=0.1,<2.0", "pdoc>=4.0.0", - "pyinstaller==4.5", + "pyinstaller==4.5.1", "pytest-asyncio>=0.10.0,<0.16,!=0.14", "pytest-cov>=2.7.1,<3", "pytest-timeout>=1.3.3,<2", @@ -108,7 +108,7 @@ setup( "pytest>=6.1.0,<7", "requests>=2.9.1,<3", "tox>=3.5,<4", - "wheel>=0.36.2,<0.37" + "wheel>=0.36.2,<0.38" ], } ) diff --git a/test/mitmproxy/addons/test_tlsconfig.py b/test/mitmproxy/addons/test_tlsconfig.py index 79e58ef59..cf407c9ed 100644 --- a/test/mitmproxy/addons/test_tlsconfig.py +++ b/test/mitmproxy/addons/test_tlsconfig.py @@ -93,7 +93,7 @@ class TestTlsConfig: ta = tlsconfig.TlsConfig() with taddons.context(ta) as tctx: ctx = context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options) - ch = tls.ClientHelloData(ctx) + ch = tls.ClientHelloData(ctx, None) # type: ignore ta.tls_clienthello(ch) assert not ch.establish_server_tls_first diff --git a/test/mitmproxy/io/test_io.py b/test/mitmproxy/io/test_io.py index ea67cbbe0..8d37dc8f7 100644 --- a/test/mitmproxy/io/test_io.py +++ b/test/mitmproxy/io/test_io.py @@ -11,6 +11,7 @@ from mitmproxy.io import FlowReader, tnetstring class TestFlowReader: @given(binary()) @example(b'51:11:12345678901#4:this,8:true!0:~,4:true!0:]4:\\x00,~}') + @example(b'0:') def test_fuzz(self, data): f = io.BytesIO(data) reader = FlowReader(f) diff --git a/test/mitmproxy/net/test_encoding.py b/test/mitmproxy/net/test_encoding.py index 55d7d6f38..328306e1d 100644 --- a/test/mitmproxy/net/test_encoding.py +++ b/test/mitmproxy/net/test_encoding.py @@ -17,6 +17,7 @@ def test_identity(encoder): @pytest.mark.parametrize("encoder", [ 'gzip', + 'GZIP', 'br', 'deflate', 'zstd', diff --git a/test/mitmproxy/tools/web/test_app.py b/test/mitmproxy/tools/web/test_app.py index 11ac05eb9..15d71b901 100644 --- a/test/mitmproxy/tools/web/test_app.py +++ b/test/mitmproxy/tools/web/test_app.py @@ -4,7 +4,6 @@ import json import json as _json import logging import re -import sys import typing from contextlib import redirect_stdout from pathlib import Path @@ -14,12 +13,6 @@ import pytest from mitmproxy.http import Headers -if sys.platform == 'win32': - # workaround for - # https://github.com/tornadoweb/tornado/issues/2751 - # https://www.tornadoweb.org/en/stable/index.html#installation - # (copied multiple times in the codebase, please remove all occurrences) - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) import tornado.testing # noqa from tornado import httpclient # noqa diff --git a/tox.ini b/tox.ini index 3734473a1..f83aa2f36 100644 --- a/tox.ini +++ b/tox.ini @@ -31,11 +31,11 @@ commands = deps = mypy==0.910 types-certifi==0.1.4 - types-Flask==1.1.1 - types-Werkzeug==1.0.2 - types-requests==2.25.2 - types-cryptography==3.3.3 - types-pyOpenSSL==20.0.4 + types-Flask==1.1.3 + types-Werkzeug==1.0.5 + types-requests==2.25.6 + types-cryptography==3.3.5 + types-pyOpenSSL==20.0.5 commands = mypy {posargs}