From d3f439cfd1f4c56545543f001132b95422fb506c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 4 Nov 2022 10:28:16 +0000 Subject: [PATCH 1/3] make asgi/wsgi apps listen on all ports by default, remove `onboarding_host` --- CHANGELOG.md | 6 ++++++ mitmproxy/addons/asgiapp.py | 8 +++++--- mitmproxy/addons/onboarding.py | 7 +------ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b8b6af0d..74ebc06ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,13 @@ ## Unreleased: mitmproxy next +* ASGI/WSGI apps can now listen on all ports for a specific hostname. + This makes it simpler to accept both HTTP and HTTPS. +### Breaking Changes + +* The `onboarding_port` option has been removed. The onboarding app now responds + to all requests for the hostname specified in `onboarding_host`. ## 02 November 2022: mitmproxy 9.0.1 diff --git a/mitmproxy/addons/asgiapp.py b/mitmproxy/addons/asgiapp.py index 9c9b1ca29..3a077a44e 100644 --- a/mitmproxy/addons/asgiapp.py +++ b/mitmproxy/addons/asgiapp.py @@ -2,6 +2,7 @@ import asyncio import logging import traceback import urllib.parse +from typing import Optional import asgiref.compatibility import asgiref.wsgi @@ -20,7 +21,7 @@ class ASGIApp: - It currently only implements the HTTP protocol (Lifespan and WebSocket are unimplemented). """ - def __init__(self, asgi_app, host: str, port: int): + def __init__(self, asgi_app, host: str, port: Optional[int]): asgi_app = asgiref.compatibility.guarantee_single_callable(asgi_app) self.asgi_app, self.host, self.port = asgi_app, host, port @@ -30,7 +31,8 @@ class ASGIApp: def should_serve(self, flow: http.HTTPFlow) -> bool: return bool( - (flow.request.pretty_host, flow.request.port) == (self.host, self.port) + flow.request.pretty_host == self.host + and (self.port is None or flow.request.port == self.port) and flow.live and not flow.error and not flow.response @@ -42,7 +44,7 @@ class ASGIApp: class WSGIApp(ASGIApp): - def __init__(self, wsgi_app, host: str, port: int): + def __init__(self, wsgi_app, host: str, port: Optional[int]): asgi_app = asgiref.wsgi.WsgiToAsgi(wsgi_app) super().__init__(asgi_app, host, port) diff --git a/mitmproxy/addons/onboarding.py b/mitmproxy/addons/onboarding.py index 272068cf9..78ff11033 100644 --- a/mitmproxy/addons/onboarding.py +++ b/mitmproxy/addons/onboarding.py @@ -3,14 +3,13 @@ from mitmproxy.addons.onboardingapp import app from mitmproxy import ctx APP_HOST = "mitm.it" -APP_PORT = 80 class Onboarding(asgiapp.WSGIApp): name = "onboarding" def __init__(self): - super().__init__(app, APP_HOST, APP_PORT) + super().__init__(app, APP_HOST, None) def load(self, loader): loader.add_option( @@ -25,13 +24,9 @@ class Onboarding(asgiapp.WSGIApp): entry for the app domain is not present. """, ) - loader.add_option( - "onboarding_port", int, APP_PORT, "Port to serve the onboarding app from." - ) def configure(self, updated): self.host = ctx.options.onboarding_host - self.port = ctx.options.onboarding_port app.config["CONFDIR"] = ctx.options.confdir async def request(self, f): From 0ad8630487218f6c851605a0869555b7bf473c88 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 4 Nov 2022 10:41:46 +0000 Subject: [PATCH 2/3] add self-test addon for binaries --- release/build-and-deploy-docker.py | 8 +++--- release/build.py | 24 ++++++++++------- release/selftest.py | 43 ++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 13 deletions(-) create mode 100644 release/selftest.py diff --git a/release/build-and-deploy-docker.py b/release/build-and-deploy-docker.py index 15ee80b4d..0a2d6f56e 100644 --- a/release/build-and-deploy-docker.py +++ b/release/build-and-deploy-docker.py @@ -21,7 +21,7 @@ if ref.startswith("refs/heads/"): elif ref.startswith("refs/tags/"): tag = ref.replace("refs/tags/", "") else: - raise AssertionError + raise AssertionError("Failed to parse $GITHUB_REF") (whl,) = root.glob("release/dist/mitmproxy-*-py3-none-any.whl") docker_build_dir = root / "release/docker" @@ -47,15 +47,17 @@ r = subprocess.run( "docker", "run", "--rm", + "-v", + f"{root / 'release'}:/release" "localtesting", "mitmdump", - "--version", + "-s", "/release/selftest.py", ], check=True, capture_output=True, ) print(r.stdout.decode()) -assert "Mitmproxy: " in r.stdout.decode() +assert "Self-test successful" in r.stdout.decode() # Now we can deploy. subprocess.check_call( diff --git a/release/build.py b/release/build.py index 2f33b3843..c5efff938 100644 --- a/release/build.py +++ b/release/build.py @@ -121,15 +121,13 @@ def standalone_binaries(): with archive(DIST_DIR / f"mitmproxy-{version()}-{operating_system()}") as f: _pyinstaller("standalone.spec") + _test_binaries(TEMP_DIR / "pyinstaller/dist") + for tool in ["mitmproxy", "mitmdump", "mitmweb"]: executable = TEMP_DIR / "pyinstaller/dist" / tool if platform.system() == "Windows": executable = executable.with_suffix(".exe") - # Test if it works at all O:-) - print(f"> {executable} --version") - subprocess.check_call([executable, "--version"]) - f.add(str(executable), str(executable.name)) print(f"Packed {f.name}.") @@ -138,11 +136,21 @@ def _ensure_pyinstaller_onedir(): if not (TEMP_DIR / "pyinstaller/dist/onedir").exists(): _pyinstaller("windows-dir.spec") + _test_binaries(TEMP_DIR / "pyinstaller/dist/onedir") + + +def _test_binaries(binary_directory: Path) -> None: for tool in ["mitmproxy", "mitmdump", "mitmweb"]: + executable = binary_directory / tool + if platform.system() == "Windows": + executable = executable.with_suffix(".exe") + print(f"> {tool} --version") - executable = (TEMP_DIR / "pyinstaller/dist/onedir" / tool).with_suffix(".exe") subprocess.check_call([executable, "--version"]) + print(f"> {tool} -s selftest.py") + subprocess.check_call([executable, "-s", here / "selftest.py"]) + @cli.command() def msix_installer(): @@ -256,11 +264,7 @@ def installbuilder_installer(): subprocess.run( [installer, "--mode", "unattended", "--unattendedmodeui", "none"], check=True ) - MITMPROXY_INSTALL_DIR = Path(rf"C:\Program Files\mitmproxy\bin") - for tool in ["mitmproxy", "mitmdump", "mitmweb"]: - executable = (MITMPROXY_INSTALL_DIR / tool).with_suffix(".exe") - print(f"> {executable} --version") - subprocess.check_call([executable, "--version"]) + _test_binaries(Path(r"C:\Program Files\mitmproxy\bin")) if __name__ == "__main__": diff --git a/release/selftest.py b/release/selftest.py new file mode 100644 index 000000000..6052a5ca8 --- /dev/null +++ b/release/selftest.py @@ -0,0 +1,43 @@ +""" +This addons is used for binaries to perform a minimal selftest. Use like so: + + mitmdump -s selftest.py -p 0 +""" +import asyncio +import logging +import ssl +import sys +from pathlib import Path + +from mitmproxy import ctx + + +def load(_): + # force a random port + ctx.options.listen_port = 0 + + +def running(): + # attach is somewhere so that it's not collected. + ctx.task = asyncio.create_task(make_request()) + + +async def make_request(): + try: + cafile = Path(ctx.options.confdir).expanduser() / "mitmproxy-ca.pem" + ssl_ctx = ssl.create_default_context(cafile=cafile) + port = ctx.master.addons.get("proxyserver").listen_addrs()[0][1] + reader, writer = await asyncio.open_connection( + "127.0.0.1", port, + ssl=ssl_ctx + ) + writer.write(b"GET / HTTP/1.1\r\nHost: mitm.it\r\nConnection: close\r\n\r\n") + await writer.drain() + resp = await reader.read() + if b"This page is served by your local mitmproxy instance" not in resp: + raise RuntimeError(resp) + logging.info("Self-test successful.") + ctx.master.shutdown() + except Exception as e: + print(f"{e!r}") + sys.exit(1) From 3b8e1f6e2af57afeceebeb9d8558def820814200 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 10 Nov 2022 13:23:10 +0100 Subject: [PATCH 3/3] skip self-test for console ui --- release/build.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/release/build.py b/release/build.py index c5efff938..6878a9e7f 100644 --- a/release/build.py +++ b/release/build.py @@ -148,6 +148,9 @@ def _test_binaries(binary_directory: Path) -> None: print(f"> {tool} --version") subprocess.check_call([executable, "--version"]) + if tool == "mitmproxy": + continue # requires a TTY, which we don't have here. + print(f"> {tool} -s selftest.py") subprocess.check_call([executable, "-s", here / "selftest.py"])