Merge pull request #5725 from mhils/selftest

Add self-test addon for release scripts

CHANGELOG.md

@@ -2,7 +2,13 @@
 
 ## Unreleased: mitmproxy next
 
+* ASGI/WSGI apps can now listen on all ports for a specific hostname. 
+  This makes it simpler to accept both HTTP and HTTPS.
 
+### Breaking Changes
+
+* The `onboarding_port` option has been removed. The onboarding app now responds
+  to all requests for the hostname specified in `onboarding_host`.
 
 ## 02 November 2022: mitmproxy 9.0.1
 

mitmproxy/addons/asgiapp.py

@@ -2,6 +2,7 @@ import asyncio
 import logging
 import traceback
 import urllib.parse
+from typing import Optional
 
 import asgiref.compatibility
 import asgiref.wsgi
@@ -20,7 +21,7 @@ class ASGIApp:
         - It currently only implements the HTTP protocol (Lifespan and WebSocket are unimplemented).
     """
 
-    def __init__(self, asgi_app, host: str, port: int):
+    def __init__(self, asgi_app, host: str, port: Optional[int]):
         asgi_app = asgiref.compatibility.guarantee_single_callable(asgi_app)
         self.asgi_app, self.host, self.port = asgi_app, host, port
 
@@ -30,7 +31,8 @@ class ASGIApp:
 
     def should_serve(self, flow: http.HTTPFlow) -> bool:
         return bool(
-            (flow.request.pretty_host, flow.request.port) == (self.host, self.port)
+            flow.request.pretty_host == self.host
+            and (self.port is None or flow.request.port == self.port)
             and flow.live
             and not flow.error
             and not flow.response
@@ -42,7 +44,7 @@ class ASGIApp:
 
 
 class WSGIApp(ASGIApp):
-    def __init__(self, wsgi_app, host: str, port: int):
+    def __init__(self, wsgi_app, host: str, port: Optional[int]):
         asgi_app = asgiref.wsgi.WsgiToAsgi(wsgi_app)
         super().__init__(asgi_app, host, port)
 

mitmproxy/addons/onboarding.py

@@ -3,14 +3,13 @@ from mitmproxy.addons.onboardingapp import app
 from mitmproxy import ctx
 
 APP_HOST = "mitm.it"
-APP_PORT = 80
 
 
 class Onboarding(asgiapp.WSGIApp):
     name = "onboarding"
 
     def __init__(self):
-        super().__init__(app, APP_HOST, APP_PORT)
+        super().__init__(app, APP_HOST, None)
 
     def load(self, loader):
         loader.add_option(
@@ -25,13 +24,9 @@ class Onboarding(asgiapp.WSGIApp):
             entry for the app domain is not present.
             """,
         )
-        loader.add_option(
-            "onboarding_port", int, APP_PORT, "Port to serve the onboarding app from."
-        )
 
     def configure(self, updated):
         self.host = ctx.options.onboarding_host
-        self.port = ctx.options.onboarding_port
         app.config["CONFDIR"] = ctx.options.confdir
 
     async def request(self, f):

release/build-and-deploy-docker.py

@@ -21,7 +21,7 @@ if ref.startswith("refs/heads/"):
 elif ref.startswith("refs/tags/"):
     tag = ref.replace("refs/tags/", "")
 else:
-    raise AssertionError
+    raise AssertionError("Failed to parse $GITHUB_REF")
 
 (whl,) = root.glob("release/dist/mitmproxy-*-py3-none-any.whl")
 docker_build_dir = root / "release/docker"
@@ -47,15 +47,17 @@ r = subprocess.run(
         "docker",
         "run",
         "--rm",
+        "-v",
+        f"{root / 'release'}:/release"
         "localtesting",
         "mitmdump",
-        "--version",
+        "-s", "/release/selftest.py",
     ],
     check=True,
     capture_output=True,
 )
 print(r.stdout.decode())
-assert "Mitmproxy: " in r.stdout.decode()
+assert "Self-test successful" in r.stdout.decode()
 
 # Now we can deploy.
 subprocess.check_call(

release/build.py

@@ -121,15 +121,13 @@ def standalone_binaries():
     with archive(DIST_DIR / f"mitmproxy-{version()}-{operating_system()}") as f:
         _pyinstaller("standalone.spec")
 
+        _test_binaries(TEMP_DIR / "pyinstaller/dist")
+
         for tool in ["mitmproxy", "mitmdump", "mitmweb"]:
             executable = TEMP_DIR / "pyinstaller/dist" / tool
             if platform.system() == "Windows":
                 executable = executable.with_suffix(".exe")
 
-            # Test if it works at all O:-)
-            print(f"> {executable} --version")
-            subprocess.check_call([executable, "--version"])
-
             f.add(str(executable), str(executable.name))
     print(f"Packed {f.name}.")
 
@@ -138,11 +136,24 @@ def _ensure_pyinstaller_onedir():
     if not (TEMP_DIR / "pyinstaller/dist/onedir").exists():
         _pyinstaller("windows-dir.spec")
 
+    _test_binaries(TEMP_DIR / "pyinstaller/dist/onedir")
+
+
+def _test_binaries(binary_directory: Path) -> None:
     for tool in ["mitmproxy", "mitmdump", "mitmweb"]:
+        executable = binary_directory / tool
+        if platform.system() == "Windows":
+            executable = executable.with_suffix(".exe")
+
         print(f"> {tool} --version")
-        executable = (TEMP_DIR / "pyinstaller/dist/onedir" / tool).with_suffix(".exe")
         subprocess.check_call([executable, "--version"])
 
+        if tool == "mitmproxy":
+            continue  # requires a TTY, which we don't have here.
+
+        print(f"> {tool} -s selftest.py")
+        subprocess.check_call([executable, "-s", here / "selftest.py"])
+
 
 @cli.command()
 def msix_installer():
@@ -256,11 +267,7 @@ def installbuilder_installer():
     subprocess.run(
         [installer, "--mode", "unattended", "--unattendedmodeui", "none"], check=True
     )
-    MITMPROXY_INSTALL_DIR = Path(rf"C:\Program Files\mitmproxy\bin")
-    for tool in ["mitmproxy", "mitmdump", "mitmweb"]:
-        executable = (MITMPROXY_INSTALL_DIR / tool).with_suffix(".exe")
-        print(f"> {executable} --version")
-        subprocess.check_call([executable, "--version"])
+    _test_binaries(Path(r"C:\Program Files\mitmproxy\bin"))
 
 
 if __name__ == "__main__":

release/selftest.py

@@ -0,0 +1,43 @@
+"""
+This addons is used for binaries to perform a minimal selftest. Use like so:
+
+  mitmdump -s selftest.py -p 0
+"""
+import asyncio
+import logging
+import ssl
+import sys
+from pathlib import Path
+
+from mitmproxy import ctx
+
+
+def load(_):
+    # force a random port
+    ctx.options.listen_port = 0
+
+
+def running():
+    # attach is somewhere so that it's not collected.
+    ctx.task = asyncio.create_task(make_request())
+
+
+async def make_request():
+    try:
+        cafile = Path(ctx.options.confdir).expanduser() / "mitmproxy-ca.pem"
+        ssl_ctx = ssl.create_default_context(cafile=cafile)
+        port = ctx.master.addons.get("proxyserver").listen_addrs()[0][1]
+        reader, writer = await asyncio.open_connection(
+            "127.0.0.1", port,
+            ssl=ssl_ctx
+        )
+        writer.write(b"GET / HTTP/1.1\r\nHost: mitm.it\r\nConnection: close\r\n\r\n")
+        await writer.drain()
+        resp = await reader.read()
+        if b"This page is served by your local mitmproxy instance" not in resp:
+            raise RuntimeError(resp)
+        logging.info("Self-test successful.")
+        ctx.master.shutdown()
+    except Exception as e:
+        print(f"{e!r}")
+        sys.exit(1)