build macOS app bundle (#6447)

.github/workflows/main.yml

@@ -81,7 +81,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - image: macos-11
+          - image: macos-12
             platform: macos
           - image: windows-2019
             platform: windows
@@ -104,12 +104,36 @@ jobs:
           path: release/installbuilder/setup
           key: installbuilder
       - run: pip install .[dev]  # pyinstaller 5.9 does not like pyproject.toml + editable installs.
-      - run: python -u release/build.py standalone-binaries
+
+      # macOS x64. Due to GHA limitations, we are currently building the Apple Silicon app bundle outside of CI.
+      - if: matrix.platform == 'macos' && github.repository == 'mitmproxy/mitmproxy'
+            && (startsWith(github.ref, 'refs/heads/') || startsWith(github.ref, 'refs/tags/'))
+        id: keychain
+        uses: apple-actions/import-codesign-certs@5565bb656f60c98c8fc515f3444dd8db73545dc2
+        with:
+          keychain: ${{ runner.temp }}/temp
+          p12-file-base64: ${{ secrets.APPLE_CERTIFICATE }}
+          p12-password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
+      - if: matrix.platform == 'macos' && github.repository == 'mitmproxy/mitmproxy'
+            && (startsWith(github.ref, 'refs/heads/') || startsWith(github.ref, 'refs/tags/'))
+        run: |
+          python -u release/build.py macos-app \
+            --keychain "${{ runner.temp }}/temp.keychain" \
+            --team-id "S8XHQB96PW" \
+            --apple-id "${{ secrets.APPLE_ID }}" \
+            --password "${{ secrets.APPLE_APP_PASSWORD }}"
+
+      # Linux
       - if: matrix.platform == 'linux'
-        run: python -u release/build.py --dirty wheel
+        run: python -u release/build.py standalone-binaries wheel
+
+      # Windows
+      - if: matrix.platform == 'windows'
+        run: python -u release/build.py standalone-binaries
       - if: matrix.platform == 'windows' && github.repository == 'mitmproxy/mitmproxy' &&
             (github.ref == 'refs/heads/citest' || startsWith(github.ref, 'refs/tags/'))
         run: python -u release/build.py --dirty installbuilder-installer msix-installer
+
       - uses: actions/upload-artifact@v3
         with:
           # artifacts must have different names, see https://github.com/actions/upload-artifact/issues/24

CHANGELOG.md

@@ -6,6 +6,11 @@
   ([#6389](https://github.com/mitmproxy/mitmproxy/pull/6389), @mhils)
 * Add a contentview for DNS-over-HTTPS.
   ([#6389](https://github.com/mitmproxy/mitmproxy/pull/6389), @mhils)
+* Replaced standalone mitmproxy binaries on macOS with an app bundle
+  that contains the mitmproxy/mitmweb/mitmdump CLI tools.
+  This change was necessary to support macOS code signing requirements.
+  Homebrew remains the recommended installation method.
+  ([#6447](https://github.com/mitmproxy/mitmproxy/pull/6447), @mhils)
 * Fix certificate generation to work with strict mode OpenSSL 3.x clients
   ([#6410](https://github.com/mitmproxy/mitmproxy/pull/6410), @mmaxim)
 * Fix path() documentation that the return value might include the query string

release/README.md

@@ -4,8 +4,9 @@
 2. Invoke the [release workflow](https://github.com/mitmproxy/mitmproxy/actions/workflows/release.yml) from the GitHub UI.
 3. The spawned workflow runs will require manual confirmation on GitHub which you need to approve twice: 
    https://github.com/mitmproxy/mitmproxy/actions
-4. Once everything has been deployed, update the website.
-5. Verify that the front-page download links for all platforms are working.
+4. Build the macOS ARM binaries outside of CI and upload them to the download server: `./build.py macos-app`.
+5. Once everything has been deployed, update the website.
+6. Verify that the front-page download links for all platforms are working.
 
 ### GitHub Releases
 
@@ -40,10 +41,7 @@
 
 ### Homebrew
 
-- The Homebrew maintainers are typically very fast and detect our new relese
-  within a day.
-- If you feel the need, you can run this from a macOS machine:
-  `brew bump-formula-pr --url https://github.com/mitmproxy/mitmproxy/archive/<version number here>.tar.gz mitmproxy`
+TODO: This is not current and needs to be replaced with Cask instructions.
 
 ### Website
 

release/build.py

@@ -9,10 +9,10 @@ import shutil
 import subprocess
 import tarfile
 import urllib.request
+import warnings
 import zipfile
 from datetime import datetime
 from pathlib import Path
-from typing import Literal
 
 import click
 import cryptography.fernet
@@ -88,16 +88,18 @@ def version() -> str:
     )
 
 
-def operating_system() -> Literal["windows", "linux", "macos", "unknown"]:
-    pf = platform.system()
-    if pf == "Windows":
-        return "windows"
-    elif pf == "Linux":
-        return "linux"
-    elif pf == "Darwin":
-        return "macos"
-    else:
-        return "unknown"
+def operating_system() -> str:
+    match (platform.system(), platform.machine()):
+        case ("Windows", _):
+            return "windows"
+        case ("Linux", _):
+            return "linux"
+        case ("Darwin", "x86_64"):
+            return "macos-x86_64"
+        case ("Darwin", "arm64"):
+            return "macos-arm64"
+    warnings.warn("Unexpected platform.")
+    return f"{platform.system()}-{platform.machine()}"
 
 
 def _pyinstaller(specfile: str) -> None:
@@ -109,7 +111,7 @@ def _pyinstaller(specfile: str) -> None:
             "--workpath",
             TEMP_DIR / "pyinstaller/temp",
             "--distpath",
-            TEMP_DIR / "pyinstaller/dist",
+            TEMP_DIR / "pyinstaller/out",
             specfile,
         ],
         cwd=here / "specs",
@@ -118,14 +120,14 @@ def _pyinstaller(specfile: str) -> None:
 
 @cli.command()
 def standalone_binaries():
-    """All platforms: Build the standalone binaries generated with PyInstaller"""
+    """Windows and Linux: Build the standalone binaries generated with PyInstaller"""
     with archive(DIST_DIR / f"mitmproxy-{version()}-{operating_system()}") as f:
         _pyinstaller("standalone.spec")
 
-        _test_binaries(TEMP_DIR / "pyinstaller/dist")
+        _test_binaries(TEMP_DIR / "pyinstaller/out")
 
         for tool in ["mitmproxy", "mitmdump", "mitmweb"]:
-            executable = TEMP_DIR / "pyinstaller/dist" / tool
+            executable = TEMP_DIR / "pyinstaller/out" / tool
             if platform.system() == "Windows":
                 executable = executable.with_suffix(".exe")
 
@@ -133,11 +135,83 @@ def standalone_binaries():
     print(f"Packed {f.name!r}.")
 
 
-def _ensure_pyinstaller_onedir():
-    if not (TEMP_DIR / "pyinstaller/dist/onedir").exists():
-        _pyinstaller("windows-dir.spec")
+@cli.command()
+@click.option("--keychain")
+@click.option("--team-id")
+@click.option("--apple-id")
+@click.option("--password")
+def macos_app(
+    keychain: str | None,
+    team_id: str | None,
+    apple_id: str | None,
+    password: str | None,
+) -> None:
+    """
+    macOS: Build into mitmproxy.app.
 
-    _test_binaries(TEMP_DIR / "pyinstaller/dist/onedir")
+    If you do not specify options, notarization is skipped.
+    """
+
+    _pyinstaller("onedir.spec")
+    _test_binaries(TEMP_DIR / "pyinstaller/out/mitmproxy.app/Contents/MacOS")
+
+    if keychain:
+        assert isinstance(team_id, str)
+        assert isinstance(apple_id, str)
+        assert isinstance(password, str)
+        # Notarize the app bundle.
+        subprocess.check_call(
+            [
+                "xcrun",
+                "notarytool",
+                "store-credentials",
+                "AC_PASSWORD",
+                *(["--keychain", keychain]),
+                *(["--team-id", team_id]),
+                *(["--apple-id", apple_id]),
+                *(["--password", password]),
+            ]
+        )
+        subprocess.check_call(
+            [
+                "ditto",
+                "-c",
+                "-k",
+                "--keepParent",
+                TEMP_DIR / "pyinstaller/out/mitmproxy.app",
+                TEMP_DIR / "notarize.zip",
+            ]
+        )
+        subprocess.check_call(
+            [
+                "xcrun",
+                "notarytool",
+                "submit",
+                TEMP_DIR / "notarize.zip",
+                *(["--keychain", keychain]),
+                *(["--keychain-profile", "AC_PASSWORD"]),
+                "--wait",
+            ]
+        )
+        # 2023: it's not possible to staple to unix executables.
+        # subprocess.check_call([
+        #     "xcrun",
+        #     "stapler",
+        #     "staple",
+        #     TEMP_DIR / "pyinstaller/out/mitmproxy.app",
+        # ])
+    else:
+        warnings.warn("Notarization skipped.")
+
+    with archive(DIST_DIR / f"mitmproxy-{version()}-{operating_system()}") as f:
+        f.add(str(TEMP_DIR / "pyinstaller/out/mitmproxy.app"), "mitmproxy.app")
+    print(f"Packed {f.name!r}.")
+
+
+def _ensure_pyinstaller_onedir():
+    if not (TEMP_DIR / "pyinstaller/out/onedir").exists():
+        _pyinstaller("onedir.spec")
+        _test_binaries(TEMP_DIR / "pyinstaller/out/onedir")
 
 
 def _test_binaries(binary_directory: Path) -> None:
@@ -162,7 +236,7 @@ def msix_installer():
     _ensure_pyinstaller_onedir()
 
     shutil.copytree(
-        TEMP_DIR / "pyinstaller/dist/onedir",
+        TEMP_DIR / "pyinstaller/out/onedir",
         TEMP_DIR / "msix",
         dirs_exist_ok=True,
     )

release/installbuilder/mitmproxy.xml

@@ -31,7 +31,7 @@
                     <distributionFileList>
                         <distributionFile>
                             <allowWildcards>1</allowWildcards>
-                            <origin>../build/pyinstaller/dist/onedir/*</origin>
+                            <origin>../build/pyinstaller/out/onedir/*</origin>
                         </distributionFile>
                         <distributionFile>
                             <origin>run.ps1</origin>

release/selftest.py

@@ -15,6 +15,10 @@ from mitmproxy import ctx
 def load(_):
     # force a random port
     ctx.options.listen_port = 0
+    try:
+        ctx.options.web_open_browser = False
+    except KeyError:
+        pass
 
 
 def running():

release/specs/.mitmproxy-wrapper

@@ -0,0 +1,3 @@
+#!/bin/bash
+dir=$(cd "$( dirname "${0}")" && pwd )
+open -a Terminal "${dir}/mitmproxy"

release/specs/onedir.spec

@@ -1,4 +1,5 @@
 from pathlib import Path
+import platform
 
 from PyInstaller.building.api import PYZ, EXE, COLLECT
 from PyInstaller.building.build_main import Analysis
@@ -6,6 +7,11 @@ from PyInstaller.building.build_main import Analysis
 here = Path(r".")
 tools = ["mitmproxy", "mitmdump", "mitmweb"]
 
+if platform.system() == "Darwin":
+    icon = "icon.icns"
+else:
+    icon = "icon.ico"
+
 analysis = Analysis(
     tools,
     excludes=["tcl", "tk", "tkinter"],
@@ -25,10 +31,11 @@ for tool in tools:
         name=tool,
         console=True,
         upx=False,
-        icon='icon.ico'
+        icon=icon,
+        codesign_identity='Developer ID Application',
     ))
 
-COLLECT(
+coll = COLLECT(
     *executables,
     analysis.binaries,
     analysis.zipfiles,
@@ -37,3 +44,15 @@ COLLECT(
     upx=False,
     name="onedir"
 )
+
+if platform.system() == "Darwin":
+    from PyInstaller.building.osx import BUNDLE
+    app = BUNDLE(
+        # hack: add dummy executable that opens the terminal,
+        # workaround for https://github.com/pyinstaller/pyinstaller/pull/5419
+        [(".mitmproxy-wrapper", str(here / ".mitmproxy-wrapper"), "EXECUTABLE")],
+        coll,
+        name='mitmproxy.app',
+        icon=icon,
+        bundle_identifier="org.mitmproxy",
+    )