diff --git a/docs/.gitignore b/docs/.gitignore index 610ffdf13..0591af60a 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -3,4 +3,3 @@ src/public/ node_modules/ public/ src/resources/_gen/ -src/content/addons-examples.md diff --git a/docs/build.sh b/docs/build.sh index aaa52a2fb..afa73d9c7 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -15,9 +15,5 @@ for script in scripts/* ; do "${script}" > "${output}" done -output="src/content/addons-examples.md" -echo "Generating examples content page into ${output} ..." -./render_examples.py > "${output}" - cd src hugo diff --git a/docs/modd.conf b/docs/modd.conf index c4e0ffd78..fbd9fe6c9 100644 --- a/docs/modd.conf +++ b/docs/modd.conf @@ -1,3 +1,7 @@ +scripts/*.py { + prep: build.sh +} + { daemon: cd src; hugo server -D } diff --git a/docs/render_examples.py b/docs/render_examples.py deleted file mode 100755 index 9c6dea745..000000000 --- a/docs/render_examples.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 - -import os -import textwrap -from pathlib import Path - -print(""" ---- -title: "Examples" -menu: - addons: - weight: 6 ---- - -# Examples of Addons and Scripts - -The most recent set of examples is also available [on our GitHub project](https://github.com/mitmproxy/mitmproxy/tree/master/examples). - -""") - -base = os.path.dirname(os.path.realpath(__file__)) -examples_path = os.path.join(base, 'src/examples/') -pathlist = Path(examples_path).glob('**/*.py') - -examples = [os.path.relpath(str(p), examples_path) for p in sorted(pathlist)] -examples = [p for p in examples if not os.path.basename(p) == '__init__.py'] -examples = [p for p in examples if not os.path.basename(p).startswith('test_')] - -current_dir = None -current_level = 2 -for ex in examples: - if os.path.dirname(ex) != current_dir: - current_dir = os.path.dirname(ex) - sanitized = current_dir.replace('/', '').replace('.', '') - print(" * [Examples: {}]({{{{< relref \"addons-examples#{}\">}}}})".format(current_dir, sanitized)) - - sanitized = ex.replace('/', '').replace('.', '') - print(" * [{}]({{{{< relref \"addons-examples#example-{}\">}}}})".format(os.path.basename(ex), sanitized)) - -current_dir = None -current_level = 2 -for ex in examples: - if os.path.dirname(ex) != current_dir: - current_dir = os.path.dirname(ex) - print("#" * current_level, current_dir) - - print(textwrap.dedent(""" - {} Example: {} - {{{{< example src="{}" lang="py" >}}}} - """.format("#" * (current_level + 1), ex, "examples/" + ex))) diff --git a/docs/scripts/examples.py b/docs/scripts/examples.py new file mode 100755 index 000000000..c0209274e --- /dev/null +++ b/docs/scripts/examples.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import re +from pathlib import Path + +here = Path(__file__).absolute().parent +example_dir = here / ".." / "src" / "examples" / "addons" +examples = example_dir.glob('*.py') + +overview = [] +listings = [] + +for example in examples: + code = example.read_text() + slug = str(example.with_suffix("").relative_to(example_dir)) + slug = re.sub(r"[^a-zA-Z]", "-", slug) + match = re.search(r''' + ^ + (?:[#][^\n]*\n)? # there might be a shebang + """ + \s* + (.+?) + \s* + (?:\n\n|""") # stop on empty line or end of comment + ''', code, re.VERBOSE) + if match: + comment = " — " + match.group(1) + else: + comment = "" + overview.append( + f" * [{example.name}](#{slug}){comment}" + ) + listings.append(f""" +

Example: {example.name}

+ +```python +{code} +``` +""") +print("\n".join(overview)) +print(""" +### Community Examples + +Additional examples contributed by the mitmproxy community can be found +[on GitHub](https://github.com/mitmproxy/mitmproxy/tree/master/examples/contrib). + +""") +print("\n".join(listings)) diff --git a/docs/src/content/addons-events.md b/docs/src/content/addons-events.md index ddf0a99f7..b0c982cb1 100644 --- a/docs/src/content/addons-events.md +++ b/docs/src/content/addons-events.md @@ -13,7 +13,7 @@ receive `Flow` objects as arguments - by modifying these objects, addons can change traffic on the fly. For instance, here is an addon that adds a response header with a count of the number of responses seen: -{{< example src="examples/addons/addheader.py" lang="py" >}} +{{< example src="examples/addons/http-add-header.py" lang="py" >}} ## Supported Events diff --git a/docs/src/content/addons-examples.md b/docs/src/content/addons-examples.md new file mode 100644 index 000000000..eaa4c7557 --- /dev/null +++ b/docs/src/content/addons-examples.md @@ -0,0 +1,11 @@ + +--- +title: "Example Addons" +menu: + addons: + weight: 6 +--- + +# Example Addons + +{{< readfile file="/generated/examples.html" markdown="true" >}} diff --git a/docs/src/content/addons-scripting.md b/docs/src/content/addons-scripting.md index c90a9037e..343f635c1 100644 --- a/docs/src/content/addons-scripting.md +++ b/docs/src/content/addons-scripting.md @@ -13,13 +13,13 @@ module as a whole to be treated as an addon object. This lets us place event handler functions in the module scope. For instance, here is a complete script that adds a header to every request. -{{< example src="examples/addons/scripting-headers.py" lang="py" >}} +{{< example src="examples/addons/scripting-minimal-example.py" lang="py" >}} Here's another example that intercepts requests to a particular URL and sends an arbitrary response instead: -{{< example src="examples/simple/send_reply_from_proxy.py" lang="py" >}} +{{< example src="examples/addons/http-reply-from-proxy.py" lang="py" >}} All events around the HTTP protocol [can be found here]({{< relref "addons-events#http-events">}}). @@ -31,7 +31,7 @@ scripting. The WebSocket protocol initially looks like a regular HTTP request, before the client and server agree to upgrade the connection to WebSocket. All scripting events for initial HTTP handshake, and also the dedicated WebSocket events [can be found here]({{< relref "addons-events#websocket-events">}}). -{{< example src="examples/simple/websocket_messages.py" lang="py" >}} +{{< example src="examples/addons/websocket-simple.py" lang="py" >}} For WebSocket-related objects please look at the [websocket][] module to find all attributes that you can use when scripting. @@ -43,7 +43,7 @@ all attributes that you can use when scripting. All events around the TCP protocol [can be found here]({{< relref "addons-events#tcp-events">}}). -{{< example src="examples/complex/tcp_message.py" lang="py" >}} +{{< example src="examples/addons/tcp-simple.py" lang="py" >}} For WebSocket-related objects please look at the [tcp][] module to find all attributes that you can use when scripting. diff --git a/docs/src/content/overview-features.md b/docs/src/content/overview-features.md index cf935adb3..4ee636aad 100644 --- a/docs/src/content/overview-features.md +++ b/docs/src/content/overview-features.md @@ -189,7 +189,7 @@ You can also use a script to customise exactly which requests or responses are streamed. Requests/Responses that should be tagged for streaming by setting their ``.stream`` attribute to ``True``: -{{< example src="examples/complex/stream.py" lang="py" >}} +{{< example src="examples/addons/http-stream-simple.py" lang="py" >}} ### Websockets diff --git a/examples/README.md b/examples/README.md index f46f322d0..6ee8187cd 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,15 +1,9 @@ -# Mitmproxy Scripting API +# Mitmproxy Examples Mitmproxy has a powerful scripting API that allows you to control almost any aspect of traffic being proxied. In fact, much of mitmproxy’s own core functionality is implemented using the exact same API -exposed to scripters (see [mitmproxy/addons](../mitmproxy/addons)). + (see [mitmproxy/addons](../mitmproxy/addons)). -This directory contains some examples of the scripting API. We recommend to start with the -ones in [simple/](./simple). | :warning: | If you are browsing this on GitHub, make sure to select the git tag matching your mitmproxy version. | |------------|------------------------------------------------------------------------------------------------------| - - -Some inline scripts may require additional dependencies, which can be installed using -`pip install mitmproxy[examples]`. \ No newline at end of file diff --git a/examples/addons/anatomy.py b/examples/addons/anatomy.py index c60afeaa4..ffe072008 100644 --- a/examples/addons/anatomy.py +++ b/examples/addons/anatomy.py @@ -1,3 +1,8 @@ +""" +Basic skeleton of a mitmproxy addon. + +Run as follows: mitmproxy -s anatomy.py +""" from mitmproxy import ctx diff --git a/examples/addons/commands-flows.py b/examples/addons/commands-flows.py index cebc8f9da..0cdd06c01 100644 --- a/examples/addons/commands-flows.py +++ b/examples/addons/commands-flows.py @@ -1,3 +1,4 @@ +"""Handle flows as command arguments.""" import typing from mitmproxy import command @@ -6,9 +7,6 @@ from mitmproxy import flow class MyAddon: - def __init__(self): - self.num = 0 - @command.command("myaddon.addheader") def addheader(self, flows: typing.Sequence[flow.Flow]) -> None: for f in flows: diff --git a/examples/addons/commands-paths.py b/examples/addons/commands-paths.py index 4d9535b92..109c3dfb1 100644 --- a/examples/addons/commands-paths.py +++ b/examples/addons/commands-paths.py @@ -1,3 +1,4 @@ +"""Handle file paths as command arguments.""" import typing from mitmproxy import command @@ -7,9 +8,6 @@ from mitmproxy import types class MyAddon: - def __init__(self): - self.num = 0 - @command.command("myaddon.histogram") def histogram( self, diff --git a/examples/addons/commands-simple.py b/examples/addons/commands-simple.py index c9cd63414..3ff857a27 100644 --- a/examples/addons/commands-simple.py +++ b/examples/addons/commands-simple.py @@ -1,3 +1,4 @@ +"""Add a custom command to mitmproxy's command prompt.""" from mitmproxy import command from mitmproxy import ctx @@ -9,7 +10,7 @@ class MyAddon: @command.command("myaddon.inc") def inc(self) -> None: self.num += 1 - ctx.log.info("num = %s" % self.num) + ctx.log.info(f"num = {self.num}") addons = [ diff --git a/examples/simple/custom_contentview.py b/examples/addons/contentview.py similarity index 80% rename from examples/simple/custom_contentview.py rename to examples/addons/contentview.py index 77d324740..0b24daf77 100644 --- a/examples/simple/custom_contentview.py +++ b/examples/addons/contentview.py @@ -1,5 +1,8 @@ """ -This example shows how one can add a custom contentview to mitmproxy. +Add a custom message body pretty-printer for use inside mitmproxy. + +This example shows how one can add a custom contentview to mitmproxy, +which is used to pretty-print HTTP bodies for example. The content view API is explained in the mitmproxy.contentviews module. """ from mitmproxy import contentviews diff --git a/examples/complex/dup_and_replay.py b/examples/addons/duplicate-modify-replay.py similarity index 86% rename from examples/complex/dup_and_replay.py rename to examples/addons/duplicate-modify-replay.py index 3ad98dc51..6ea254724 100644 --- a/examples/complex/dup_and_replay.py +++ b/examples/addons/duplicate-modify-replay.py @@ -1,3 +1,4 @@ +"""Take incoming HTTP requests and replay them with modified parameters.""" from mitmproxy import ctx diff --git a/examples/addons/events-http-specific.py b/examples/addons/events-http-specific.py index 37d9f91ab..8e101fce0 100644 --- a/examples/addons/events-http-specific.py +++ b/examples/addons/events-http-specific.py @@ -1,8 +1,8 @@ +"""HTTP-specific events.""" import mitmproxy.http class Events: - # HTTP lifecycle def http_connect(self, flow: mitmproxy.http.HTTPFlow): """ An HTTP CONNECT request was received. Setting a non 2xx response on diff --git a/examples/addons/events-tcp-specific.py b/examples/addons/events-tcp-specific.py index d150d0f93..f5a577b85 100644 --- a/examples/addons/events-tcp-specific.py +++ b/examples/addons/events-tcp-specific.py @@ -1,8 +1,8 @@ +"""TCP-specific events.""" import mitmproxy.tcp class Events: - # TCP lifecycle def tcp_start(self, flow: mitmproxy.tcp.TCPFlow): """ A TCP connection has started. diff --git a/examples/addons/events-websocket-specific.py b/examples/addons/events-websocket-specific.py index 60069fdb0..17cbd0795 100644 --- a/examples/addons/events-websocket-specific.py +++ b/examples/addons/events-websocket-specific.py @@ -1,3 +1,4 @@ +"""WebSocket-specific events.""" import mitmproxy.http import mitmproxy.websocket diff --git a/examples/addons/events.py b/examples/addons/events.py index 958e7d39c..0e7526a87 100644 --- a/examples/addons/events.py +++ b/examples/addons/events.py @@ -1,11 +1,9 @@ +"""Generic event hooks.""" import typing import mitmproxy.addonmanager import mitmproxy.connections -import mitmproxy.http import mitmproxy.log -import mitmproxy.tcp -import mitmproxy.websocket import mitmproxy.proxy.protocol diff --git a/examples/simple/filter_flows.py b/examples/addons/filter-flows.py similarity index 88% rename from examples/simple/filter_flows.py rename to examples/addons/filter-flows.py index 94d179d09..a4e09ba3f 100644 --- a/examples/simple/filter_flows.py +++ b/examples/addons/filter-flows.py @@ -1,5 +1,5 @@ """ -This script demonstrates how to use mitmproxy's filter pattern in scripts. +Use mitmproxy's filter pattern in scripts. """ from mitmproxy import flowfilter from mitmproxy import ctx, http diff --git a/examples/addons/addheader.py b/examples/addons/http-add-header.py similarity index 82% rename from examples/addons/addheader.py rename to examples/addons/http-add-header.py index f4b29268e..7badacf44 100644 --- a/examples/addons/addheader.py +++ b/examples/addons/http-add-header.py @@ -1,3 +1,5 @@ +"""Add an HTTP header to each response.""" + class AddHeader: def __init__(self): diff --git a/examples/simple/modify_form.py b/examples/addons/http-modify-form.py similarity index 92% rename from examples/simple/modify_form.py rename to examples/addons/http-modify-form.py index 8742a976f..d296c4445 100644 --- a/examples/simple/modify_form.py +++ b/examples/addons/http-modify-form.py @@ -1,3 +1,4 @@ +"""Modify an HTTP form submission.""" from mitmproxy import http diff --git a/examples/simple/modify_querystring.py b/examples/addons/http-modify-query-string.py similarity index 76% rename from examples/simple/modify_querystring.py rename to examples/addons/http-modify-query-string.py index 12b16fda7..0139769d1 100644 --- a/examples/simple/modify_querystring.py +++ b/examples/addons/http-modify-query-string.py @@ -1,3 +1,4 @@ +"""Modify HTTP query parameters.""" from mitmproxy import http diff --git a/examples/simple/redirect_requests.py b/examples/addons/http-redirect-requests.py similarity index 81% rename from examples/simple/redirect_requests.py rename to examples/addons/http-redirect-requests.py index ddb89961f..c5908aa49 100644 --- a/examples/simple/redirect_requests.py +++ b/examples/addons/http-redirect-requests.py @@ -1,6 +1,4 @@ -""" -This example shows two ways to redirect flows to another server. -""" +"""Redirect HTTP requests to another server.""" from mitmproxy import http diff --git a/examples/simple/send_reply_from_proxy.py b/examples/addons/http-reply-from-proxy.py similarity index 55% rename from examples/simple/send_reply_from_proxy.py rename to examples/addons/http-reply-from-proxy.py index 5011fd2e5..303f416d4 100644 --- a/examples/simple/send_reply_from_proxy.py +++ b/examples/addons/http-reply-from-proxy.py @@ -1,14 +1,8 @@ -""" -This example shows how to send a reply from the proxy immediately -without sending any data to the remote server. -""" +"""Send a reply from the proxy without sending any data to the remote server.""" from mitmproxy import http def request(flow: http.HTTPFlow) -> None: - # pretty_url takes the "Host" header of the request into account, which - # is useful in transparent mode where we usually only have the IP otherwise. - if flow.request.pretty_url == "http://example.com/path": flow.response = http.HTTPResponse.make( 200, # (optional) status code diff --git a/examples/complex/stream_modify.py b/examples/addons/http-stream-modify.py similarity index 75% rename from examples/complex/stream_modify.py rename to examples/addons/http-stream-modify.py index 46bdcb78d..ad7522872 100644 --- a/examples/complex/stream_modify.py +++ b/examples/addons/http-stream-modify.py @@ -1,7 +1,8 @@ """ -This inline script modifies a streamed response. -If you do not need streaming, see the modify_response_body example. -Be aware that content replacement isn't trivial: +Modify a streamed response. + +Generally speaking, we recommend *not* to stream messages you need to modify. +Modifying streamed responses is tricky and brittle: - If the transfer encoding isn't chunked, you cannot simply change the content length. - If you want to replace all occurrences of "foobar", make sure to catch the cases where one chunk ends with [...]foo" and the next starts with "bar[...]. diff --git a/examples/complex/stream.py b/examples/addons/http-stream-simple.py similarity index 52% rename from examples/complex/stream.py rename to examples/addons/http-stream-simple.py index ae365ec5b..454f4792e 100644 --- a/examples/complex/stream.py +++ b/examples/addons/http-stream-simple.py @@ -1,3 +1,11 @@ +""" +Select which responses should be streamed. + +Enable response streaming for all HTTP flows. +This is equivalent to passing `--set stream_large_bodies=1` to mitmproxy. +""" + + def responseheaders(flow): """ Enables streaming for all responses. diff --git a/examples/simple/internet_in_mirror.py b/examples/addons/internet_in_mirror.py similarity index 79% rename from examples/simple/internet_in_mirror.py rename to examples/addons/internet_in_mirror.py index 5d3e555d4..7ee1821d1 100644 --- a/examples/simple/internet_in_mirror.py +++ b/examples/addons/internet_in_mirror.py @@ -1,5 +1,7 @@ """ -This script reflects all content passing through the proxy. +Mirror all web pages. + +Useful if you are living down under. """ from mitmproxy import http diff --git a/examples/simple/io_read_dumpfile.py b/examples/addons/io-read-saved-flows.py similarity index 88% rename from examples/simple/io_read_dumpfile.py rename to examples/addons/io-read-saved-flows.py index 534f357b9..76fa1ff28 100644 --- a/examples/simple/io_read_dumpfile.py +++ b/examples/addons/io-read-saved-flows.py @@ -1,13 +1,12 @@ #!/usr/bin/env python -# -# Simple script showing how to read a mitmproxy dump file -# +""" +Read a mitmproxy dump file. +""" from mitmproxy import io from mitmproxy.exceptions import FlowReadException import pprint import sys - with open(sys.argv[1], "rb") as logfile: freader = io.FlowReader(logfile) pp = pprint.PrettyPrinter(indent=4) diff --git a/examples/simple/io_write_dumpfile.py b/examples/addons/io-write-flow-file.py similarity index 95% rename from examples/simple/io_write_dumpfile.py rename to examples/addons/io-write-flow-file.py index 5fcb07290..e885051d4 100644 --- a/examples/simple/io_write_dumpfile.py +++ b/examples/addons/io-write-flow-file.py @@ -1,4 +1,6 @@ """ +Generate a mitmproxy dump file. + This script demonstrates how to generate a mitmproxy dump file, as it would also be generated by passing `-w` to mitmproxy. In contrast to `-w`, this gives you full control over which diff --git a/examples/simple/log_events.py b/examples/addons/log-events.py similarity index 78% rename from examples/simple/log_events.py rename to examples/addons/log-events.py index 4f70e340d..c31d6bc50 100644 --- a/examples/simple/log_events.py +++ b/examples/addons/log-events.py @@ -1,3 +1,4 @@ +"""Post messages to mitmproxy's event log.""" from mitmproxy import ctx diff --git a/examples/complex/nonblocking.py b/examples/addons/nonblocking.py similarity index 59% rename from examples/complex/nonblocking.py rename to examples/addons/nonblocking.py index dd74aec07..4b17657b6 100644 --- a/examples/complex/nonblocking.py +++ b/examples/addons/nonblocking.py @@ -1,3 +1,9 @@ +""" +Make events hooks non-blocking. + +When event hooks are decorated with @concurrent, they will be run in their own thread, freeing the main event loop. +Please note that this generally opens the door to race conditions and decreases performance if not required. +""" import time from mitmproxy.script import concurrent diff --git a/examples/addons/options-configure.py b/examples/addons/options-configure.py index c7638e87f..39e982ba9 100644 --- a/examples/addons/options-configure.py +++ b/examples/addons/options-configure.py @@ -1,3 +1,4 @@ +"""React to configuration changes.""" import typing from mitmproxy import ctx diff --git a/examples/addons/options-simple.py b/examples/addons/options-simple.py index 0acefb3f7..4eb8d54d9 100644 --- a/examples/addons/options-simple.py +++ b/examples/addons/options-simple.py @@ -1,3 +1,10 @@ +""" +Add a new mitmproxy option. + +Usage: + + mitmproxy -s options-simple.py --set addheader true +""" from mitmproxy import ctx diff --git a/examples/addons/scripting-headers.py b/examples/addons/scripting-minimal-example.py similarity index 100% rename from examples/addons/scripting-headers.py rename to examples/addons/scripting-minimal-example.py diff --git a/examples/addons/tcp-simple.py b/examples/addons/tcp-simple.py new file mode 100644 index 000000000..8fc3bccc8 --- /dev/null +++ b/examples/addons/tcp-simple.py @@ -0,0 +1,24 @@ +""" +Process individual messages from a TCP connection. + +This script replaces full occurences of "foo" with "bar" and prints various details for each message. +Please note that TCP is stream-based and *not* message-based. mitmproxy splits stream contents into "messages" +as they are received by socket.recv(). This is pretty arbitrary and should not be relied on. +However, it is sometimes good enough as a quick hack. + +Example Invocation: + + mitmdump --rawtcp --tcp-hosts ".*" -s examples/tcp-simple.py +""" +from mitmproxy.utils import strutils +from mitmproxy import ctx +from mitmproxy import tcp + + +def tcp_message(flow: tcp.TCPFlow): + message = flow.messages[-1] + message.content = message.content.replace(b"foo", b"bar") + + ctx.log.info( + f"tcp_message[from_client={message.from_client}), content={strutils.bytes_to_escaped_str(message.content)}]" + ) diff --git a/examples/complex/websocket_inject_message.py b/examples/addons/websocket-inject-message.py similarity index 80% rename from examples/complex/websocket_inject_message.py rename to examples/addons/websocket-inject-message.py index 38be55557..3999be3b6 100644 --- a/examples/complex/websocket_inject_message.py +++ b/examples/addons/websocket-inject-message.py @@ -1,4 +1,6 @@ """ +Inject a WebSocket message into a running connection. + This example shows how to inject a WebSocket message to the client. Every new WebSocket connection will trigger a new asyncio task that periodically injects a new message to the client. @@ -8,12 +10,11 @@ import mitmproxy.websocket class InjectWebSocketMessage: - async def inject(self, flow: mitmproxy.websocket.WebSocketFlow): i = 0 while not flow.ended and not flow.error: await asyncio.sleep(5) - flow.inject_message(flow.client_conn, 'This is the #{} injected message!'.format(i)) + flow.inject_message(flow.client_conn, f'This is the #{i} injected message!') i += 1 def websocket_start(self, flow): diff --git a/examples/simple/websocket_messages.py b/examples/addons/websocket-simple.py similarity index 90% rename from examples/simple/websocket_messages.py rename to examples/addons/websocket-simple.py index 071ea21fd..b62623672 100644 --- a/examples/simple/websocket_messages.py +++ b/examples/addons/websocket-simple.py @@ -1,3 +1,4 @@ +"""Process individual messages from a WebSocket connection.""" import re from mitmproxy import ctx diff --git a/examples/simple/wsgi_flask_app.py b/examples/addons/wsgi-flask-app.py similarity index 96% rename from examples/simple/wsgi_flask_app.py rename to examples/addons/wsgi-flask-app.py index b34fbc837..2a9f0e2b7 100644 --- a/examples/simple/wsgi_flask_app.py +++ b/examples/addons/wsgi-flask-app.py @@ -1,4 +1,6 @@ """ +Host a WSGI app in mitmproxy. + This example shows how to graft a WSGI app onto mitmproxy. In this instance, we're using the Flask framework (http://flask.pocoo.org/) to expose a single simplest-possible page. diff --git a/examples/complex/README.md b/examples/complex/README.md deleted file mode 100644 index 923aadf16..000000000 --- a/examples/complex/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## Complex Examples - -| Filename | Description | -|:-------------------------|:----------------------------------------------------------------------------------------------| -| block_dns_over_https.py | Use mitmproxy to block DNS over HTTPS (DoH) queries | -| change_upstream_proxy.py | Dynamically change the upstream proxy. | -| dns_spoofing.py | Use mitmproxy in a DNS spoofing scenario. | -| dup_and_replay.py | Duplicates each request, changes it, and then replays the modified request. | -| full_transparency_shim.c | Setuid wrapper that can be used to run mitmproxy in full transparency mode, as a normal user. | -| har_dump.py | Dump flows as HAR files. | -| mitmproxywrapper.py | Bracket mitmproxy run with proxy enable/disable on OS X | -| nonblocking.py | Demonstrate parallel processing with a blocking script | -| remote_debug.py | This script enables remote debugging of the mitmproxy _UI_ with PyCharm. | -| sslstrip.py | sslstrip-like functionality implemented with mitmproxy | -| stream.py | Enable streaming for all responses. | -| stream_modify.py | Modify a streamed response body. | -| tcp_message.py | Modify a raw TCP connection | -| tls_passthrough.py | Use conditional TLS interception based on a user-defined strategy. | -| xss_scanner.py | Scan all visited webpages. | diff --git a/examples/complex/__init__.py b/examples/complex/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/complex/tcp_message.py b/examples/complex/tcp_message.py deleted file mode 100644 index b1311d08e..000000000 --- a/examples/complex/tcp_message.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -tcp_message Inline Script Hook API Demonstration ------------------------------------------------- - -* modifies packets containing "foo" to "bar" -* prints various details for each packet. - -example cmdline invocation: -mitmdump --rawtcp --tcp-host ".*" -s examples/complex/tcp_message.py -""" -from mitmproxy.utils import strutils -from mitmproxy import ctx -from mitmproxy import tcp - - -def tcp_message(flow: tcp.TCPFlow): - message = flow.messages[-1] - old_content = message.content - message.content = old_content.replace(b"foo", b"bar") - - ctx.log.info( - "[tcp_message{}] from {} to {}:\n{}".format( - " (modified)" if message.content != old_content else "", - "client" if message.from_client else "server", - "server" if message.from_client else "client", - strutils.bytes_to_escaped_str(message.content)) - ) diff --git a/examples/complex/webscanner_helper/__init__.py b/examples/complex/webscanner_helper/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/contrib/README.md b/examples/contrib/README.md new file mode 100644 index 000000000..94580dbc1 --- /dev/null +++ b/examples/contrib/README.md @@ -0,0 +1,4 @@ +# Community-Contributed Examples + +Examples in this directory are contributed by the mitmproxy community. +We do _not_ maintain them, but we welcome PRs that add/fix/modernize/clean up examples. \ No newline at end of file diff --git a/examples/complex/block_dns_over_https.py b/examples/contrib/block_dns_over_https.py similarity index 100% rename from examples/complex/block_dns_over_https.py rename to examples/contrib/block_dns_over_https.py diff --git a/examples/complex/change_upstream_proxy.py b/examples/contrib/change_upstream_proxy.py similarity index 100% rename from examples/complex/change_upstream_proxy.py rename to examples/contrib/change_upstream_proxy.py diff --git a/examples/complex/dns_spoofing.py b/examples/contrib/dns_spoofing.py similarity index 100% rename from examples/complex/dns_spoofing.py rename to examples/contrib/dns_spoofing.py diff --git a/examples/complex/full_transparency_shim.c b/examples/contrib/full_transparency_shim.c similarity index 100% rename from examples/complex/full_transparency_shim.c rename to examples/contrib/full_transparency_shim.c diff --git a/examples/complex/har_dump.py b/examples/contrib/har_dump.py similarity index 100% rename from examples/complex/har_dump.py rename to examples/contrib/har_dump.py diff --git a/examples/simple/link_expander.py b/examples/contrib/link_expander.py similarity index 100% rename from examples/simple/link_expander.py rename to examples/contrib/link_expander.py diff --git a/examples/complex/mitmproxywrapper.py b/examples/contrib/mitmproxywrapper.py similarity index 100% rename from examples/complex/mitmproxywrapper.py rename to examples/contrib/mitmproxywrapper.py diff --git a/examples/simple/modify_body_inject_iframe.py b/examples/contrib/modify_body_inject_iframe.py similarity index 100% rename from examples/simple/modify_body_inject_iframe.py rename to examples/contrib/modify_body_inject_iframe.py diff --git a/examples/complex/remote_debug.py b/examples/contrib/remote-debug.py similarity index 90% rename from examples/complex/remote_debug.py rename to examples/contrib/remote-debug.py index 5129c9dbf..767d828cd 100644 --- a/examples/complex/remote_debug.py +++ b/examples/contrib/remote-debug.py @@ -1,5 +1,5 @@ """ -This script enables remote debugging of the mitmproxy *UI* with PyCharm. +This script enables remote debugging of the mitmproxy console *UI* with PyCharm. For general debugging purposes, it is easier to just debug mitmdump within PyCharm. Usage: diff --git a/examples/complex/sslstrip.py b/examples/contrib/sslstrip.py similarity index 100% rename from examples/complex/sslstrip.py rename to examples/contrib/sslstrip.py diff --git a/test/examples/test_har_dump.py b/examples/contrib/test_har_dump.py similarity index 100% rename from test/examples/test_har_dump.py rename to examples/contrib/test_har_dump.py diff --git a/test/examples/test_xss_scanner.py b/examples/contrib/test_xss_scanner.py similarity index 100% rename from test/examples/test_xss_scanner.py rename to examples/contrib/test_xss_scanner.py diff --git a/examples/complex/tls_passthrough.py b/examples/contrib/tls_passthrough.py similarity index 100% rename from examples/complex/tls_passthrough.py rename to examples/contrib/tls_passthrough.py diff --git a/examples/__init__.py b/examples/contrib/webscanner_helper/__init__.py similarity index 100% rename from examples/__init__.py rename to examples/contrib/webscanner_helper/__init__.py diff --git a/examples/complex/webscanner_helper/mapping.py b/examples/contrib/webscanner_helper/mapping.py similarity index 100% rename from examples/complex/webscanner_helper/mapping.py rename to examples/contrib/webscanner_helper/mapping.py diff --git a/test/examples/webscanner_helper/test_mapping.py b/examples/contrib/webscanner_helper/test_mapping.py similarity index 100% rename from test/examples/webscanner_helper/test_mapping.py rename to examples/contrib/webscanner_helper/test_mapping.py diff --git a/test/examples/webscanner_helper/test_urldict.py b/examples/contrib/webscanner_helper/test_urldict.py similarity index 100% rename from test/examples/webscanner_helper/test_urldict.py rename to examples/contrib/webscanner_helper/test_urldict.py diff --git a/test/examples/webscanner_helper/test_urlindex.py b/examples/contrib/webscanner_helper/test_urlindex.py similarity index 100% rename from test/examples/webscanner_helper/test_urlindex.py rename to examples/contrib/webscanner_helper/test_urlindex.py diff --git a/test/examples/webscanner_helper/test_urlinjection.py b/examples/contrib/webscanner_helper/test_urlinjection.py similarity index 100% rename from test/examples/webscanner_helper/test_urlinjection.py rename to examples/contrib/webscanner_helper/test_urlinjection.py diff --git a/test/examples/webscanner_helper/test_watchdog.py b/examples/contrib/webscanner_helper/test_watchdog.py similarity index 100% rename from test/examples/webscanner_helper/test_watchdog.py rename to examples/contrib/webscanner_helper/test_watchdog.py diff --git a/examples/complex/webscanner_helper/urldict.py b/examples/contrib/webscanner_helper/urldict.py similarity index 100% rename from examples/complex/webscanner_helper/urldict.py rename to examples/contrib/webscanner_helper/urldict.py diff --git a/examples/complex/webscanner_helper/urlindex.py b/examples/contrib/webscanner_helper/urlindex.py similarity index 100% rename from examples/complex/webscanner_helper/urlindex.py rename to examples/contrib/webscanner_helper/urlindex.py diff --git a/examples/complex/webscanner_helper/urlinjection.py b/examples/contrib/webscanner_helper/urlinjection.py similarity index 100% rename from examples/complex/webscanner_helper/urlinjection.py rename to examples/contrib/webscanner_helper/urlinjection.py diff --git a/examples/complex/webscanner_helper/watchdog.py b/examples/contrib/webscanner_helper/watchdog.py similarity index 100% rename from examples/complex/webscanner_helper/watchdog.py rename to examples/contrib/webscanner_helper/watchdog.py diff --git a/examples/complex/xss_scanner.py b/examples/contrib/xss_scanner.py old mode 100755 new mode 100644 similarity index 100% rename from examples/complex/xss_scanner.py rename to examples/contrib/xss_scanner.py diff --git a/examples/simple/README.md b/examples/simple/README.md deleted file mode 100644 index 66a05b30b..000000000 --- a/examples/simple/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## Simple Examples - -| Filename | Description | -| :----------------------------- | :--------------------------------------------------------------------------- | -| add_header.py | Simple script that just adds a header to every request. | -| custom_contentview.py | Add a custom content view to the mitmproxy UI. | -| custom_option.py | Add arguments to a script. | -| filter_flows.py | This script demonstrates how to use mitmproxy's filter pattern in scripts. | -| io_read_dumpfile.py | Read a dumpfile generated by mitmproxy. | -| io_write_dumpfile.py | Only write selected flows into a mitmproxy dumpfile. | -| link_expander.py | Discover relative links in HTML traffic and replace them with absolute paths | -| log_events.py | Use mitmproxy's logging API. | -| modify_body_inject_iframe.py | Inject configurable iframe into pages. | -| modify_form.py | Modify HTTP form submissions. | -| modify_querystring.py | Modify HTTP query strings. | -| redirect_requests.py | Redirect a request to a different server. | -| send_reply_from_proxy.py | Send a HTTP response directly from the proxy. | -| internet_in_mirror.py | Turn all images upside down. | -| wsgi_flask_app.py | Embed a WSGI app into mitmproxy. | diff --git a/examples/simple/add_header.py b/examples/simple/add_header.py deleted file mode 100644 index 64fc6267a..000000000 --- a/examples/simple/add_header.py +++ /dev/null @@ -1,5 +0,0 @@ -from mitmproxy import http - - -def response(flow: http.HTTPFlow) -> None: - flow.response.headers["newheader"] = "foo" diff --git a/examples/simple/add_header_class.py b/examples/simple/add_header_class.py deleted file mode 100644 index 419c99ac2..000000000 --- a/examples/simple/add_header_class.py +++ /dev/null @@ -1,9 +0,0 @@ -from mitmproxy import http - - -class AddHeader: - def response(self, flow: http.HTTPFlow) -> None: - flow.response.headers["newheader"] = "foo" - - -addons = [AddHeader()] diff --git a/examples/simple/custom_option.py b/examples/simple/custom_option.py deleted file mode 100644 index 8d0cfe7f5..000000000 --- a/examples/simple/custom_option.py +++ /dev/null @@ -1,21 +0,0 @@ -""" -This example shows how addons can register custom options -that can be configured at startup or during execution -from the options dialog within mitmproxy. - -Example: - -$ mitmproxy --set custom=true -$ mitmproxy --set custom # shorthand for boolean options -""" -from mitmproxy import ctx - - -def load(l): - ctx.log.info("Registering option 'custom'") - l.add_option("custom", bool, False, "A custom option") - - -def configure(updated): - if "custom" in updated: - ctx.log.info("custom option value: %s" % ctx.options.custom) diff --git a/requirements.txt b/requirements.txt index 28a0b4957..aefbcb6d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ --e .[dev,examples] +-e .[dev] diff --git a/setup.py b/setup.py index 07e828633..4ce2ac520 100644 --- a/setup.py +++ b/setup.py @@ -102,9 +102,6 @@ setup( "pytest>=5.1.3,<6", "requests>=2.9.1,<3", "tox>=3.5,<3.15", - ], - 'examples': [ - "beautifulsoup4>=4.4.1,<4.9" ] } ) diff --git a/test/examples/test_examples.py b/test/examples/test_examples.py index 255dbf713..bd3446fb5 100644 --- a/test/examples/test_examples.py +++ b/test/examples/test_examples.py @@ -10,35 +10,21 @@ from ..mitmproxy import tservers class TestScripts(tservers.MasterTest): def test_add_header(self, tdata): with taddons.context() as tctx: - a = tctx.script(tdata.path("../examples/simple/add_header.py")) - f = tflow.tflow(resp=tutils.tresp()) - a.response(f) - assert f.response.headers["newheader"] == "foo" + a = tctx.script(tdata.path("../examples/addons/scripting-minimal-example.py")) + f = tflow.tflow() + a.request(f) + assert f.request.headers["myheader"] == "value" def test_custom_contentviews(self, tdata): with taddons.context() as tctx: - tctx.script(tdata.path("../examples/simple/custom_contentview.py")) + tctx.script(tdata.path("../examples/addons/contentview.py")) swapcase = contentviews.get("swapcase") _, fmt = swapcase(b"Test!") assert any(b'tEST!' in val[0][1] for val in fmt) - def test_iframe_injector(self, tdata): - with taddons.context() as tctx: - sc = tctx.script(tdata.path("../examples/simple/modify_body_inject_iframe.py")) - tctx.configure( - sc, - iframe = "http://example.org/evil_iframe" - ) - f = tflow.tflow( - resp=tutils.tresp(content=b"mitmproxy") - ) - tctx.master.addons.invoke_addon(sc, "response", f) - content = f.response.content - assert b'iframe' in content and b'evil_iframe' in content - def test_modify_form(self, tdata): with taddons.context() as tctx: - sc = tctx.script(tdata.path("../examples/simple/modify_form.py")) + sc = tctx.script(tdata.path("../examples/addons/http-modify-form.py")) form_header = Headers(content_type="application/x-www-form-urlencoded") f = tflow.tflow(req=tutils.treq(headers=form_header)) @@ -52,7 +38,7 @@ class TestScripts(tservers.MasterTest): def test_modify_querystring(self, tdata): with taddons.context() as tctx: - sc = tctx.script(tdata.path("../examples/simple/modify_querystring.py")) + sc = tctx.script(tdata.path("../examples/addons/http-modify-query-string.py")) f = tflow.tflow(req=tutils.treq(path="/search?q=term")) sc.request(f) @@ -64,36 +50,14 @@ class TestScripts(tservers.MasterTest): def test_redirect_requests(self, tdata): with taddons.context() as tctx: - sc = tctx.script(tdata.path("../examples/simple/redirect_requests.py")) + sc = tctx.script(tdata.path("../examples/addons/http-redirect-requests.py")) f = tflow.tflow(req=tutils.treq(host="example.org")) sc.request(f) assert f.request.host == "mitmproxy.org" def test_send_reply_from_proxy(self, tdata): with taddons.context() as tctx: - sc = tctx.script(tdata.path("../examples/simple/send_reply_from_proxy.py")) + sc = tctx.script(tdata.path("../examples/addons/http-reply-from-proxy.py")) f = tflow.tflow(req=tutils.treq(host="example.com", port=80)) sc.request(f) assert f.response.content == b"Hello World" - - def test_dns_spoofing(self, tdata): - with taddons.context() as tctx: - sc = tctx.script(tdata.path("../examples/complex/dns_spoofing.py")) - - original_host = "example.com" - - host_header = Headers(host=original_host) - f = tflow.tflow(req=tutils.treq(headers=host_header, port=80)) - - tctx.master.addons.invoke_addon(sc, "requestheaders", f) - - # Rewrite by reverse proxy mode - f.request.scheme = "https" - f.request.port = 443 - - tctx.master.addons.invoke_addon(sc, "request", f) - - assert f.request.scheme == "http" - assert f.request.port == 80 - - assert f.request.headers["Host"] == original_host diff --git a/test/examples/webscanner_helper/__init__.py b/test/examples/webscanner_helper/__init__.py deleted file mode 100644 index e69de29bb..000000000