diff --git a/CHANGELOG.md b/CHANGELOG.md index f4254e9e3..90c59769a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,8 @@ ([#7642](https://github.com/mitmproxy/mitmproxy/pull/7642), @lups2000) - Fix mitmweb crash when searching or highlighting using ~h, ~hq, or ~hs. ([#7652](https://github.com/mitmproxy/mitmproxy/pull/7652), @lups2000) +- `mitmproxy.dns.Message` has been renamed to `mitmproxy.dns.DNSMessage` + ([#7670](https://github.com/mitmproxy/mitmproxy/pull/7670), @mhils) [tree-sitter]: https://tree-sitter.github.io/tree-sitter/ diff --git a/docs/src/content/addons/api-changelog.md b/docs/src/content/addons/api-changelog.md index cc0bda420..4ecc3624c 100644 --- a/docs/src/content/addons/api-changelog.md +++ b/docs/src/content/addons/api-changelog.md @@ -15,6 +15,8 @@ The Contentviews API has drastically simplified, see the new [contentview docume [contentview documentation]: {{< relref "/addons/contentviews" >}} +`mitmproxy.dns.Message` has been renamed to `mitmproxy.dns.DNSMessage`. + ## mitmproxy 9.1 `mitmproxy.connection.Client` and `mitmproxy.connection.Server` now accept keyword arguments only. diff --git a/mitmproxy/addons/dns_resolver.py b/mitmproxy/addons/dns_resolver.py index 40d678716..a250a21dc 100644 --- a/mitmproxy/addons/dns_resolver.py +++ b/mitmproxy/addons/dns_resolver.py @@ -115,8 +115,8 @@ class DnsResolver: async def resolve( self, - message: dns.Message, - ) -> dns.Message: + message: dns.DNSMessage, + ) -> dns.DNSMessage: q = message.question assert q try: diff --git a/mitmproxy/contentviews/__init__.py b/mitmproxy/contentviews/__init__.py index a0d0ccd93..37beb56d6 100644 --- a/mitmproxy/contentviews/__init__.py +++ b/mitmproxy/contentviews/__init__.py @@ -13,20 +13,6 @@ import warnings from dataclasses import dataclass from ..addonmanager import cut_traceback -from ..tcp import TCPMessage -from ..udp import UDPMessage -from ..websocket import WebSocketMessage -from . import dns -from . import graphql -from . import image -from . import javascript -from . import mqtt -from . import multipart -from . import query -from . import socketio -from . import urlencoded -from . import wbxml -from . import xml_html from ._api import Contentview from ._api import InteractiveContentview from ._api import Metadata @@ -35,16 +21,27 @@ from ._compat import get # noqa: F401 from ._compat import LegacyContentview from ._compat import remove # noqa: F401 from ._registry import ContentviewRegistry +from ._utils import ContentviewMessage from ._utils import get_data from ._utils import make_metadata from ._view_css import css +from ._view_dns import dns +from ._view_graphql import graphql from ._view_http3 import http3 +from ._view_image import image +from ._view_javascript import javascript from ._view_json import json_view +from ._view_mqtt import mqtt +from ._view_multipart import multipart +from ._view_query import query from ._view_raw import raw +from ._view_socketio import socket_io +from ._view_urlencoded import urlencoded +from ._view_wbxml import wbxml +from ._view_xml_html import xml_html from .base import View import mitmproxy_rs.contentviews from mitmproxy import flow -from mitmproxy import http from mitmproxy.utils import strutils logger = logging.getLogger(__name__) @@ -62,7 +59,7 @@ registry = ContentviewRegistry() def prettify_message( - message: http.Message | TCPMessage | UDPMessage | WebSocketMessage, + message: ContentviewMessage, flow: flow.Flow, view_name: str = "auto", registry: ContentviewRegistry = registry, @@ -121,7 +118,7 @@ def prettify_message( def reencode_message( prettified: str, - message: http.Message | TCPMessage | UDPMessage | WebSocketMessage, + message: ContentviewMessage, flow: flow.Flow, view_name: str, ) -> bytes: @@ -132,28 +129,22 @@ def reencode_message( return view.reencode(prettified, metadata) -# Legacy contentviews need to be registered explicitly. -_legacy_views = [ - graphql.ViewGraphQL, - xml_html.ViewXmlHtml, - wbxml.ViewWBXML, - javascript.ViewJavaScript, - urlencoded.ViewURLEncoded, - multipart.ViewMultipart, - image.ViewImage, - query.ViewQuery, - mqtt.ViewMQTT, - dns.ViewDns, - socketio.ViewSocketIO, -] -for ViewCls in _legacy_views: - registry.register(LegacyContentview(ViewCls())) # type: ignore[abstract] - _views: list[Contentview] = [ - json_view, - raw, css, + dns, + graphql, http3, + image, + javascript, + json_view, + mqtt, + multipart, + query, + raw, + socket_io, + urlencoded, + wbxml, + xml_html, ] for view in _views: registry.register(view) diff --git a/mitmproxy/contentviews/_api.py b/mitmproxy/contentviews/_api.py index 600e360d8..f0db11058 100644 --- a/mitmproxy/contentviews/_api.py +++ b/mitmproxy/contentviews/_api.py @@ -10,6 +10,7 @@ from typing import Literal from mitmproxy import http from mitmproxy import tcp from mitmproxy import udp +from mitmproxy.dns import DNSMessage from mitmproxy.flow import Flow from mitmproxy.websocket import WebSocketMessage @@ -102,6 +103,8 @@ class Metadata: """The UDP message that the data belongs to, if any.""" websocket_message: WebSocketMessage | None = None """The websocket message that the data belongs to, if any.""" + dns_message: DNSMessage | None = None + """The DNS message that the data belongs to, if any.""" protobuf_definitions: Path | None = None """Path to a .proto file that's used to resolve Protobuf field names.""" diff --git a/mitmproxy/contentviews/_registry.py b/mitmproxy/contentviews/_registry.py index 8e1551f0b..c3b5f66cb 100644 --- a/mitmproxy/contentviews/_registry.py +++ b/mitmproxy/contentviews/_registry.py @@ -53,10 +53,10 @@ class ContentviewRegistry(Mapping[str, Contentview]): try: priority = view.render_priority(data, metadata) assert isinstance(priority, (int, float)), ( - f"Render_priority for {view.name} did not return a number." + f"render_priority for {view.name} did not return a number." ) except Exception: - logger.exception("Error in render_priority") + logger.exception(f"Error in {view.name}.render_priority") else: if max_prio is None or max_prio[0] < priority: max_prio = (priority, view) diff --git a/mitmproxy/contentviews/_utils.py b/mitmproxy/contentviews/_utils.py index a59a0d2d3..8fa6a719b 100644 --- a/mitmproxy/contentviews/_utils.py +++ b/mitmproxy/contentviews/_utils.py @@ -1,16 +1,28 @@ +import io +import typing +from collections.abc import Iterable from pathlib import Path +from typing import Any + +from ruamel.yaml import YAML from .. import ctx from .. import http +from ..dns import DNSMessage from ..flow import Flow from ..tcp import TCPMessage from ..udp import UDPMessage +from ..utils import strutils from ..websocket import WebSocketMessage from ._api import Metadata +type ContentviewMessage = ( + http.Message | TCPMessage | UDPMessage | WebSocketMessage | DNSMessage +) + def make_metadata( - message: http.Message | TCPMessage | UDPMessage | WebSocketMessage, + message: ContentviewMessage, flow: Flow, ) -> Metadata: metadata = Metadata( @@ -32,12 +44,16 @@ def make_metadata( metadata.udp_message = message case WebSocketMessage(): metadata.websocket_message = message + case DNSMessage(): + metadata.dns_message = message + case other: # pragma: no cover + typing.assert_never(other) return metadata def get_data( - message: http.Message | TCPMessage | UDPMessage | WebSocketMessage, + message: ContentviewMessage, ) -> tuple[bytes | None, str]: content: bytes | None try: @@ -53,3 +69,38 @@ def get_data( enc = "" return content, enc + + +def yaml_dumps(d: Any) -> str: + if not d: + return "" + out = io.StringIO() + YAML(typ="rt", pure=True).dump(d, out) + return out.getvalue() + + +def yaml_loads(yaml: str) -> Any: + return YAML(typ="safe", pure=True).load(yaml) + + +def merge_repeated_keys(items: Iterable[tuple[str, str]]) -> dict[str, str | list[str]]: + """ + Helper function that takes a list of pairs and merges repeated keys. + """ + ret: dict[str, str | list[str]] = {} + for key, value in items: + if existing := ret.get(key): + if isinstance(existing, list): + existing.append(value) + else: + ret[key] = [existing, value] + else: + ret[key] = value + return ret + + +def byte_pairs_to_str_pairs( + items: Iterable[tuple[bytes, bytes]], +) -> Iterable[tuple[str, str]]: + for key, value in items: + yield (strutils.bytes_to_escaped_str(key), strutils.bytes_to_escaped_str(value)) diff --git a/mitmproxy/contentviews/_view_dns.py b/mitmproxy/contentviews/_view_dns.py new file mode 100644 index 000000000..4672da015 --- /dev/null +++ b/mitmproxy/contentviews/_view_dns.py @@ -0,0 +1,53 @@ +from mitmproxy.contentviews._api import InteractiveContentview +from mitmproxy.contentviews._api import Metadata +from mitmproxy.contentviews._utils import yaml_dumps +from mitmproxy.contentviews._utils import yaml_loads +from mitmproxy.dns import DNSMessage as DNSMessage +from mitmproxy.proxy.layers.dns import pack_message + + +def _is_dns_tcp(metadata: Metadata) -> bool: + return bool(metadata.tcp_message or metadata.http_message) + + +class DNSContentview(InteractiveContentview): + syntax_highlight = "yaml" + + def prettify( + self, + data: bytes, + metadata: Metadata, + ) -> str: + if _is_dns_tcp(metadata): + data = data[2:] # hack: cut off length label and hope for the best + message = DNSMessage.unpack(data).to_json() + del message["status_code"] + message.pop("timestamp", None) + return yaml_dumps(message) + + def reencode( + self, + prettified: str, + metadata: Metadata, + ) -> bytes: + data = yaml_loads(prettified) + message = DNSMessage.from_json(data) + return pack_message(message, "tcp" if _is_dns_tcp(metadata) else "udp") + + def render_priority( + self, + data: bytes, + metadata: Metadata, + ) -> float: + return float( + metadata.content_type == "application/dns-message" + or bool( + metadata.flow + and metadata.flow.server_conn + and metadata.flow.server_conn.address + and metadata.flow.server_conn.address[1] in (53, 5353) + ) + ) + + +dns = DNSContentview() diff --git a/mitmproxy/contentviews/graphql.py b/mitmproxy/contentviews/_view_graphql.py similarity index 61% rename from mitmproxy/contentviews/graphql.py rename to mitmproxy/contentviews/_view_graphql.py index 5b336ef8c..3551ec23a 100644 --- a/mitmproxy/contentviews/graphql.py +++ b/mitmproxy/contentviews/_view_graphql.py @@ -1,7 +1,8 @@ import json from typing import Any -from mitmproxy.contentviews import base +from mitmproxy.contentviews._api import Contentview +from mitmproxy.contentviews._api import Metadata def format_graphql(data): @@ -36,20 +37,26 @@ def is_graphql_batch_query(data): ) -class ViewGraphQL(base.View): - name = "GraphQL" - - def __call__(self, data, **metadata): - data = json.loads(data) - if is_graphql_query(data): - return "GraphQL", base.format_text(format_graphql(data)) - elif is_graphql_batch_query(data): - return "GraphQL", base.format_text(format_query_list(data)) +class GraphQLContentview(Contentview): + def prettify( + self, + data: bytes, + metadata: Metadata, + ) -> str: + gql = json.loads(data) + if is_graphql_query(gql): + return format_graphql(gql) + elif is_graphql_batch_query(gql): + return format_query_list(gql) + else: + raise ValueError("Not a GraphQL message.") def render_priority( - self, data: bytes, *, content_type: str | None = None, **metadata + self, + data: bytes, + metadata: Metadata, ) -> float: - if content_type != "application/json" or not data: + if metadata.content_type != "application/json" or not data: return 0 try: @@ -60,3 +67,6 @@ class ViewGraphQL(base.View): pass return 0 + + +graphql = GraphQLContentview() diff --git a/mitmproxy/contentviews/_view_image/__init__.py b/mitmproxy/contentviews/_view_image/__init__.py new file mode 100644 index 000000000..02b42d0ae --- /dev/null +++ b/mitmproxy/contentviews/_view_image/__init__.py @@ -0,0 +1,3 @@ +from .view import image + +__all__ = ["image"] diff --git a/mitmproxy/contentviews/image/image_parser.py b/mitmproxy/contentviews/_view_image/image_parser.py similarity index 92% rename from mitmproxy/contentviews/image/image_parser.py rename to mitmproxy/contentviews/_view_image/image_parser.py index c5d0c3ea8..20d8aa3dc 100644 --- a/mitmproxy/contentviews/image/image_parser.py +++ b/mitmproxy/contentviews/_view_image/image_parser.py @@ -7,10 +7,10 @@ from mitmproxy.contrib.kaitaistruct import ico from mitmproxy.contrib.kaitaistruct import jpeg from mitmproxy.contrib.kaitaistruct import png -Metadata = list[tuple[str, str]] +type ImageMetadata = list[tuple[str, str]] -def parse_png(data: bytes) -> Metadata: +def parse_png(data: bytes) -> ImageMetadata: img = png.Png(KaitaiStream(io.BytesIO(data))) parts = [ ("Format", "Portable network graphics"), @@ -34,7 +34,7 @@ def parse_png(data: bytes) -> Metadata: return parts -def parse_gif(data: bytes) -> Metadata: +def parse_gif(data: bytes) -> ImageMetadata: img = gif.Gif(KaitaiStream(io.BytesIO(data))) descriptor = img.logical_screen_descriptor parts = [ @@ -60,7 +60,7 @@ def parse_gif(data: bytes) -> Metadata: return parts -def parse_jpeg(data: bytes) -> Metadata: +def parse_jpeg(data: bytes) -> ImageMetadata: img = jpeg.Jpeg(KaitaiStream(io.BytesIO(data))) parts = [("Format", "JPEG (ISO 10918)")] for segment in img.segments: @@ -83,7 +83,7 @@ def parse_jpeg(data: bytes) -> Metadata: ) parts.append(("jfif_unit", str(segment.data.density_units._value_))) if segment.marker._name_ == "com": - parts.append(("comment", str(segment.data))) + parts.append(("comment", segment.data.decode("utf8", "backslashreplace"))) if segment.marker._name_ == "app1": if hasattr(segment.data, "body"): for field in segment.data.body.data.body.ifd0.fields: @@ -94,7 +94,7 @@ def parse_jpeg(data: bytes) -> Metadata: return parts -def parse_ico(data: bytes) -> Metadata: +def parse_ico(data: bytes) -> ImageMetadata: img = ico.Ico(KaitaiStream(io.BytesIO(data))) parts = [ ("Format", "ICO"), diff --git a/mitmproxy/contentviews/image/view.py b/mitmproxy/contentviews/_view_image/view.py similarity index 53% rename from mitmproxy/contentviews/image/view.py rename to mitmproxy/contentviews/_view_image/view.py index 181c12758..5389cba95 100644 --- a/mitmproxy/contentviews/image/view.py +++ b/mitmproxy/contentviews/_view_image/view.py @@ -1,21 +1,28 @@ +from .._utils import merge_repeated_keys +from .._utils import yaml_dumps from . import image_parser -from mitmproxy.contentviews import base +from mitmproxy.contentviews._api import Contentview +from mitmproxy.contentviews._api import Metadata from mitmproxy.contrib import imghdr -from mitmproxy.coretypes import multidict def test_ico(h, f): if h.startswith(b"\x00\x00\x01\x00"): return "ico" + return None imghdr.tests.append(test_ico) -class ViewImage(base.View): - name = "Image" +class ImageContentview(Contentview): + syntax_highlight = "yaml" - def __call__(self, data, **metadata): + def prettify( + self, + data: bytes, + metadata: Metadata, + ) -> str: image_type = imghdr.what("", h=data) if image_type == "png": image_metadata = image_parser.parse_png(data) @@ -26,20 +33,25 @@ class ViewImage(base.View): elif image_type == "ico": image_metadata = image_parser.parse_ico(data) else: - image_metadata = [("Image Format", image_type or "unknown")] + image_metadata = [] if image_type: view_name = f"{image_type.upper()} Image" else: view_name = "Unknown Image" - return view_name, base.format_dict(multidict.MultiDict(image_metadata)) + return f"# {view_name}\n" + yaml_dumps(merge_repeated_keys(image_metadata)) def render_priority( - self, data: bytes, *, content_type: str | None = None, **metadata + self, + data: bytes, + metadata: Metadata, ) -> float: return float( bool( - content_type - and content_type.startswith("image/") - and content_type != "image/svg+xml" + metadata.content_type + and metadata.content_type.startswith("image/") + and not metadata.content_type.endswith("+xml") ) ) + + +image = ImageContentview() diff --git a/mitmproxy/contentviews/javascript.py b/mitmproxy/contentviews/_view_javascript.py similarity index 74% rename from mitmproxy/contentviews/javascript.py rename to mitmproxy/contentviews/_view_javascript.py index 02b47d53e..5070ac7ad 100644 --- a/mitmproxy/contentviews/javascript.py +++ b/mitmproxy/contentviews/_view_javascript.py @@ -1,7 +1,8 @@ import io import re -from mitmproxy.contentviews import base +from mitmproxy.contentviews._api import Contentview +from mitmproxy.contentviews._api import Metadata from mitmproxy.utils import strutils DELIMITERS = "{};\n" @@ -40,20 +41,27 @@ def beautify(data): return data -class ViewJavaScript(base.View): - name = "JavaScript" +class JavaScriptContentview(Contentview): __content_types = ( "application/x-javascript", "application/javascript", "text/javascript", ) - def __call__(self, data, **metadata): - data = data.decode("utf-8", "replace") - res = beautify(data) - return "JavaScript", base.format_text(res) + def prettify( + self, + data: bytes, + metadata: Metadata, + ) -> str: + data_str = data.decode("utf-8", "replace") + return beautify(data_str) def render_priority( - self, data: bytes, *, content_type: str | None = None, **metadata + self, + data: bytes, + metadata: Metadata, ) -> float: - return float(bool(data) and content_type in self.__content_types) + return float(bool(data) and metadata.content_type in self.__content_types) + + +javascript = JavaScriptContentview() diff --git a/mitmproxy/contentviews/_view_json.py b/mitmproxy/contentviews/_view_json.py index fb46e8cfa..4966d3b74 100644 --- a/mitmproxy/contentviews/_view_json.py +++ b/mitmproxy/contentviews/_view_json.py @@ -22,7 +22,7 @@ class JSONContentview(Contentview): if ( metadata.content_type and metadata.content_type.startswith("application/") - and metadata.content_type.endswith("+json") + and metadata.content_type.endswith("json") ): return 1 return 0 diff --git a/mitmproxy/contentviews/mqtt.py b/mitmproxy/contentviews/_view_mqtt.py similarity index 96% rename from mitmproxy/contentviews/mqtt.py rename to mitmproxy/contentviews/_view_mqtt.py index 548998ae3..f8377af93 100644 --- a/mitmproxy/contentviews/mqtt.py +++ b/mitmproxy/contentviews/_view_mqtt.py @@ -1,6 +1,7 @@ import struct -from mitmproxy.contentviews import base +from mitmproxy.contentviews._api import Contentview +from mitmproxy.contentviews._api import Metadata from mitmproxy.utils import strutils # from https://github.com/nikitastupin/mitmproxy-mqtt-script @@ -263,15 +264,14 @@ Password: {strutils.bytes_to_escaped_str(self.payload.get("Password", b"None"))} self.packet_identifier = self._packet[offset : offset + 2] -class ViewMQTT(base.View): - name = "MQTT" - - def __call__(self, data, **metadata): +class MQTTContentview(Contentview): + def prettify( + self, + data: bytes, + metadata: Metadata, + ) -> str: mqtt_packet = MQTTControlPacket(data) - text = mqtt_packet.pprint() - return "MQTT", base.format_text(text) + return mqtt_packet.pprint() - def render_priority( - self, data: bytes, *, content_type: str | None = None, **metadata - ) -> float: - return 0 + +mqtt = MQTTContentview() diff --git a/mitmproxy/contentviews/_view_multipart.py b/mitmproxy/contentviews/_view_multipart.py new file mode 100644 index 000000000..ede241695 --- /dev/null +++ b/mitmproxy/contentviews/_view_multipart.py @@ -0,0 +1,32 @@ +from ._utils import byte_pairs_to_str_pairs +from ._utils import merge_repeated_keys +from ._utils import yaml_dumps +from mitmproxy.contentviews._api import Contentview +from mitmproxy.contentviews._api import Metadata +from mitmproxy.net.http.multipart import decode_multipart + + +class MultipartContentview(Contentview): + name = "Multipart Form" + syntax_highlight = "yaml" + + def prettify( + self, + data: bytes, + metadata: Metadata, + ) -> str: + if not metadata.http_message: + raise ValueError("Not an HTTP message") + content_type = metadata.http_message.headers["content-type"] + items = decode_multipart(content_type, data) + return yaml_dumps(merge_repeated_keys(byte_pairs_to_str_pairs(items))) + + def render_priority( + self, + data: bytes, + metadata: Metadata, + ) -> float: + return float(bool(data) and metadata.content_type == "multipart/form-data") + + +multipart = MultipartContentview() diff --git a/mitmproxy/contentviews/_view_query.py b/mitmproxy/contentviews/_view_query.py new file mode 100644 index 000000000..ad4bd6a5b --- /dev/null +++ b/mitmproxy/contentviews/_view_query.py @@ -0,0 +1,31 @@ +from .. import http +from ._utils import merge_repeated_keys +from ._utils import yaml_dumps +from mitmproxy.contentviews._api import Contentview +from mitmproxy.contentviews._api import Metadata + + +class QueryContentview(Contentview): + syntax_highlight = "yaml" + + def prettify( + self, + data: bytes, + metadata: Metadata, + ) -> str: + if not isinstance(metadata.http_message, http.Request): + raise ValueError("Not an HTTP request.") + items = metadata.http_message.query.items(multi=True) + return yaml_dumps(merge_repeated_keys(items)) + + def render_priority( + self, + data: bytes, + metadata: Metadata, + ) -> float: + return 0.3 * float( + not data and bool(getattr(metadata.http_message, "query", False)) + ) + + +query = QueryContentview() diff --git a/mitmproxy/contentviews/socketio.py b/mitmproxy/contentviews/_view_socketio.py similarity index 57% rename from mitmproxy/contentviews/socketio.py rename to mitmproxy/contentviews/_view_socketio.py index 25db65c10..7e8295095 100644 --- a/mitmproxy/contentviews/socketio.py +++ b/mitmproxy/contentviews/_view_socketio.py @@ -1,9 +1,9 @@ from enum import Enum -from typing import Tuple -from mitmproxy.contentviews import base -from mitmproxy.flow import Flow +from mitmproxy.contentviews._api import Contentview +from mitmproxy.contentviews._api import Metadata from mitmproxy.http import HTTPFlow +from mitmproxy.utils import strutils class PacketType(Enum): @@ -49,7 +49,7 @@ class SocketIO(PacketType): ) -def parse_packet(data) -> Tuple[PacketType, bytes | str]: +def parse_packet(data: bytes) -> tuple[PacketType, bytes]: # throws IndexError/ValueError if invalid packet engineio_type = EngineIO(data[0]) data = data[1:] @@ -63,39 +63,32 @@ def parse_packet(data) -> Tuple[PacketType, bytes | str]: return socketio_type, data -def format_packet(packet_type: PacketType, data): - if not packet_type.visible: - return "Socket.IO", iter([]) - - return "Socket.IO", iter( - [ - [ - ("content_none", f"{packet_type} "), - ("text", data), - ] - ] - ) - - -class ViewSocketIO(base.View): +class SocketIOContentview(Contentview): name = "Socket.IO" - def __call__(self, data, **metadata): - try: - packet_type, data = parse_packet(data) - except (IndexError, ValueError): - return None - - return format_packet(packet_type, data) + def prettify( + self, + data: bytes, + metadata: Metadata, + ) -> str: + packet_type, msg = parse_packet(data) + if not packet_type.visible: + return "" + return f"{packet_type} {strutils.bytes_to_escaped_str(msg)}" def render_priority( - self, data: bytes, *, flow: Flow | None = None, **metadata + self, + data: bytes, + metadata: Metadata, ) -> float: - if ( - data - and isinstance(flow, HTTPFlow) - and flow.websocket is not None - and "/socket.io/?" in flow.request.path - ): - return 1 - return 0 + return float( + bool( + data + and isinstance(metadata.flow, HTTPFlow) + and metadata.flow.websocket is not None + and "/socket.io/?" in metadata.flow.request.path + ) + ) + + +socket_io = SocketIOContentview() diff --git a/mitmproxy/contentviews/_view_urlencoded.py b/mitmproxy/contentviews/_view_urlencoded.py new file mode 100644 index 000000000..cf4aa8f63 --- /dev/null +++ b/mitmproxy/contentviews/_view_urlencoded.py @@ -0,0 +1,33 @@ +import urllib +import urllib.parse + +from ._utils import byte_pairs_to_str_pairs +from ._utils import merge_repeated_keys +from ._utils import yaml_dumps +from mitmproxy.contentviews._api import Contentview +from mitmproxy.contentviews._api import Metadata + + +class URLEncodedContentview(Contentview): + name = "URL-encoded" + syntax_highlight = "yaml" + + def prettify( + self, + data: bytes, + metadata: Metadata, + ) -> str: + items = urllib.parse.parse_qsl(data, keep_blank_values=True) + return yaml_dumps(merge_repeated_keys(byte_pairs_to_str_pairs(items))) + + def render_priority( + self, + data: bytes, + metadata: Metadata, + ) -> float: + return float( + bool(data) and metadata.content_type == "application/x-www-form-urlencoded" + ) + + +urlencoded = URLEncodedContentview() diff --git a/mitmproxy/contentviews/_view_wbxml.py b/mitmproxy/contentviews/_view_wbxml.py new file mode 100644 index 000000000..a7ad742a5 --- /dev/null +++ b/mitmproxy/contentviews/_view_wbxml.py @@ -0,0 +1,25 @@ +from mitmproxy.contentviews._api import Contentview +from mitmproxy.contentviews._api import Metadata +from mitmproxy.contrib.wbxml import ASCommandResponse + + +class WBXMLContentview(Contentview): + __content_types = ("application/vnd.wap.wbxml", "application/vnd.ms-sync.wbxml") + syntax_highlight = "xml" + + def prettify( + self, + data: bytes, + metadata: Metadata, + ) -> str: + return ASCommandResponse.ASCommandResponse(data).xmlString + + def render_priority( + self, + data: bytes, + metadata: Metadata, + ) -> float: + return float(bool(data) and metadata.content_type in self.__content_types) + + +wbxml = WBXMLContentview() diff --git a/mitmproxy/contentviews/xml_html.py b/mitmproxy/contentviews/_view_xml_html.py similarity index 88% rename from mitmproxy/contentviews/xml_html.py rename to mitmproxy/contentviews/_view_xml_html.py index 4bac6effe..b43400772 100644 --- a/mitmproxy/contentviews/xml_html.py +++ b/mitmproxy/contentviews/_view_xml_html.py @@ -3,7 +3,8 @@ import re import textwrap from collections.abc import Iterable -from mitmproxy.contentviews import base +from mitmproxy.contentviews._api import Contentview +from mitmproxy.contentviews._api import Metadata from mitmproxy.utils import sliding_window from mitmproxy.utils import strutils @@ -243,37 +244,35 @@ def format_xml(tokens: Iterable[Token]) -> str: return out.getvalue() -class ViewXmlHtml(base.View): - name = "XML/HTML" +class XmlHtmlContentview(Contentview): __content_types = ("text/xml", "text/html") - + name = "XML/HTML" syntax_highlight = "xml" - def __call__(self, data, **metadata): - # TODO: - # We should really have the message text as str here, - # not the message content as bytes. - # https://github.com/mitmproxy/mitmproxy/issues/1662#issuecomment-266192578 - data = data.decode("utf8", "xmlcharrefreplace") - tokens = tokenize(data) - # TODO: - # Performance: Don't render the whole document right away. - # Let's wait with this until we have a sequence-like interface, - # this thing is reasonably fast right now anyway. - pretty = base.format_text(format_xml(tokens)) - if "html" in data.lower(): - t = "HTML" + def prettify( + self, + data: bytes, + metadata: Metadata, + ) -> str: + if metadata.http_message: + data_str = metadata.http_message.get_text(strict=False) or "" else: - t = "XML" - return t, pretty + data_str = data.decode("utf8", "backslashreplace") + tokens = tokenize(data_str) + return format_xml(tokens) def render_priority( - self, data: bytes, *, content_type: str | None = None, **metadata + self, + data: bytes, + metadata: Metadata, ) -> float: if not data: return 0 - if content_type in self.__content_types: + if metadata.content_type in self.__content_types: return 1 elif strutils.is_xml(data): return 0.4 return 0 + + +xml_html = XmlHtmlContentview() diff --git a/mitmproxy/contentviews/dns.py b/mitmproxy/contentviews/dns.py deleted file mode 100644 index 7ea1e6ec6..000000000 --- a/mitmproxy/contentviews/dns.py +++ /dev/null @@ -1,21 +0,0 @@ -import json - -from mitmproxy.contentviews import base -from mitmproxy.dns import Message - - -class ViewDns(base.View): - name = "DNS-over-HTTPS" - - def __call__(self, data, **metadata): - try: - message = Message.unpack(data) - except Exception: - pass - else: - return "DoH", json.dumps(message.to_json(), indent=4) - - def render_priority( - self, data: bytes, *, content_type: str | None = None, **metadata - ) -> float: - return float(content_type == "application/dns-message") diff --git a/mitmproxy/contentviews/image/__init__.py b/mitmproxy/contentviews/image/__init__.py deleted file mode 100644 index 33356bd7a..000000000 --- a/mitmproxy/contentviews/image/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .view import ViewImage - -__all__ = ["ViewImage"] diff --git a/mitmproxy/contentviews/multipart.py b/mitmproxy/contentviews/multipart.py deleted file mode 100644 index 450b42741..000000000 --- a/mitmproxy/contentviews/multipart.py +++ /dev/null @@ -1,35 +0,0 @@ -from .. import http -from . import base -from mitmproxy.coretypes import multidict -from mitmproxy.net.http import multipart - - -class ViewMultipart(base.View): - name = "Multipart Form" - - @staticmethod - def _format(v): - yield [("highlight", "Form data:\n")] - yield from base.format_dict(multidict.MultiDict(v)) - - def __call__( - self, - data: bytes, - content_type: str | None = None, - http_message: http.Message | None = None, - **metadata, - ): - # The content_type doesn't have the boundary, so we get it from the header again - headers = getattr(http_message, "headers", None) - if headers: - content_type = headers.get("content-type") - if content_type is None: - return - v = multipart.decode_multipart(content_type, data) - if v: - return "Multipart form", self._format(v) - - def render_priority( - self, data: bytes, *, content_type: str | None = None, **metadata - ) -> float: - return float(bool(data) and content_type == "multipart/form-data") diff --git a/mitmproxy/contentviews/query.py b/mitmproxy/contentviews/query.py deleted file mode 100644 index bcbb39cfd..000000000 --- a/mitmproxy/contentviews/query.py +++ /dev/null @@ -1,20 +0,0 @@ -from .. import http -from . import base - - -class ViewQuery(base.View): - name = "Query" - - def __call__( - self, data: bytes, http_message: http.Message | None = None, **metadata - ): - query = getattr(http_message, "query", None) - if query: - return "Query", base.format_pairs(query.items(multi=True)) - else: - return "Query", base.format_text("") - - def render_priority( - self, data: bytes, *, http_message: http.Message | None = None, **metadata - ) -> float: - return 0.3 * float(bool(getattr(http_message, "query", False) and not data)) diff --git a/mitmproxy/contentviews/urlencoded.py b/mitmproxy/contentviews/urlencoded.py deleted file mode 100644 index 5065dca33..000000000 --- a/mitmproxy/contentviews/urlencoded.py +++ /dev/null @@ -1,19 +0,0 @@ -from . import base -from mitmproxy.net.http import url - - -class ViewURLEncoded(base.View): - name = "URL-encoded" - - def __call__(self, data, **metadata): - try: - data = data.decode("ascii", "strict") - except ValueError: - return None - d = url.decode(data) - return "URLEncoded form", base.format_pairs(d) - - def render_priority( - self, data: bytes, *, content_type: str | None = None, **metadata - ) -> float: - return float(bool(data) and content_type == "application/x-www-form-urlencoded") diff --git a/mitmproxy/contentviews/wbxml.py b/mitmproxy/contentviews/wbxml.py deleted file mode 100644 index 77bceed17..000000000 --- a/mitmproxy/contentviews/wbxml.py +++ /dev/null @@ -1,21 +0,0 @@ -from . import base -from mitmproxy.contrib.wbxml import ASCommandResponse - - -class ViewWBXML(base.View): - name = "WBXML" - __content_types = ("application/vnd.wap.wbxml", "application/vnd.ms-sync.wbxml") - - def __call__(self, data, **metadata): - try: - parser = ASCommandResponse.ASCommandResponse(data) - parsedContent = parser.xmlString - if parsedContent: - return "WBXML", base.format_text(parsedContent) - except Exception: - return None - - def render_priority( - self, data: bytes, *, content_type: str | None = None, **metadata - ) -> float: - return float(bool(data) and content_type in self.__content_types) diff --git a/mitmproxy/dns.py b/mitmproxy/dns.py index cd79cdcb2..0c0dd7641 100644 --- a/mitmproxy/dns.py +++ b/mitmproxy/dns.py @@ -9,7 +9,10 @@ from collections.abc import Iterable from dataclasses import dataclass from ipaddress import IPv4Address from ipaddress import IPv6Address +from typing import Any +from typing import cast from typing import ClassVar +from typing import Self from mitmproxy import flow from mitmproxy.coretypes import serializable @@ -20,6 +23,7 @@ from mitmproxy.net.dns import op_codes from mitmproxy.net.dns import response_codes from mitmproxy.net.dns import types from mitmproxy.net.dns.https_records import HTTPSRecord +from mitmproxy.net.dns.https_records import HTTPSRecordJSON from mitmproxy.net.dns.https_records import SVCParamKeys # DNS parameters taken from https://www.iana.org/assignments/dns-parameters/dns-parameters.xml @@ -47,6 +51,14 @@ class Question(serializable.SerializableDataclass): "class": classes.to_str(self.class_), } + @classmethod + def from_json(cls, data: dict[str, str]) -> Self: + return cls( + name=data["name"], + type=types.from_str(data["type"]), + class_=classes.from_str(data["class"]), + ) + @dataclass class ResourceRecord(serializable.SerializableDataclass): @@ -60,20 +72,7 @@ class ResourceRecord(serializable.SerializableDataclass): data: bytes def __str__(self) -> str: - try: - if self.type == types.A: - return str(self.ipv4_address) - if self.type == types.AAAA: - return str(self.ipv6_address) - if self.type in (types.NS, types.CNAME, types.PTR): - return self.domain_name - if self.type == types.TXT: - return self.text - if self.type == types.HTTPS: - return str(https_records.unpack(self.data)) - except Exception: - return f"0x{self.data.hex()} (invalid {types.to_str(self.type)} data)" - return f"0x{self.data.hex()}" + return str(self._data_json()) @property def text(self) -> str: @@ -151,7 +150,25 @@ class ResourceRecord(serializable.SerializableDataclass): record.params[SVCParamKeys.ECH.value] = ech_bytes self.data = https_records.pack(record) - def to_json(self) -> dict: + def _data_json(self) -> str | HTTPSRecordJSON: + try: + match self.type: + case types.A: + return str(self.ipv4_address) + case types.AAAA: + return str(self.ipv6_address) + case types.NS | types.CNAME | types.PTR: + return self.domain_name + case types.TXT: + return self.text + case types.HTTPS: + return https_records.unpack(self.data).to_json() + case _: + return f"0x{self.data.hex()}" + except Exception: + return f"0x{self.data.hex()} (invalid {types.to_str(self.type)} data)" + + def to_json(self) -> dict[str, str | int | HTTPSRecordJSON]: """ Converts the resource record into json for mitmweb. Sync with web/src/flow.ts. @@ -161,9 +178,40 @@ class ResourceRecord(serializable.SerializableDataclass): "type": types.to_str(self.type), "class": classes.to_str(self.class_), "ttl": self.ttl, - "data": str(self), + "data": self._data_json(), } + @classmethod + def from_json(cls, data: dict[str, Any]) -> Self: + inst = cls( + name=data["name"], + type=types.from_str(data["type"]), + class_=classes.from_str(data["class"]), + ttl=data["ttl"], + data=b"", + ) + + d: str = data["data"] + try: + match inst.type: + case types.A: + inst.ipv4_address = IPv4Address(d) + case types.AAAA: + inst.ipv6_address = IPv6Address(d) + case types.NS | types.CNAME | types.PTR: + inst.domain_name = d + case types.TXT: + inst.text = d + case types.HTTPS: + record = HTTPSRecord.from_json(cast(HTTPSRecordJSON, d)) + inst.data = https_records.pack(record) + case _: + raise ValueError + except Exception: + inst.data = bytes.fromhex(d.removeprefix("0x").partition(" (")[0]) + + return inst + @classmethod def A(cls, name: str, ip: IPv4Address, *, ttl: int = DEFAULT_TTL) -> ResourceRecord: """Create an IPv4 resource record.""" @@ -203,11 +251,9 @@ class ResourceRecord(serializable.SerializableDataclass): # comments are taken from rfc1035 @dataclass -class Message(serializable.SerializableDataclass): +class DNSMessage(serializable.SerializableDataclass): HEADER: ClassVar[struct.Struct] = struct.Struct("!HHHHHH") - timestamp: float - """The time at which the message was sent or received.""" id: int """An identifier assigned by the program that generates any kind of query.""" query: bool @@ -247,6 +293,9 @@ class Message(serializable.SerializableDataclass): additionals: list[ResourceRecord] """Third resource record section.""" + timestamp: float | None = None + """The time at which the message was sent or received.""" + def __str__(self) -> str: return "\r\n".join( map( @@ -259,8 +308,7 @@ class Message(serializable.SerializableDataclass): @property def content(self) -> bytes: - """Returns the user-friendly content of all parts as encoded bytes.""" - return str(self).encode() + return self.packed @property def question(self) -> Question | None: @@ -280,10 +328,10 @@ class Message(serializable.SerializableDataclass): ) ) - def fail(self, response_code: int) -> Message: + def fail(self, response_code: int) -> DNSMessage: if response_code == response_codes.NOERROR: raise ValueError("response_code must be an error code.") - return Message( + return DNSMessage( timestamp=time.time(), id=self.id, query=False, @@ -300,8 +348,8 @@ class Message(serializable.SerializableDataclass): additionals=[], ) - def succeed(self, answers: list[ResourceRecord]) -> Message: - return Message( + def succeed(self, answers: list[ResourceRecord]) -> DNSMessage: + return DNSMessage( timestamp=time.time(), id=self.id, query=False, @@ -319,15 +367,17 @@ class Message(serializable.SerializableDataclass): ) @classmethod - def unpack(cls, buffer: bytes) -> Message: + def unpack(cls, buffer: bytes, timestamp: float | None = None) -> DNSMessage: """Converts the entire given buffer into a DNS message.""" - length, msg = cls.unpack_from(buffer, 0) + length, msg = cls.unpack_from(buffer, 0, timestamp) if length != len(buffer): raise struct.error(f"unpack requires a buffer of {length} bytes") return msg @classmethod - def unpack_from(cls, buffer: bytes | bytearray, offset: int) -> tuple[int, Message]: + def unpack_from( + cls, buffer: bytes | bytearray, offset: int, timestamp: float | None = None + ) -> tuple[int, DNSMessage]: """Converts the buffer from a given offset into a DNS message and also returns its length.""" ( id, @@ -336,9 +386,9 @@ class Message(serializable.SerializableDataclass): len_answers, len_authorities, len_additionals, - ) = Message.HEADER.unpack_from(buffer, offset) - msg = Message( - timestamp=time.time(), + ) = DNSMessage.HEADER.unpack_from(buffer, offset) + msg = DNSMessage( + timestamp=timestamp, id=id, query=(flags & (1 << 15)) == 0, op_code=(flags >> 11) & 0b1111, @@ -353,7 +403,7 @@ class Message(serializable.SerializableDataclass): authorities=[], additionals=[], ) - offset += Message.HEADER.size + offset += DNSMessage.HEADER.size cached_names = domain_names.cache() def unpack_domain_name() -> str: @@ -437,7 +487,7 @@ class Message(serializable.SerializableDataclass): flags |= self.response_code data = bytearray() data.extend( - Message.HEADER.pack( + DNSMessage.HEADER.pack( self.id, flags, len(self.questions), @@ -463,7 +513,7 @@ class Message(serializable.SerializableDataclass): Converts the message into json for mitmweb. Sync with web/src/flow.ts. """ - return { + ret = { "id": self.id, "query": self.query, "op_code": op_codes.to_str(self.op_code), @@ -478,22 +528,46 @@ class Message(serializable.SerializableDataclass): "authorities": [rr.to_json() for rr in self.authorities], "additionals": [rr.to_json() for rr in self.additionals], "size": self.size, - "timestamp": self.timestamp, } + if self.timestamp: + ret["timestamp"] = self.timestamp + return ret - def copy(self) -> Message: + @classmethod + def from_json(cls, data: Any) -> DNSMessage: + """Reconstruct a DNS message from JSON.""" + inst = cls( + id=data["id"], + query=data["query"], + op_code=op_codes.from_str(data["op_code"]), + authoritative_answer=data["authoritative_answer"], + truncation=data["truncation"], + recursion_desired=data["recursion_desired"], + recursion_available=data["recursion_available"], + reserved=0, + response_code=response_codes.from_str(data["response_code"]), + questions=[Question.from_json(x) for x in data["questions"]], + answers=[ResourceRecord.from_json(x) for x in data["answers"]], + authorities=[ResourceRecord.from_json(x) for x in data["authorities"]], + additionals=[ResourceRecord.from_json(x) for x in data["additionals"]], + ) + if ts := data.get("timestamp"): + inst.timestamp = ts + return inst + + def copy(self) -> DNSMessage: # we keep the copy semantics but change the ID generation state = self.get_state() state["id"] = random.randint(0, 65535) - return Message.from_state(state) + return DNSMessage.from_state(state) class DNSFlow(flow.Flow): """A DNSFlow is a collection of DNS messages representing a single DNS query.""" - request: Message + request: DNSMessage """The DNS request.""" - response: Message | None = None + response: DNSMessage | None = None """The DNS response.""" def get_state(self) -> serializable.State: @@ -504,8 +578,10 @@ class DNSFlow(flow.Flow): } def set_state(self, state: serializable.State) -> None: - self.request = Message.from_state(state.pop("request")) - self.response = Message.from_state(r) if (r := state.pop("response")) else None + self.request = DNSMessage.from_state(state.pop("request")) + self.response = ( + DNSMessage.from_state(r) if (r := state.pop("response")) else None + ) super().set_state(state) def __repr__(self) -> str: diff --git a/mitmproxy/flowfilter.py b/mitmproxy/flowfilter.py index 92e235dfb..689e90982 100644 --- a/mitmproxy/flowfilter.py +++ b/mitmproxy/flowfilter.py @@ -320,9 +320,9 @@ class FBod(_Rex): if msg.content is not None and self.re.search(msg.content): return True elif isinstance(f, dns.DNSFlow): - if f.request and self.re.search(f.request.content): + if f.request and self.re.search(str(f.request).encode()): return True - if f.response and self.re.search(f.response.content): + if f.response and self.re.search(str(f.response).encode()): return True return False @@ -350,7 +350,7 @@ class FBodRequest(_Rex): if msg.from_client and self.re.search(msg.content): return True elif isinstance(f, dns.DNSFlow): - if f.request and self.re.search(f.request.content): + if f.request and self.re.search(str(f.request).encode()): return True @@ -377,7 +377,7 @@ class FBodResponse(_Rex): if not msg.from_client and self.re.search(msg.content): return True elif isinstance(f, dns.DNSFlow): - if f.response and self.re.search(f.response.content): + if f.response and self.re.search(str(f.response).encode()): return True diff --git a/mitmproxy/net/dns/classes.py b/mitmproxy/net/dns/classes.py index 3f17defde..1f2258c91 100644 --- a/mitmproxy/net/dns/classes.py +++ b/mitmproxy/net/dns/classes.py @@ -5,7 +5,15 @@ NONE = 254 ANY = 255 _STRINGS = {IN: "IN", CH: "CH", HS: "HS", NONE: "NONE", ANY: "ANY"} +_INTS = {v: k for k, v in _STRINGS.items()} def to_str(class_: int) -> str: return _STRINGS.get(class_, f"CLASS({class_})") + + +def from_str(class_: str) -> int: + try: + return _INTS[class_] + except KeyError: + return int(class_.removeprefix("CLASS(").removesuffix(")")) diff --git a/mitmproxy/net/dns/https_records.py b/mitmproxy/net/dns/https_records.py index 165a4235a..acf0bad32 100644 --- a/mitmproxy/net/dns/https_records.py +++ b/mitmproxy/net/dns/https_records.py @@ -1,7 +1,9 @@ import enum import struct from dataclasses import dataclass +from typing import Self +from ...utils import strutils from . import domain_names """ @@ -30,6 +32,9 @@ class SVCParamKeys(enum.Enum): IPV6HINT = 6 +type HTTPSRecordJSON = dict[str | int, str | int] + + @dataclass class HTTPSRecord: priority: int @@ -37,14 +42,35 @@ class HTTPSRecord: params: dict[int, bytes] def __repr__(self): - params = {} - for param_type, param_value in self.params.items(): + return str(self.to_json()) + + def to_json(self) -> HTTPSRecordJSON: + ret: HTTPSRecordJSON = { + "target_name": self.target_name, + "priority": self.priority, + } + typ: str | int + for typ, val in self.params.items(): try: - name = SVCParamKeys(param_type).name.lower() + typ = SVCParamKeys(typ).name.lower() except ValueError: - name = f"key{param_type}" - params[name] = param_value - return f"priority: {self.priority} target_name: '{self.target_name}' {params}" + pass + ret[typ] = strutils.bytes_to_escaped_str(val) + return ret + + @classmethod + def from_json(cls, data: HTTPSRecordJSON) -> Self: + target_name = data.pop("target_name") + assert isinstance(target_name, str) + priority = data.pop("priority") + assert isinstance(priority, int) + params: dict[int, bytes] = {} + for k, v in data.items(): + if isinstance(k, str): + k = SVCParamKeys[k.upper()].value + assert isinstance(v, str) + params[k] = strutils.escaped_str_to_bytes(v) + return cls(target_name=target_name, priority=priority, params=params) def _unpack_params(data: bytes, offset: int) -> dict[int, bytes]: diff --git a/mitmproxy/net/dns/op_codes.py b/mitmproxy/net/dns/op_codes.py index 1e1e1dc93..ed2a6b757 100644 --- a/mitmproxy/net/dns/op_codes.py +++ b/mitmproxy/net/dns/op_codes.py @@ -13,7 +13,15 @@ _STRINGS = { UPDATE: "UPDATE", DSO: "DSO", } +_INTS = {v: k for k, v in _STRINGS.items()} def to_str(op_code: int) -> str: return _STRINGS.get(op_code, f"OPCODE({op_code})") + + +def from_str(op_code: str) -> int: + try: + return _INTS[op_code] + except KeyError: + return int(op_code.removeprefix("OPCODE(").removesuffix(")")) diff --git a/mitmproxy/net/dns/response_codes.py b/mitmproxy/net/dns/response_codes.py index 8a5024e0e..d41b84de7 100644 --- a/mitmproxy/net/dns/response_codes.py +++ b/mitmproxy/net/dns/response_codes.py @@ -40,6 +40,7 @@ _STRINGS = { NOTZONE: "NOTZONE", DSOTYPENI: "DSOTYPENI", } +_INTS = {v: k for k, v in _STRINGS.items()} def http_equiv_status_code(response_code: int) -> int: @@ -48,3 +49,10 @@ def http_equiv_status_code(response_code: int) -> int: def to_str(response_code: int) -> str: return _STRINGS.get(response_code, f"RCODE({response_code})") + + +def from_str(response_code: str) -> int: + try: + return _INTS[response_code] + except KeyError: + return int(response_code.removeprefix("RCODE(").removesuffix(")")) diff --git a/mitmproxy/net/dns/types.py b/mitmproxy/net/dns/types.py index 1ad44d194..94294a016 100644 --- a/mitmproxy/net/dns/types.py +++ b/mitmproxy/net/dns/types.py @@ -179,7 +179,15 @@ _STRINGS = { TA: "TA", DLV: "DLV", } +_INTS = {v: k for k, v in _STRINGS.items()} -def to_str(type: int) -> str: - return _STRINGS.get(type, f"TYPE({type})") +def to_str(type_: int) -> str: + return _STRINGS.get(type_, f"TYPE({type_})") + + +def from_str(type_: str) -> int: + try: + return _INTS[type_] + except KeyError: + return int(type_.removeprefix("TYPE(").removesuffix(")")) diff --git a/mitmproxy/net/local_ip.py b/mitmproxy/net/local_ip.py index 919d31dd3..865cf3e25 100644 --- a/mitmproxy/net/local_ip.py +++ b/mitmproxy/net/local_ip.py @@ -33,8 +33,8 @@ def get_local_ip6(reachable: str = "2001:4860:4860::8888") -> str | None: s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) s.connect((reachable, 80)) return s.getsockname()[0] # pragma: no cover - except OSError: - return None # pragma: no cover + except OSError: # pragma: no cover + return None finally: if s is not None: s.close() diff --git a/mitmproxy/proxy/layers/dns.py b/mitmproxy/proxy/layers/dns.py index 238b0feff..48e1a66a1 100644 --- a/mitmproxy/proxy/layers/dns.py +++ b/mitmproxy/proxy/layers/dns.py @@ -1,4 +1,5 @@ import struct +import time from dataclasses import dataclass from typing import List from typing import Literal @@ -43,7 +44,7 @@ class DnsErrorHook(commands.StartHook): def pack_message( - message: dns.Message, transport_protocol: Literal["tcp", "udp"] + message: dns.DNSMessage, transport_protocol: Literal["tcp", "udp"] ) -> bytes: packed = message.packed if transport_protocol == "tcp": @@ -68,7 +69,7 @@ class DNSLayer(layer.Layer): self.resp_buf = bytearray() def handle_request( - self, flow: dns.DNSFlow, msg: dns.Message + self, flow: dns.DNSFlow, msg: dns.DNSMessage ) -> layer.CommandGenerator[None]: flow.request = msg # if already set, continue and query upstream again yield DnsRequestHook(flow) @@ -91,7 +92,7 @@ class DNSLayer(layer.Layer): yield commands.SendData(self.context.server, packed) def handle_response( - self, flow: dns.DNSFlow, msg: dns.Message + self, flow: dns.DNSFlow, msg: dns.DNSMessage ) -> layer.CommandGenerator[None]: flow.response = msg yield DnsResponseHook(flow) @@ -108,13 +109,13 @@ class DNSLayer(layer.Layer): pack_message(servfail, flow.client_conn.transport_protocol), ) - def unpack_message(self, data: bytes, from_client: bool) -> List[dns.Message]: - msgs: List[dns.Message] = [] + def unpack_message(self, data: bytes, from_client: bool) -> List[dns.DNSMessage]: + msgs: List[dns.DNSMessage] = [] buf = self.req_buf if from_client else self.resp_buf if self.context.client.transport_protocol == "udp": - msgs.append(dns.Message.unpack(data)) + msgs.append(dns.DNSMessage.unpack(data, timestamp=time.time())) elif self.context.client.transport_protocol == "tcp": buf.extend(data) size = len(buf) @@ -134,7 +135,7 @@ class DNSLayer(layer.Layer): data = bytes(buf[offset : expected_size + offset]) offset += expected_size - msgs.append(dns.Message.unpack(data)) + msgs.append(dns.DNSMessage.unpack(data, timestamp=time.time())) del buf[:offset] return msgs @@ -150,7 +151,7 @@ class DNSLayer(layer.Layer): from_client = event.connection is self.context.client if isinstance(event, events.DataReceived): - msgs: List[dns.Message] = [] + msgs: List[dns.DNSMessage] = [] try: msgs = self.unpack_message(event.data, from_client) except struct.error as e: diff --git a/mitmproxy/test/tflow.py b/mitmproxy/test/tflow.py index 7aab2e3f8..ec779da87 100644 --- a/mitmproxy/test/tflow.py +++ b/mitmproxy/test/tflow.py @@ -1,3 +1,4 @@ +import time import uuid from wsproto.frame_protocol import Opcode @@ -123,8 +124,8 @@ def tdnsflow( *, client_conn: connection.Client | None = None, server_conn: connection.Server | None = None, - req: dns.Message | None = None, - resp: bool | dns.Message = False, + req: dns.DNSMessage | None = None, + resp: bool | dns.DNSMessage = False, err: bool | flow.Error = False, live: bool = True, ) -> dns.DNSFlow: @@ -144,11 +145,11 @@ def tdnsflow( if err is True: err = terr() - assert resp is False or isinstance(resp, dns.Message) + assert resp is False or isinstance(resp, dns.DNSMessage) assert err is False or isinstance(err, flow.Error) f = dns.DNSFlow(client_conn, server_conn) - f.timestamp_created = req.timestamp + f.timestamp_created = req.timestamp or time.time() f.request = req f.response = resp or None f.error = err or None diff --git a/mitmproxy/test/tutils.py b/mitmproxy/test/tutils.py index 4a2fb67f3..62abfb1e9 100644 --- a/mitmproxy/test/tutils.py +++ b/mitmproxy/test/tutils.py @@ -2,11 +2,7 @@ from mitmproxy import dns from mitmproxy import http -def tdnsreq(**kwargs) -> dns.Message: - """ - Returns: - mitmproxy.dns.Message - """ +def tdnsreq(**kwargs) -> dns.DNSMessage: default = dict( timestamp=946681200, id=42, @@ -24,14 +20,10 @@ def tdnsreq(**kwargs) -> dns.Message: additionals=[], ) default.update(kwargs) - return dns.Message(**default) # type: ignore + return dns.DNSMessage(**default) # type: ignore -def tdnsresp(**kwargs) -> dns.Message: - """ - Returns: - mitmproxy.dns.Message - """ +def tdnsresp(**kwargs) -> dns.DNSMessage: default = dict( timestamp=946681201, id=42, @@ -56,7 +48,7 @@ def tdnsresp(**kwargs) -> dns.Message: additionals=[], ) default.update(kwargs) - return dns.Message(**default) # type: ignore + return dns.DNSMessage(**default) # type: ignore def treq(**kwargs) -> http.Request: diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index ad97b418e..773d24d9d 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -782,8 +782,11 @@ def format_flow( error_message=error_message, ) elif isinstance(f, DNSFlow): - if f.response: + if f.request.timestamp and f.response and f.response.timestamp: duration = f.response.timestamp - f.request.timestamp + else: + duration = None + if f.response: response_code_str: str | None = dns.response_codes.to_str( f.response.response_code ) @@ -792,7 +795,6 @@ def format_flow( ) answer = ", ".join(str(x) for x in f.response.answers) else: - duration = None response_code_str = None response_code_http_equiv = 0 answer = None diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py index 06afdca08..628320506 100644 --- a/mitmproxy/tools/console/consoleaddons.py +++ b/mitmproxy/tools/console/consoleaddons.py @@ -15,6 +15,7 @@ from mitmproxy import http from mitmproxy import log from mitmproxy import tcp from mitmproxy import udp +from mitmproxy.contentviews import ContentviewMessage from mitmproxy.exceptions import CommandError from mitmproxy.log import ALERT from mitmproxy.tools.console import keymap @@ -398,50 +399,58 @@ class ConsoleAddon: flow = self.master.view.focus.flow focus_options = [] + try: + view_name = self.master.commands.call("console.flowview.mode") + except CommandError: + view_name = "auto" + + def add_message_edit_option( + message_name: str, message: ContentviewMessage | None + ) -> None: + if message is None: + return + data, _ = contentviews.get_data(message) + cv = contentviews.registry.get_view( + data or b"", + contentviews.make_metadata(message, flow), + view_name, + ) + if isinstance(cv, contentviews.InteractiveContentview): + focus_options.append(f"{message_name} ({cv.name})") + if flow is None: raise exceptions.CommandError("No flow selected.") elif isinstance(flow, tcp.TCPFlow): - focus_options = ["tcp-message"] + focus_options.append("tcp-message") + add_message_edit_option("tcp-message", flow.messages[-1]) elif isinstance(flow, udp.UDPFlow): - focus_options = ["udp-message"] + focus_options.append("udp-message") + add_message_edit_option("udp-message", flow.messages[-1]) elif isinstance(flow, http.HTTPFlow): - focus_options = [ - "cookies", - "urlencoded form", - "multipart form", - "path", - "method", - "query", - "reason", - "request-headers", - "response-headers", - "request-body", - "response-body", - "status_code", - "set-cookies", - "url", - ] - try: - view_name = self.master.commands.call("console.flowview.mode") - except CommandError: - view_name = "auto" - request_cv = contentviews.registry.get_view( - contentviews.get_data(flow.request)[0] or b"", - contentviews.make_metadata(flow.request, flow), - view_name, + focus_options.extend( + [ + "cookies", + "urlencoded form", + "multipart form", + "path", + "method", + "query", + "reason", + "request-headers", + "response-headers", + "request-body", + "response-body", + "status_code", + "set-cookies", + "url", + ] ) - if isinstance(request_cv, contentviews.InteractiveContentview): - focus_options.append(f"request-body ({request_cv.name})") - if flow.response: - response_cv = contentviews.registry.get_view( - contentviews.get_data(flow.response)[0] or b"", - contentviews.make_metadata(flow.response, flow), - view_name, - ) - if isinstance(response_cv, contentviews.InteractiveContentview): - focus_options.append(f"response-body ({response_cv.name})") + add_message_edit_option("request-body", flow.request) + add_message_edit_option("response-body", flow.response) if flow.websocket: - focus_options.append("websocket-message") + add_message_edit_option( + "websocket-message", flow.websocket.messages[-1] + ) elif isinstance(flow, dns.DNSFlow): raise exceptions.CommandError( "Cannot edit DNS flows yet, please submit a patch." @@ -485,12 +494,20 @@ class ConsoleAddon: elif flow_part == "response-headers": self.master.switch_view("edit_focus_response_headers") elif m := re.match( - r"(?Prequest|response)-body \((?P.+)\)", flow_part + r"(?P(request|response)-body|(tcp|udp|websocket)-message)( \((?P.+)\))?", + flow_part, ): - if m["part"] == "request": - message = flow.request - else: - message = flow.response + match m["message"]: + case "request-body": + message = flow.request + case "response-body": + message = flow.response + case "tcp-message" | "udp-message": + message = flow.messages[-1] + case "websocket-message": + message = flow.websocket.messages[-1] + case _: + assert False, "should be exhaustive" cv = contentviews.registry.get(m["contentview"]) if not cv or not isinstance(cv, contentviews.InteractiveContentview): @@ -519,9 +536,8 @@ class ConsoleAddon: # cause problems. We strip trailing newlines by default, but this # behavior is configurable. if self.master.options.console_strip_trailing_newlines: - message.content = c.rstrip(b"\n") - else: - message.content = c + c = c.rstrip(b"\n") + message.content = c elif flow_part == "set-cookies": self.master.switch_view("edit_focus_setcookies") elif flow_part == "url": @@ -533,13 +549,14 @@ class ConsoleAddon: self.master.commands.call_strings( "console.command", ["flow.set", "@focus", flow_part] ) - elif flow_part in ["tcp-message", "udp-message"]: - message = flow.messages[-1] - c = self.master.spawn_editor(message.content or b"") - message.content = c.rstrip(b"\n") - elif flow_part == "websocket-message": - message = flow.websocket.messages[-1] + elif flow_part in ["tcp-message", "udp-message", "websocket-message"]: + if flow_part == "websocket-message": + message = flow.websocket.messages[-1] + else: + message = flow.messages[-1] c = self.master.spawn_editor(message.content or b"") + if self.master.options.console_strip_trailing_newlines: + c = c.rstrip(b"\n") message.content = c.rstrip(b"\n") def _grideditor(self): diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index 6532c9561..e41a0b1ac 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -12,6 +12,7 @@ from mitmproxy import dns from mitmproxy import http from mitmproxy import tcp from mitmproxy import udp +from mitmproxy.dns import DNSMessage from mitmproxy.tools.console import common from mitmproxy.tools.console import flowdetailview from mitmproxy.tools.console import layoutwidget @@ -394,25 +395,7 @@ class FlowDetails(tabs.Tabs): viewmode = self.master.commands.call("console.flowview.mode") msg, body = self.content_view(viewmode, conn) - cols = [ - urwid.Text( - [ - ("heading", msg), - ] - ), - urwid.Text( - [ - " ", - ("heading", "["), - ("heading_key", "m"), - ("heading", (":%s]" % viewmode)), - ], - align="right", - ), - ] - title = urwid.AttrMap(urwid.Columns(cols), "heading") - - txt.append(title) + txt.append(self._contentview_status_bar(msg, viewmode)) txt.extend(body) else: txt = [ @@ -428,8 +411,26 @@ class FlowDetails(tabs.Tabs): return searchable.Searchable(txt) def dns_message_text( - self, type: str, message: dns.Message | None + self, type: str, message: DNSMessage | None ) -> searchable.Searchable: + """ + Alternative: + if not message: + return searchable.Searchable([urwid.Text(("highlight", f"No {typ}."))]) + + viewmode = self.master.commands.call("console.flowview.mode") + pretty = contentviews.prettify_message(message, flow, viewmode) + chunks = mitmproxy_rs.syntax_highlight.highlight( + pretty.text, + language=pretty.syntax_highlight, + ) + + widget_lines = [ + self._contentview_status_bar(viewmode.capitalize(), viewmode), + urwid.Text(chunks) + ] + return searchable.Searchable(widget_lines) + """ # Keep in sync with web/src/js/components/FlowView/DnsMessages.tsx if message: diff --git a/pyproject.toml b/pyproject.toml index fd4fa8d89..f81de358f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ dependencies = [ "pyOpenSSL>=22.1,<=25.0.0", "pyparsing>=2.4.2,<=3.2.3", "pyperclip<=1.9.0,>=1.9.0", - "ruamel.yaml>=0.16,<=0.18.10", + "ruamel.yaml>=0.18.10,<=0.18.10", "sortedcontainers>=2.3,<=2.4.0", "tornado>=6.4.1,<=6.4.2", "typing-extensions>=4.13.2,<=4.14; python_version < '3.13'", @@ -285,6 +285,9 @@ section-order = ["future", "standard-library", "third-party", "local-folder","fi no-lines-before = ["first-party"] known-first-party = ["test", "mitmproxy", "mitmproxy_rs"] +[tool.tox] +env_list = ["lint", "mypy", "py"] + [tool.tox.env_run_base] runner = "uv-venv-lock-runner" package = "editable" diff --git a/test/mitmproxy/addons/test_proxyserver.py b/test/mitmproxy/addons/test_proxyserver.py index e67ca88b5..11679b590 100644 --- a/test/mitmproxy/addons/test_proxyserver.py +++ b/test/mitmproxy/addons/test_proxyserver.py @@ -290,16 +290,16 @@ async def test_dns(caplog_async, monkeypatch) -> None: s = await mitmproxy_rs.udp.open_udp_connection(*dns_addr) req = tdnsreq() s.write(req.packed) - resp = dns.Message.unpack(await s.read(65535)) + resp = dns.DNSMessage.unpack(await s.read(65535)) assert req.id == resp.id and "8.8.8.8" in str(resp) assert len(ps.connections) == 1 s.write(req.packed) - resp = dns.Message.unpack(await s.read(65535)) + resp = dns.DNSMessage.unpack(await s.read(65535)) assert req.id == resp.id and "8.8.8.8" in str(resp) assert len(ps.connections) == 1 req.id = req.id + 1 s.write(req.packed) - resp = dns.Message.unpack(await s.read(65535)) + resp = dns.DNSMessage.unpack(await s.read(65535)) assert req.id == resp.id and "8.8.8.8" in str(resp) assert len(ps.connections) == 1 (dns_conn,) = ps.connections.values() diff --git a/test/mitmproxy/contentviews/__init__.py b/test/mitmproxy/contentviews/__init__.py index 4caadc27c..e69de29bb 100644 --- a/test/mitmproxy/contentviews/__init__.py +++ b/test/mitmproxy/contentviews/__init__.py @@ -1,18 +0,0 @@ -import sys - -if sys.version_info < (3, 13): # pragma: no cover - from typing_extensions import deprecated -else: - from warnings import deprecated - - -@deprecated("Use `mitmproxy.contentviews.Contentview` instead.") -def full_eval(instance): - def call(data, **metadata): - x = instance(data, **metadata) - if x is None: - return None - name, generator = x - return name, list(generator) - - return call diff --git a/test/mitmproxy/contentviews/image/__init__.py b/test/mitmproxy/contentviews/_view_image/__init__.py similarity index 100% rename from test/mitmproxy/contentviews/image/__init__.py rename to test/mitmproxy/contentviews/_view_image/__init__.py diff --git a/test/mitmproxy/contentviews/image/test_image_parser.py b/test/mitmproxy/contentviews/_view_image/test_image_parser.py similarity index 87% rename from test/mitmproxy/contentviews/image/test_image_parser.py rename to test/mitmproxy/contentviews/_view_image/test_image_parser.py index c4180b6f2..2d8cd9848 100644 --- a/test/mitmproxy/contentviews/image/test_image_parser.py +++ b/test/mitmproxy/contentviews/_view_image/test_image_parser.py @@ -1,6 +1,6 @@ import pytest -from mitmproxy.contentviews.image import image_parser +from mitmproxy.contentviews._view_image import image_parser @pytest.mark.parametrize( @@ -137,7 +137,7 @@ def test_parse_gif(filename, metadata, tdata): ("jfif_version", "(1, 1)"), ("jfif_density", "(96, 96)"), ("jfif_unit", "1"), - ("comment", "b'mitmproxy test image'"), + ("comment", "mitmproxy test image"), ("Size", "256 x 256 px"), ], # check app1 @@ -159,10 +159,10 @@ def test_parse_gif(filename, metadata, tdata): ("jfif_unit", "1"), ( "comment", - "b'BARTOLOMEO DI FRUOSINO\\r\\n(b. ca. 1366, Firenze, d. 1441, " - "Firenze)\\r\\n\\r\\nInferno, from the Divine Comedy by Dante (Folio 1v)" - "\\r\\n1430-35\\r\\nTempera, gold, and silver on parchment, 365 x 265 mm" - "\\r\\nBiblioth\\xe8que Nationale, Paris\\r\\n\\r\\nThe codex in Paris " + "BARTOLOMEO DI FRUOSINO\r\n(b. ca. 1366, Firenze, d. 1441, " + "Firenze)\r\n\r\nInferno, from the Divine Comedy by Dante (Folio 1v)" + "\r\n1430-35\r\nTempera, gold, and silver on parchment, 365 x 265 mm" + "\r\nBiblioth\\xe8que Nationale, Paris\r\n\r\nThe codex in Paris " "contains the text of the Inferno, the first of three books of the Divine " "Comedy, the masterpiece of the Florentine poet Dante Alighieri (1265-1321)." " The codex begins with two full-page illuminations. On folio 1v Dante and " @@ -170,15 +170,15 @@ def test_parse_gif(filename, metadata, tdata): "nine different zones. Dante and Virgil are to wade through successive " "circles teeming with images of the damned. The gates of Hell appear in " "the middle, a scarlet row of open sarcophagi before them. Devils orchestrate" - " the movements of the wretched souls.\\r\\n\\r\\nThe vision of the fiery " + " the movements of the wretched souls.\r\n\r\nThe vision of the fiery " 'inferno follows a convention established by ' - "Nardo di Cione\\'s fresco in the church of Santa Maria Novella, Florence." + '(\'/html/n/nardo/strozzi3.html\')" HREF="/html/n/nardo/strozzi3.html">' + "Nardo di Cione's fresco in the church of Santa Maria Novella, Florence." " Of remarkable vivacity and intensity of expression, the illumination is " - "executed in Bartolomeo\\'s late style.\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n\\r\\n" - "--- Keywords: --------------\\r\\n\\r\\nAuthor: BARTOLOMEO DI FRUOSINO" - "\\r\\nTitle: Inferno, from the Divine Comedy by Dante (Folio 1v)\\r\\nTime-line:" - " 1401-1450\\r\\nSchool: Italian\\r\\nForm: illumination\\r\\nType: other\\r\\n'", + "executed in Bartolomeo's late style.\r\n\r\n\r\n\r\n\r\n\r\n\r\n" + "--- Keywords: --------------\r\n\r\nAuthor: BARTOLOMEO DI FRUOSINO" + "\r\nTitle: Inferno, from the Divine Comedy by Dante (Folio 1v)\r\nTime-line:" + " 1401-1450\r\nSchool: Italian\r\nForm: illumination\r\nType: other\r\n", ), ("Size", "750 x 1055 px"), ], diff --git a/test/mitmproxy/contentviews/_view_image/test_view.py b/test/mitmproxy/contentviews/_view_image/test_view.py new file mode 100644 index 000000000..41faa616a --- /dev/null +++ b/test/mitmproxy/contentviews/_view_image/test_view.py @@ -0,0 +1,29 @@ +from mitmproxy.contentviews import image +from mitmproxy.contentviews import Metadata + + +def test_view_image(tdata): + for img in [ + "mitmproxy/data/image.png", + "mitmproxy/data/image.gif", + "mitmproxy/data/all.jpeg", + "mitmproxy/data/image.ico", + ]: + with open(tdata.path(img), "rb") as f: + desc = image.prettify(f.read(), Metadata()) + assert img.split(".")[-1].upper() in desc + + assert image.prettify(b"flibble", Metadata()) == ("# Unknown Image\n") + + +def test_render_priority(): + assert image.render_priority(b"", Metadata(content_type="image/png")) + assert image.render_priority(b"", Metadata(content_type="image/jpeg")) + assert image.render_priority(b"", Metadata(content_type="image/gif")) + assert image.render_priority(b"", Metadata(content_type="image/vnd.microsoft.icon")) + assert image.render_priority(b"", Metadata(content_type="image/x-icon")) + assert image.render_priority(b"", Metadata(content_type="image/webp")) + assert image.render_priority( + b"", Metadata(content_type="image/future-unknown-format-42") + ) + assert not image.render_priority(b"", Metadata(content_type="image/svg+xml")) diff --git a/test/mitmproxy/contentviews/image/test_view.py b/test/mitmproxy/contentviews/image/test_view.py deleted file mode 100644 index 61c0c6379..000000000 --- a/test/mitmproxy/contentviews/image/test_view.py +++ /dev/null @@ -1,32 +0,0 @@ -from .. import full_eval -from mitmproxy.contentviews import image - - -def test_view_image(tdata): - v = full_eval(image.ViewImage()) - for img in [ - "mitmproxy/data/image.png", - "mitmproxy/data/image.gif", - "mitmproxy/data/all.jpeg", - "mitmproxy/data/image.ico", - ]: - with open(tdata.path(img), "rb") as f: - viewname, lines = v(f.read()) - assert img.split(".")[-1].upper() in viewname - - assert v(b"flibble") == ( - "Unknown Image", - [[("header", "Image Format: "), ("text", "unknown")]], - ) - - -def test_render_priority(): - v = image.ViewImage() - assert v.render_priority(b"", content_type="image/png") - assert v.render_priority(b"", content_type="image/jpeg") - assert v.render_priority(b"", content_type="image/gif") - assert v.render_priority(b"", content_type="image/vnd.microsoft.icon") - assert v.render_priority(b"", content_type="image/x-icon") - assert v.render_priority(b"", content_type="image/webp") - assert v.render_priority(b"", content_type="image/future-unknown-format-42") - assert not v.render_priority(b"", content_type="image/svg+xml") diff --git a/test/mitmproxy/contentviews/test__registry.py b/test/mitmproxy/contentviews/test__registry.py index 9b84bc644..36bcbaa30 100644 --- a/test/mitmproxy/contentviews/test__registry.py +++ b/test/mitmproxy/contentviews/test__registry.py @@ -68,4 +68,4 @@ def test_render_priority_error(caplog): v = registry.get_view(b"data", Metadata()) assert v.name == "Example" - assert "Error in render_priority" in caplog.text + assert "Error in FailingRenderPriority.render_priority" in caplog.text diff --git a/test/mitmproxy/contentviews/test__utils.py b/test/mitmproxy/contentviews/test__utils.py index 41671e716..571fa6b22 100644 --- a/test/mitmproxy/contentviews/test__utils.py +++ b/test/mitmproxy/contentviews/test__utils.py @@ -1,6 +1,10 @@ from mitmproxy import tcp +from mitmproxy.contentviews._utils import byte_pairs_to_str_pairs from mitmproxy.contentviews._utils import get_data from mitmproxy.contentviews._utils import make_metadata +from mitmproxy.contentviews._utils import merge_repeated_keys +from mitmproxy.contentviews._utils import yaml_dumps +from mitmproxy.contentviews._utils import yaml_loads from mitmproxy.test import taddons from mitmproxy.test import tflow @@ -45,6 +49,14 @@ class TestMetadata: assert metadata.websocket_message == msg assert metadata.flow == f + def test_make_metadata_dns(self): + with taddons.context(): + f = tflow.tdnsflow() + msg = f.request + metadata = make_metadata(msg, f) + assert metadata.dns_message == msg + assert metadata.flow == f + class TestGetData: def test_get_data_regular_content(self): @@ -69,3 +81,34 @@ class TestGetData: content, enc = get_data(f.request) assert content == b"invalid" assert enc == "[cannot decode]" + + +def test_yaml_dumps(): + assert yaml_dumps({}) == "" + assert yaml_dumps({"foo": "bar"}) == "foo: bar\n" + + +def test_yaml_loads(): + assert yaml_loads("") is None + assert yaml_loads("foo: bar\n") == {"foo": "bar"} + + +def test_merge_repeated_keys(): + assert merge_repeated_keys([]) == {} + assert merge_repeated_keys([("foo", "bar")]) == {"foo": "bar"} + assert merge_repeated_keys([("foo", "bar"), ("foo", "baz")]) == { + "foo": ["bar", "baz"] + } + assert merge_repeated_keys( + [ + ("foo", "bar"), + ("foo", "baz"), + ("foo", "qux"), + ("bar", "quux"), + ] + ) == {"foo": ["bar", "baz", "qux"], "bar": "quux"} + + +def test_byte_pairs_to_str_pairs(): + assert list(byte_pairs_to_str_pairs([(b"foo", b"bar")])) == [("foo", "bar")] + assert list(byte_pairs_to_str_pairs([(b"\xfa", b"\xff")])) == [(r"\xfa", r"\xff")] diff --git a/test/mitmproxy/contentviews/test__view_dns.py b/test/mitmproxy/contentviews/test__view_dns.py new file mode 100644 index 000000000..e6b97e361 --- /dev/null +++ b/test/mitmproxy/contentviews/test__view_dns.py @@ -0,0 +1,74 @@ +import struct + +import pytest + +from mitmproxy.contentviews import Metadata +from mitmproxy.contentviews._view_dns import dns +from mitmproxy.tcp import TCPMessage + +DNS_HTTPS_RECORD_RESPONSE = bytes.fromhex( + "00008180000100010000000107746c732d656368036465760000410001c00c004100010000003c00520001000005004b0049fe0d00" + "452b00200020015881d41a3e2ef8f2208185dc479245d20624ddd0918a8056f2e26af47e2628000800010001000100034012707562" + "6c69632e746c732d6563682e646576000000002904d0000000000000" +) +DNS_A_QUERY = bytes.fromhex("002a0100000100000000000003646e7306676f6f676c650000010001") +TCP_MESSAGE = struct.pack("!H", len(DNS_A_QUERY)) + DNS_A_QUERY + + +def test_simple(): + assert ( + dns.prettify(DNS_HTTPS_RECORD_RESPONSE, Metadata()) + == r"""id: 0 +query: false +op_code: QUERY +authoritative_answer: false +truncation: false +recursion_desired: true +recursion_available: true +response_code: NOERROR +questions: +- name: tls-ech.dev + type: HTTPS + class: IN +answers: +- name: tls-ech.dev + type: HTTPS + class: IN + ttl: 60 + data: + target_name: '' + priority: 1 + ech: \x00I\xfe\r\x00E+\x00 \x00 \x01X\x81\xd4\x1a>.\xf8\xf2 + \x81\x85\xdcG\x92E\xd2\x06$\xdd\xd0\x91\x8a\x80V\xf2\xe2j\xf4~&(\x00\x08\x00\x01\x00\x01\x00\x01\x00\x03@\x12public.tls-ech.dev\x00\x00 +authorities: [] +additionals: +- name: '' + type: OPT + class: CLASS(1232) + ttl: 0 + data: 0x +size: 82 +""" + ) + + +def test_invalid(): + with pytest.raises(Exception): + dns.prettify(b"foobar", Metadata()) + + +def test_tcp(): + assert "type: A" in dns.prettify( + TCP_MESSAGE, Metadata(tcp_message=TCPMessage(False, TCP_MESSAGE, 946681204.2)) + ) + + +def test_roundtrip(): + meta = Metadata() + assert dns.reencode(dns.prettify(DNS_A_QUERY, meta), meta) == DNS_A_QUERY + + +def test_render_priority(): + assert dns.render_priority(b"", Metadata(content_type="application/dns-message")) + assert not dns.render_priority(b"", Metadata(content_type="text/plain")) + assert not dns.render_priority(b"", Metadata()) diff --git a/test/mitmproxy/contentviews/test__view_graphql.py b/test/mitmproxy/contentviews/test__view_graphql.py new file mode 100644 index 000000000..665b2e096 --- /dev/null +++ b/test/mitmproxy/contentviews/test__view_graphql.py @@ -0,0 +1,45 @@ +import pytest + +from mitmproxy.contentviews import Metadata +from mitmproxy.contentviews._view_graphql import format_graphql +from mitmproxy.contentviews._view_graphql import format_query_list +from mitmproxy.contentviews._view_graphql import graphql + + +def test_render_priority(): + assert 2 == graphql.render_priority( + b"""{"query": "query P { \\n }"}""", Metadata(content_type="application/json") + ) + assert 2 == graphql.render_priority( + b"""[{"query": "query P { \\n }"}]""", Metadata(content_type="application/json") + ) + assert 0 == graphql.render_priority( + b"""[{"query": "query P { \\n }"}]""", Metadata(content_type="text/html") + ) + assert 0 == graphql.render_priority( + b"""[{"xquery": "query P { \\n }"}]""", + Metadata(content_type="application/json"), + ) + assert 0 == graphql.render_priority( + b"""[]""", Metadata(content_type="application/json") + ) + assert 0 == graphql.render_priority(b"}", Metadata(content_type="application/json")) + + +def test_format_graphql(): + assert format_graphql({"query": "query P { \\n }"}) + + +def test_format_query_list(): + assert format_query_list([{"query": "query P { \\n }"}]) + + +def test_view_graphql(): + assert graphql.prettify( + b"""{"query": "query P { \\n }"}""", Metadata(content_type="application/json") + ) + assert graphql.prettify( + b"""[{"query": "query P { \\n }"}]""", Metadata(content_type="application/json") + ) + with pytest.raises(ValueError): + assert graphql.prettify(b'"valid json"', Metadata()) diff --git a/test/mitmproxy/contentviews/test__view_javascript.py b/test/mitmproxy/contentviews/test__view_javascript.py new file mode 100644 index 000000000..aadeac533 --- /dev/null +++ b/test/mitmproxy/contentviews/test__view_javascript.py @@ -0,0 +1,41 @@ +import pytest + +from mitmproxy.contentviews import Metadata +from mitmproxy.contentviews._view_javascript import beautify +from mitmproxy.contentviews._view_javascript import javascript + + +def test_view_javascript(): + assert javascript.prettify(b"[1, 2, 3]", Metadata()) + assert javascript.prettify(b"[1, 2, 3", Metadata()) + assert javascript.prettify(b"function(a){[1, 2, 3]}", Metadata()) == ( + "function(a) {\n [1, 2, 3]\n}\n" + ) + assert javascript.prettify(b"\xfe", Metadata()) # invalid utf-8 + + +@pytest.mark.parametrize( + "filename", + [ + "simple.js", + ], +) +def test_format_xml(filename, tdata): + path = tdata.path("mitmproxy/contentviews/test_js_data/" + filename) + with open(path) as f: + input = f.read() + with open("-formatted.".join(path.rsplit(".", 1))) as f: + expected = f.read() + js = beautify(input) + assert js == expected + + +def test_render_priority(): + assert javascript.render_priority( + b"data", Metadata(content_type="application/x-javascript") + ) + assert javascript.render_priority( + b"data", Metadata(content_type="application/javascript") + ) + assert javascript.render_priority(b"data", Metadata(content_type="text/javascript")) + assert not javascript.render_priority(b"data", Metadata(content_type="text/plain")) diff --git a/test/mitmproxy/contentviews/test_mqtt.py b/test/mitmproxy/contentviews/test__view_mqtt.py similarity index 67% rename from test/mitmproxy/contentviews/test_mqtt.py rename to test/mitmproxy/contentviews/test__view_mqtt.py index 9c966eb25..fcd62ef7a 100644 --- a/test/mitmproxy/contentviews/test_mqtt.py +++ b/test/mitmproxy/contentviews/test__view_mqtt.py @@ -1,7 +1,7 @@ import pytest -from . import full_eval -from mitmproxy.contentviews import mqtt +from mitmproxy.contentviews import Metadata +from mitmproxy.contentviews._view_mqtt import mqtt @pytest.mark.parametrize( @@ -33,38 +33,26 @@ from mitmproxy.contentviews import mqtt b"""\x10\xba\x01\x00\x04MQTT\x04\x06\x00\x1e\x00\x1156:6F:5E:6A:01:05\x00-""" b"""xxxx/yy/zzzzzz/56:6F:5E:6A:01:05/messages/out""" b"""\x00l{"body":{"parameters":null},"header":{"from":"56:6F:5E:6A:01:05","messageId":"disconnected","type":"event"}}""", - [ - "[CONNECT]", - "", - "Client Id: 56:6F:5E:6A:01:05", - "Will Topic: xxxx/yy/zzzzzz/56:6F:5E:6A:01:05/messages/out", - """Will Message: {"body":{"parameters":null},"header":{"from":"56:6F:5E:6A:01:05",""" - """"messageId":"disconnected","type":"event"}}""", - "User Name: None", - "Password: None", - ], + ( + "[CONNECT]\n" + "\n" + "Client Id: 56:6F:5E:6A:01:05\n" + "Will Topic: xxxx/yy/zzzzzz/56:6F:5E:6A:01:05/messages/out\n" + "Will Message: " + '{"body":{"parameters":null},"header":{"from":"56:6F:5E:6A:01:05","messageId":"disconnected","type":"event"}}\n' + "User Name: None\n" + "Password: None\n" + ), id="CONNECT", ), ], ) def test_view_mqtt(data, expected_text): """testing helper for single line messages""" - v = full_eval(mqtt.ViewMQTT()) - content_type, output = v(data) - assert content_type == "MQTT" - if isinstance(expected_text, list): - assert output == [[("text", text)] for text in expected_text] - else: - assert output == [[("text", expected_text)]] + assert mqtt.prettify(data, Metadata()) == expected_text @pytest.mark.parametrize("data", [b"\xc0\xff\xff\xff\xff"]) def test_mqtt_malformed(data): - v = full_eval(mqtt.ViewMQTT()) with pytest.raises(Exception): - v(data) - - -def test_render_priority(): - # missing: good MQTT heuristics. - assert mqtt.ViewMQTT().render_priority(b"") == 0 + mqtt.prettify(data, Metadata()) diff --git a/test/mitmproxy/contentviews/test__view_multipart.py b/test/mitmproxy/contentviews/test__view_multipart.py new file mode 100644 index 000000000..c9fbba763 --- /dev/null +++ b/test/mitmproxy/contentviews/test__view_multipart.py @@ -0,0 +1,42 @@ +import pytest + +from mitmproxy import http +from mitmproxy.contentviews import Metadata +from mitmproxy.contentviews._view_multipart import multipart + + +def meta(content_type: str) -> Metadata: + return Metadata( + content_type=content_type.split(";")[0], + http_message=http.Request.make( + "POST", "https://example.com/", headers={"content-type": content_type} + ), + ) + + +def test_view_multipart(): + v = b""" +--AaB03x +Content-Disposition: form-data; name="submit-name" + +Larry +--AaB03x + """.strip() + assert ( + multipart.prettify(v, meta("multipart/form-data; boundary=AaB03x")) + == "submit-name: Larry\n" + ) + + with pytest.raises(ValueError): + assert not multipart.prettify(v, Metadata()) + + assert not multipart.prettify(v, meta("multipart/form-data")) + + assert not multipart.prettify(v, meta("unparseable")) + + +def test_render_priority(): + assert multipart.render_priority( + b"data", Metadata(content_type="multipart/form-data") + ) + assert not multipart.render_priority(b"data", Metadata(content_type="text/plain")) diff --git a/test/mitmproxy/contentviews/test__view_query.py b/test/mitmproxy/contentviews/test__view_query.py new file mode 100644 index 000000000..f15da4914 --- /dev/null +++ b/test/mitmproxy/contentviews/test__view_query.py @@ -0,0 +1,23 @@ +import pytest + +from mitmproxy.contentviews import Metadata +from mitmproxy.contentviews._view_query import query +from mitmproxy.test import tutils + + +def test_view_query(): + d = "" + req = tutils.treq() + req.query = [("foo", "bar"), ("foo", "baz")] + out = query.prettify(d, Metadata(http_message=req)) + assert out == "foo:\n- bar\n- baz\n" + + with pytest.raises(ValueError): + query.prettify(d, Metadata()) + + +def test_render_priority(): + req = tutils.treq() + req.query = [("foo", "bar"), ("foo", "baz")] + assert query.render_priority(b"", Metadata(http_message=req)) + assert not query.render_priority(b"", Metadata()) diff --git a/test/mitmproxy/contentviews/test__view_socketio.py b/test/mitmproxy/contentviews/test__view_socketio.py new file mode 100644 index 000000000..7381cb82a --- /dev/null +++ b/test/mitmproxy/contentviews/test__view_socketio.py @@ -0,0 +1,41 @@ +import pytest + +from mitmproxy.contentviews import Metadata +from mitmproxy.contentviews._view_socketio import EngineIO +from mitmproxy.contentviews._view_socketio import parse_packet +from mitmproxy.contentviews._view_socketio import socket_io +from mitmproxy.contentviews._view_socketio import SocketIO +from mitmproxy.test import tflow + + +def test_parse_packet(): + assert parse_packet(b"0payload") == (EngineIO.OPEN, b"payload") + assert parse_packet(b"40") == (SocketIO.CONNECT, b"") + assert parse_packet(b"40payload") == (SocketIO.CONNECT, b"payload") + + +def test_view(): + with pytest.raises(Exception): + socket_io.prettify(b"HTTP/1.1", Metadata()) + with pytest.raises(Exception): + socket_io.prettify(b"GET", Metadata()) + assert socket_io.prettify(b"0", Metadata()) + assert socket_io.prettify(b"6", Metadata()) + assert socket_io.prettify(b"40", Metadata()) + with pytest.raises(Exception): + socket_io.prettify(b"4", Metadata()) + assert socket_io.prettify(b"42", Metadata()) + assert socket_io.prettify(b"42eventdata", Metadata()) + assert socket_io.prettify(b"2", Metadata()) == "" + + +def test_render_priority(): + assert not socket_io.render_priority(b"", Metadata()) + + flow = tflow.twebsocketflow() + assert not socket_io.render_priority(b"", Metadata(flow=flow)) + assert not socket_io.render_priority(b"message", Metadata(flow=flow)) + + flow.request.path = b"/asdf/socket.io/?..." + assert socket_io.render_priority(b"message", Metadata(flow=flow)) + assert not socket_io.render_priority(b"", Metadata(flow=flow)) diff --git a/test/mitmproxy/contentviews/test__view_urlencoded.py b/test/mitmproxy/contentviews/test__view_urlencoded.py new file mode 100644 index 000000000..825c7677b --- /dev/null +++ b/test/mitmproxy/contentviews/test__view_urlencoded.py @@ -0,0 +1,20 @@ +from mitmproxy.contentviews import Metadata +from mitmproxy.contentviews._view_urlencoded import urlencoded +from mitmproxy.net.http import url + + +def test_view_urlencoded(): + d = url.encode([("one", "two"), ("three", "four")]).encode() + assert urlencoded.prettify(d, Metadata()) == "one: two\nthree: four\n" + + d = url.encode([("adsfa", "")]).encode() + assert urlencoded.prettify(d, Metadata()) == "adsfa: ''\n" + + assert urlencoded.prettify(b"\xff\x00", Metadata()) == "\\xff\\x00: ''\n" + + +def test_render_priority(): + assert urlencoded.render_priority( + b"data", Metadata(content_type="application/x-www-form-urlencoded") + ) + assert not urlencoded.render_priority(b"data", Metadata(content_type="text/plain")) diff --git a/test/mitmproxy/contentviews/test__view_wbxml.py b/test/mitmproxy/contentviews/test__view_wbxml.py new file mode 100644 index 000000000..c72c6df29 --- /dev/null +++ b/test/mitmproxy/contentviews/test__view_wbxml.py @@ -0,0 +1,31 @@ +import pytest + +from mitmproxy.contentviews import Metadata +from mitmproxy.contentviews._view_wbxml import wbxml + +datadir = "mitmproxy/contentviews/test_wbxml_data/" + + +def test_wbxml(tdata): + assert wbxml.prettify(b"\x03\x01\x6a\x00", Metadata()) == '\n' + with pytest.raises(Exception): + wbxml.prettify(b"foo", Metadata()) + + # File taken from https://github.com/davidpshaw/PyWBXMLDecoder/tree/master/wbxml_samples + path = tdata.path(datadir + "data.wbxml") + with open(path, "rb") as f: + input = f.read() + with open("-formatted.".join(path.rsplit(".", 1))) as f: + expected = f.read() + + assert wbxml.prettify(input, Metadata()) == expected + + +def test_render_priority(): + assert wbxml.render_priority( + b"data", Metadata(content_type="application/vnd.wap.wbxml") + ) + assert wbxml.render_priority( + b"data", Metadata(content_type="application/vnd.ms-sync.wbxml") + ) + assert not wbxml.render_priority(b"data", Metadata(content_type="text/plain")) diff --git a/test/mitmproxy/contentviews/test__view_xml_html.py b/test/mitmproxy/contentviews/test__view_xml_html.py new file mode 100644 index 000000000..99a7affba --- /dev/null +++ b/test/mitmproxy/contentviews/test__view_xml_html.py @@ -0,0 +1,53 @@ +import pytest + +from mitmproxy import http +from mitmproxy.contentviews import Metadata +from mitmproxy.contentviews._view_xml_html import tokenize +from mitmproxy.contentviews._view_xml_html import xml_html + +datadir = "mitmproxy/contentviews/test_xml_html_data/" + + +def test_simple(tdata): + assert xml_html.prettify(b"foo", Metadata()) == "foo\n" + assert xml_html.prettify(b"", Metadata()) == "\n" + assert xml_html.prettify(b"<>", Metadata()) == "<>\n" + assert xml_html.prettify(b")" + + +def test_use_text(): + meta1 = Metadata() + meta2 = Metadata( + http_message=http.Response.make( + content=b"\xf8", + ) + ) + assert xml_html.prettify(b"\xf8", meta1) != "ø\n" + assert xml_html.prettify(b"\xf8", meta2) == "ø\n" + + +@pytest.mark.parametrize( + "filename", ["simple.html", "cdata.xml", "comment.xml", "inline.html", "test.html"] +) +def test_format_xml(filename, tdata): + path = tdata.path(datadir + filename) + with open(path, "rb") as f: + input = f.read() + with open("-formatted.".join(path.rsplit(".", 1))) as f: + expected = f.read() + + assert xml_html.prettify(input, Metadata()) == expected + + +def test_render_priority(): + assert xml_html.render_priority(b"data", Metadata(content_type="text/xml")) + assert xml_html.render_priority(b"data", Metadata(content_type="text/xml")) + assert xml_html.render_priority(b"data", Metadata(content_type="text/html")) + assert not xml_html.render_priority(b"data", Metadata(content_type="text/plain")) + assert not xml_html.render_priority(b"", Metadata(content_type="text/xml")) + assert xml_html.render_priority(b"", Metadata()) diff --git a/test/mitmproxy/contentviews/test_dns.py b/test/mitmproxy/contentviews/test_dns.py deleted file mode 100644 index f10296ade..000000000 --- a/test/mitmproxy/contentviews/test_dns.py +++ /dev/null @@ -1,21 +0,0 @@ -from . import full_eval -from mitmproxy.contentviews import dns - -DNS_HTTPS_RECORD_RESPONSE = bytes.fromhex( - "00008180000100010000000107746c732d656368036465760000410001c00c004100010000003c00520001000005004b0049fe0d00" - "452b00200020015881d41a3e2ef8f2208185dc479245d20624ddd0918a8056f2e26af47e2628000800010001000100034012707562" - "6c69632e746c732d6563682e646576000000002904d0000000000000" -) - - -def test_simple(): - v = full_eval(dns.ViewDns()) - assert v(DNS_HTTPS_RECORD_RESPONSE) - assert not v(b"foobar") - - -def test_render_priority(): - v = dns.ViewDns() - assert v.render_priority(b"", content_type="application/dns-message") - assert not v.render_priority(b"", content_type="text/plain") - assert not v.render_priority(b"") diff --git a/test/mitmproxy/contentviews/test_graphql.py b/test/mitmproxy/contentviews/test_graphql.py deleted file mode 100644 index d488a5b8d..000000000 --- a/test/mitmproxy/contentviews/test_graphql.py +++ /dev/null @@ -1,33 +0,0 @@ -from mitmproxy.contentviews import graphql - - -def test_render_priority(): - v = graphql.ViewGraphQL() - assert 2 == v.render_priority( - b"""{"query": "query P { \\n }"}""", content_type="application/json" - ) - assert 2 == v.render_priority( - b"""[{"query": "query P { \\n }"}]""", content_type="application/json" - ) - assert 0 == v.render_priority( - b"""[{"query": "query P { \\n }"}]""", content_type="text/html" - ) - assert 0 == v.render_priority( - b"""[{"xquery": "query P { \\n }"}]""", content_type="application/json" - ) - assert 0 == v.render_priority(b"""[]""", content_type="application/json") - assert 0 == v.render_priority(b"}", content_type="application/json") - - -def test_format_graphql(): - assert graphql.format_graphql({"query": "query P { \\n }"}) - - -def test_format_query_list(): - assert graphql.format_query_list([{"query": "query P { \\n }"}]) - - -def test_view_graphql(): - v = graphql.ViewGraphQL() - assert v(b"""{"query": "query P { \\n }"}""", content_type="application/json") - assert v(b"""[{"query": "query P { \\n }"}]""", content_type="application/json") diff --git a/test/mitmproxy/contentviews/test_javascript.py b/test/mitmproxy/contentviews/test_javascript.py deleted file mode 100644 index 64647446d..000000000 --- a/test/mitmproxy/contentviews/test_javascript.py +++ /dev/null @@ -1,39 +0,0 @@ -import pytest - -from . import full_eval -from mitmproxy.contentviews import javascript - - -def test_view_javascript(): - v = full_eval(javascript.ViewJavaScript()) - assert v(b"[1, 2, 3]") - assert v(b"[1, 2, 3") - assert v(b"function(a){[1, 2, 3]}") == ( - "JavaScript", - [[("text", "function(a) {")], [("text", " [1, 2, 3]")], [("text", "}")]], - ) - assert v(b"\xfe") # invalid utf-8 - - -@pytest.mark.parametrize( - "filename", - [ - "simple.js", - ], -) -def test_format_xml(filename, tdata): - path = tdata.path("mitmproxy/contentviews/test_js_data/" + filename) - with open(path) as f: - input = f.read() - with open("-formatted.".join(path.rsplit(".", 1))) as f: - expected = f.read() - js = javascript.beautify(input) - assert js == expected - - -def test_render_priority(): - v = javascript.ViewJavaScript() - assert v.render_priority(b"data", content_type="application/x-javascript") - assert v.render_priority(b"data", content_type="application/javascript") - assert v.render_priority(b"data", content_type="text/javascript") - assert not v.render_priority(b"data", content_type="text/plain") diff --git a/test/mitmproxy/contentviews/test_multipart.py b/test/mitmproxy/contentviews/test_multipart.py deleted file mode 100644 index a9b43c5f8..000000000 --- a/test/mitmproxy/contentviews/test_multipart.py +++ /dev/null @@ -1,35 +0,0 @@ -from . import full_eval -from mitmproxy.contentviews import multipart -from mitmproxy.test import tutils - - -def test_view_multipart(): - view = full_eval(multipart.ViewMultipart()) - v = b""" ---AaB03x -Content-Disposition: form-data; name="submit-name" - -Larry ---AaB03x - """.strip() - assert view(v, content_type="multipart/form-data; boundary=AaB03x") - - req = tutils.treq() - req.headers["content-type"] = "multipart/form-data; boundary=AaB03x" - req.content = v - - assert view( - v, content_type="multipart/form-data; boundary=AaB03x", http_message=req - ) - - assert not view(v) - - assert not view(v, content_type="multipart/form-data") - - assert not view(v, content_type="unparseable") - - -def test_render_priority(): - v = multipart.ViewMultipart() - assert v.render_priority(b"data", content_type="multipart/form-data") - assert not v.render_priority(b"data", content_type="text/plain") diff --git a/test/mitmproxy/contentviews/test_query.py b/test/mitmproxy/contentviews/test_query.py deleted file mode 100644 index b4b1408ef..000000000 --- a/test/mitmproxy/contentviews/test_query.py +++ /dev/null @@ -1,26 +0,0 @@ -from . import full_eval -from mitmproxy.contentviews import query -from mitmproxy.test import tutils - - -def test_view_query(): - d = "" - v = full_eval(query.ViewQuery()) - req = tutils.treq() - req.query = [("foo", "bar"), ("foo", "baz")] - f = v(d, http_message=req) - assert f[0] == "Query" - assert f[1] == [ - [("header", "foo: "), ("text", "bar")], - [("header", "foo: "), ("text", "baz")], - ] - - assert v(d) == ("Query", []) - - -def test_render_priority(): - view = query.ViewQuery() - req = tutils.treq() - req.query = [("foo", "bar"), ("foo", "baz")] - assert view.render_priority(b"", http_message=req) - assert not view.render_priority(b"") diff --git a/test/mitmproxy/contentviews/test_socketio.py b/test/mitmproxy/contentviews/test_socketio.py deleted file mode 100644 index a3257137d..000000000 --- a/test/mitmproxy/contentviews/test_socketio.py +++ /dev/null @@ -1,58 +0,0 @@ -from hypothesis import given -from hypothesis.strategies import binary - -from . import full_eval -from mitmproxy.contentviews.socketio import EngineIO -from mitmproxy.contentviews.socketio import format_packet -from mitmproxy.contentviews.socketio import parse_packet -from mitmproxy.contentviews.socketio import SocketIO -from mitmproxy.contentviews.socketio import ViewSocketIO -from mitmproxy.test import tflow - - -def test_parse_packet(): - assert parse_packet(b"0payload") == (EngineIO.OPEN, b"payload") - assert parse_packet(b"40") == (SocketIO.CONNECT, b"") - assert parse_packet(b"40payload") == (SocketIO.CONNECT, b"payload") - - -def test_format_packet(): - assert list(format_packet(SocketIO.EVENT, b"data")[1]) == [ - [ - ("content_none", "SocketIO.EVENT "), - ("text", b"data"), - ], - ] - assert not list(format_packet(EngineIO.PING, b"")[1]) - assert not list(format_packet(SocketIO.ACK, b"")[1]) - - -def test_view(): - v = full_eval(ViewSocketIO()) - assert not v(b"HTTP/1.1") - assert not v(b"GET") - assert v(b"0") - assert v(b"6") - assert v(b"40") - assert not v(b"4") - assert v(b"42") - assert v(b"42eventdata") - - -@given(binary()) -def test_view_doesnt_crash(data): - v = full_eval(ViewSocketIO()) - v(data) - - -def test_render_priority(): - v = ViewSocketIO() - assert not v.render_priority(b"") - - flow = tflow.twebsocketflow() - assert not v.render_priority(b"", flow=flow) - assert not v.render_priority(b"message", flow=flow) - - flow.request.path = b"/asdf/socket.io/?..." - assert v.render_priority(b"message", flow=flow) - assert not v.render_priority(b"", flow=flow) diff --git a/test/mitmproxy/contentviews/test_urlencoded.py b/test/mitmproxy/contentviews/test_urlencoded.py deleted file mode 100644 index a91dab736..000000000 --- a/test/mitmproxy/contentviews/test_urlencoded.py +++ /dev/null @@ -1,21 +0,0 @@ -from . import full_eval -from mitmproxy.contentviews import urlencoded -from mitmproxy.net.http import url - - -def test_view_urlencoded(): - v = full_eval(urlencoded.ViewURLEncoded()) - - d = url.encode([("one", "two"), ("three", "four")]).encode() - assert v(d) - - d = url.encode([("adsfa", "")]).encode() - assert v(d) - - assert not v(b"\xff\x00") - - -def test_render_priority(): - v = urlencoded.ViewURLEncoded() - assert v.render_priority(b"data", content_type="application/x-www-form-urlencoded") - assert not v.render_priority(b"data", content_type="text/plain") diff --git a/test/mitmproxy/contentviews/test_wbxml.py b/test/mitmproxy/contentviews/test_wbxml.py deleted file mode 100644 index d81a1c543..000000000 --- a/test/mitmproxy/contentviews/test_wbxml.py +++ /dev/null @@ -1,29 +0,0 @@ -from . import full_eval -from mitmproxy.contentviews import wbxml - -datadir = "mitmproxy/contentviews/test_wbxml_data/" - - -def test_wbxml(tdata): - v = full_eval(wbxml.ViewWBXML()) - - assert v(b"\x03\x01\x6a\x00") == ("WBXML", [[("text", '')]]) - assert v(b"foo") is None - - path = tdata.path( - datadir + "data.wbxml" - ) # File taken from https://github.com/davidpshaw/PyWBXMLDecoder/tree/master/wbxml_samples - with open(path, "rb") as f: - input = f.read() - with open("-formatted.".join(path.rsplit(".", 1))) as f: - expected = f.read() - - p = wbxml.ASCommandResponse.ASCommandResponse(input) - assert p.xmlString == expected - - -def test_render_priority(): - v = wbxml.ViewWBXML() - assert v.render_priority(b"data", content_type="application/vnd.wap.wbxml") - assert v.render_priority(b"data", content_type="application/vnd.ms-sync.wbxml") - assert not v.render_priority(b"data", content_type="text/plain") diff --git a/test/mitmproxy/contentviews/test_xml_html.py b/test/mitmproxy/contentviews/test_xml_html.py deleted file mode 100644 index de2b8d59f..000000000 --- a/test/mitmproxy/contentviews/test_xml_html.py +++ /dev/null @@ -1,42 +0,0 @@ -import pytest - -from . import full_eval -from mitmproxy.contentviews import xml_html - -datadir = "mitmproxy/contentviews/test_xml_html_data/" - - -def test_simple(tdata): - v = full_eval(xml_html.ViewXmlHtml()) - assert v(b"foo") == ("XML", [[("text", "foo")]]) - assert v(b"") == ("HTML", [[("text", "")]]) - assert v(b"<>") == ("XML", [[("text", "<>")]]) - assert v(b")" - - -@pytest.mark.parametrize( - "filename", ["simple.html", "cdata.xml", "comment.xml", "inline.html", "test.html"] -) -def test_format_xml(filename, tdata): - path = tdata.path(datadir + filename) - with open(path) as f: - input = f.read() - with open("-formatted.".join(path.rsplit(".", 1))) as f: - expected = f.read() - tokens = xml_html.tokenize(input) - assert xml_html.format_xml(tokens) == expected - - -def test_render_priority(): - v = xml_html.ViewXmlHtml() - assert v.render_priority(b"data", content_type="text/xml") - assert v.render_priority(b"data", content_type="text/xml") - assert v.render_priority(b"data", content_type="text/html") - assert not v.render_priority(b"data", content_type="text/plain") - assert not v.render_priority(b"", content_type="text/xml") - assert v.render_priority(b"") diff --git a/test/mitmproxy/net/dns/test_classes.py b/test/mitmproxy/net/dns/test_classes.py index 81c4e10ea..7afda9d58 100644 --- a/test/mitmproxy/net/dns/test_classes.py +++ b/test/mitmproxy/net/dns/test_classes.py @@ -1,7 +1,11 @@ from mitmproxy.net.dns import classes -def test_simple(): - assert classes.IN == 1 +def test_to_str(): assert classes.to_str(classes.IN) == "IN" assert classes.to_str(0) == "CLASS(0)" + + +def test_from_str(): + assert classes.from_str("IN") == classes.IN + assert classes.from_str("CLASS(0)") == 0 diff --git a/test/mitmproxy/net/dns/test_https_records.py b/test/mitmproxy/net/dns/test_https_records.py index 77db52850..78407066e 100644 --- a/test/mitmproxy/net/dns/test_https_records.py +++ b/test/mitmproxy/net/dns/test_https_records.py @@ -86,7 +86,7 @@ class TestHTTPSRecords: except struct.error: pass - def test_str(self): + def test_to_json(self): params = { 0: b"\x00", 1: b"\x01", @@ -97,13 +97,39 @@ class TestHTTPSRecords: 6: b"\x05", } record = https_records.HTTPSRecord(1, "example.com", params) - assert ( - str(record) - == "priority: 1 target_name: 'example.com' {'mandatory': b'\\x00', 'alpn': b'\\x01', 'no_default_alpn': b'', 'port': b'\\x02', 'ipv4hint': b'\\x03', 'ech': b'\\x04', 'ipv6hint': b'\\x05'}" - ) + assert record.to_json() == { + "alpn": r"\x01", + "ech": r"\x04", + "ipv4hint": r"\x03", + "ipv6hint": r"\x05", + "mandatory": r"\x00", + "no_default_alpn": "", + "port": r"\x02", + "priority": 1, + "target_name": "example.com", + } params = {111: b"\x00"} record = https_records.HTTPSRecord(1, "example.com", params) + assert record.to_json() == { + 111: r"\x00", + "priority": 1, + "target_name": "example.com", + } assert ( - str(record) == "priority: 1 target_name: 'example.com' {'key111': b'\\x00'}" + str(record) + == r"{'target_name': 'example.com', 'priority': 1, 111: '\\x00'}" ) + + def test_from_json(self): + record = https_records.HTTPSRecord.from_json( + { + "mandatory": r"\x00", + "no_default_alpn": "", + "priority": 1, + "target_name": "example.com", + } + ) + assert record.target_name == "example.com" + assert record.priority == 1 + assert record.params == {0: b"\x00", 2: b""} diff --git a/test/mitmproxy/net/dns/test_op_codes.py b/test/mitmproxy/net/dns/test_op_codes.py index f2641a58a..4c2932614 100644 --- a/test/mitmproxy/net/dns/test_op_codes.py +++ b/test/mitmproxy/net/dns/test_op_codes.py @@ -1,7 +1,11 @@ from mitmproxy.net.dns import op_codes -def test_simple(): - assert op_codes.QUERY == 0 +def test_to_str(): assert op_codes.to_str(op_codes.QUERY) == "QUERY" assert op_codes.to_str(100) == "OPCODE(100)" + + +def test_from_str(): + assert op_codes.from_str("QUERY") == op_codes.QUERY + assert op_codes.from_str("OPCODE(100)") == 100 diff --git a/test/mitmproxy/net/dns/test_response_codes.py b/test/mitmproxy/net/dns/test_response_codes.py index 28eef77bb..4c6bf04f2 100644 --- a/test/mitmproxy/net/dns/test_response_codes.py +++ b/test/mitmproxy/net/dns/test_response_codes.py @@ -1,8 +1,15 @@ from mitmproxy.net.dns import response_codes -def test_simple(): - assert response_codes.NOERROR == 0 +def test_to_str(): assert response_codes.to_str(response_codes.NOERROR) == "NOERROR" assert response_codes.to_str(100) == "RCODE(100)" + + +def test_from_str(): + assert response_codes.from_str("NOERROR") == response_codes.NOERROR + assert response_codes.from_str("RCODE(100)") == 100 + + +def test_http_equiv_status_code(): assert response_codes.http_equiv_status_code(response_codes.NOERROR) == 200 diff --git a/test/mitmproxy/net/dns/test_types.py b/test/mitmproxy/net/dns/test_types.py index 21e65417a..3cb305eca 100644 --- a/test/mitmproxy/net/dns/test_types.py +++ b/test/mitmproxy/net/dns/test_types.py @@ -1,7 +1,11 @@ from mitmproxy.net.dns import types -def test_simple(): - assert types.A == 1 +def test_to_str(): assert types.to_str(types.A) == "A" assert types.to_str(0) == "TYPE(0)" + + +def test_from_str(): + assert types.from_str("A") == types.A + assert types.from_str("TYPE(0)") == 0 diff --git a/test/mitmproxy/test_dns.py b/test/mitmproxy/test_dns.py index b6fa7c7b2..51e49ee85 100644 --- a/test/mitmproxy/test_dns.py +++ b/test/mitmproxy/test_dns.py @@ -5,6 +5,7 @@ import pytest from mitmproxy import dns from mitmproxy import flowfilter +from mitmproxy.dns import DNSMessage from mitmproxy.test import tflow from mitmproxy.test import tutils @@ -40,7 +41,7 @@ class TestResourceRecord: record = dns.https_records.HTTPSRecord(1, "example.com", params) assert ( str(dns.ResourceRecord.HTTPS("example.com", record)) - == "priority: 1 target_name: 'example.com' {'mandatory': b'\\x00', 'alpn': b'\\x01', 'no_default_alpn': b'', 'port': b'\\x02', 'ipv4hint': b'\\x03', 'ech': b'\\x04', 'ipv6hint': b'\\x05'}" + == r"{'target_name': 'example.com', 'priority': 1, 'mandatory': '\\x00', 'alpn': '\\x01', 'no_default_alpn': '', 'port': '\\x02', 'ipv4hint': '\\x03', 'ech': '\\x04', 'ipv6hint': '\\x05'}" ) assert ( str( @@ -113,7 +114,11 @@ class TestResourceRecord: class TestMessage: - def test_json(self): + def test_content_alias(self): + m = tutils.tdnsresp() + assert m.content == m.packed + + def test_to_json(self): resp = tutils.tdnsresp() json = resp.to_json() assert json["id"] == resp.id @@ -122,6 +127,62 @@ class TestMessage: assert len(json["answers"]) == len(resp.answers) assert json["answers"][0]["data"] == str(resp.answers[0]) + assert DNSMessage.from_json(resp.to_json()) == resp + + def test_from_json(self): + assert DNSMessage.from_json( + { + **tutils.tdnsresp().to_json(), + "answers": [ + { + "name": "dns.google", + "type": "A", + "class": "IN", + "ttl": 32, + "data": "8.8.8.8", + }, + { + "name": "dns.google", + "type": "AAAA", + "class": "IN", + "ttl": 32, + "data": "::1", + }, + { + "name": "dns.google", + "type": "CNAME", + "class": "IN", + "ttl": 32, + "data": "alias.google", + }, + { + "name": "dns.google", + "type": "TXT", + "class": "IN", + "ttl": 32, + "data": "random text", + }, + { + "name": "dns.google", + "type": "HTTPS", + "class": "IN", + "ttl": 32, + "data": { + "target_name": "dns.google", + "priority": 42, + }, + }, + { + "name": "dns.google", + "type": "TYPE(42)", + "class": "IN", + "ttl": 32, + "data": "0xffff", + }, + ], + } + ) + def test_responses(self): req = tutils.tdnsreq() resp = tutils.tdnsresp() @@ -149,12 +210,12 @@ class TestMessage: def test(what: str, min: int, max: int): req = tutils.tdnsreq() setattr(req, what, min) - assert getattr(dns.Message.unpack(req.packed), what) == min + assert getattr(dns.DNSMessage.unpack(req.packed), what) == min setattr(req, what, min - 1) with pytest.raises(ValueError): req.packed setattr(req, what, max) - assert getattr(dns.Message.unpack(req.packed), what) == max + assert getattr(dns.DNSMessage.unpack(req.packed), what) == max setattr(req, what, max + 1) with pytest.raises(ValueError): req.packed @@ -165,9 +226,8 @@ class TestMessage: test("response_code", 0, 0b1111) def test_packing(self): - def assert_eq(m: dns.Message, b: bytes) -> None: - m_b = dns.Message.unpack(b) - m_b.timestamp = m.timestamp + def assert_eq(m: dns.DNSMessage, b: bytes) -> None: + m_b = dns.DNSMessage.unpack(b, m.timestamp) assert m_b == m assert m_b.packed == m.packed @@ -176,7 +236,7 @@ class TestMessage: b"\x00\x2a\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x03dns\x06google\x00\x00\x01\x00\x01", ) with pytest.raises(struct.error): - dns.Message.unpack( + dns.DNSMessage.unpack( b"\x00\x2a\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x03dns\x06google\x00\x00\x01\x00\x01\x00" ) assert_eq( @@ -188,22 +248,22 @@ class TestMessage: ), ) with pytest.raises(struct.error): # question error - dns.Message.unpack( + dns.DNSMessage.unpack( b"\x00\x2a\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x03dns\x06goo" ) with pytest.raises(struct.error): # rr length error - dns.Message.unpack( + dns.DNSMessage.unpack( b"\x00\x2a\x81\x80\x00\x01\x00\x02\x00\x00\x00\x00\x03dns\x06google\x00\x00\x01\x00\x01" + b"\xc0\x0c\x00\x01\x00\x01\x00\x00\x00 \x00\x04\x08\x08\x08\x08\xc0\x0c\x00\x01\x00\x01\x00\x00\x00 \x00\x04\x08\x08\x04" ) - txt = dns.Message.unpack( + txt = dns.DNSMessage.unpack( b"V\x1a\x81\x80\x00\x01\x00\x01\x00\x01\x00\x01\x05alive\x06github\x03com\x00\x00" + b"\x10\x00\x01\xc0\x0c\x00\x05\x00\x01\x00\x00\x0b\xc6\x00\x07\x04live\xc0\x12\xc0\x12\x00\x06\x00\x01" + b"\x00\x00\x03\x84\x00H\x07ns-1707\tawsdns-21\x02co\x02uk\x00\x11awsdns-hostmaster\x06amazon\xc0\x19\x00" + b"\x00\x00\x01\x00\x00\x1c \x00\x00\x03\x84\x00\x12u\x00\x00\x01Q\x80\x00\x00)\x02\x00\x00\x00\x00\x00\x00\x00" ) assert txt.answers[0].domain_name == "live.github.com" - invalid_rr_domain_name = dns.Message.unpack( + invalid_rr_domain_name = dns.DNSMessage.unpack( b"V\x1a\x81\x80\x00\x01\x00\x01\x00\x01\x00\x01\x05alive\x06github\x03com\x00\x00" + b"\x10\x00\x01\xc0\x0c\x00\x05\x00\x01\x00\x00\x0b\xc6\x00\x07\x99live\xc0\x12\xc0\x12\x00\x06\x00\x01" + b"\x00\x00\x03\x84\x00H\x07ns-1707\tawsdns-21\x02co\x02uk\x00\x11awsdns-hostmaster\x06amazon\xc0\x19\x00" @@ -212,7 +272,7 @@ class TestMessage: assert ( invalid_rr_domain_name.answers[0].data == b"\x99live\x06github\x03com\x00" ) - valid_compressed_rr_data = dns.Message.unpack( + valid_compressed_rr_data = dns.DNSMessage.unpack( b"\x10}\x81\x80\x00\x01\x00\x01\x00\x00\x00\x01\x06google\x03com\x00\x00\x06\x00\x01\xc0\x0c\x00\x06\x00" + b"\x01\x00\x00\x00\x0c\x00&\x03ns1\xc0\x0c\tdns-admin\xc0\x0c&~gw\x00\x00\x03\x84\x00\x00\x03\x84\x00" + b"\x00\x07\x08\x00\x00\x00<\x00\x00)\x02\x00\x00\x00\x00\x00\x00\x00" @@ -222,7 +282,7 @@ class TestMessage: == b"\x03ns1\x06google\x03com\x00\tdns-admin\x06google\x03com\x00&~gw\x00\x00\x03\x84\x00\x00\x03\x84\x00" + b"\x00\x07\x08\x00\x00\x00<" ) - A_record_data_contains_pointer_label = dns.Message.unpack( + A_record_data_contains_pointer_label = dns.DNSMessage.unpack( b"\x98A\x81\x80\x00\x01\x00\x01\x00\x00\x00\x01\x06google\x03com\x00\x00\x01\x00\x01\xc0\x0c\x00\x01\x00" + b"\x01\x00\x00\x00/\x00\x04\xd8:\xc4\xae\x00\x00)\x02\x00\x00\x00\x00\x00\x00\x00" ) @@ -235,13 +295,13 @@ class TestMessage: "recursion_available", ): setattr(req, flag, True) - assert getattr(dns.Message.unpack(req.packed), flag) is True + assert getattr(dns.DNSMessage.unpack(req.packed), flag) is True setattr(req, flag, False) - assert getattr(dns.Message.unpack(req.packed), flag) is False + assert getattr(dns.DNSMessage.unpack(req.packed), flag) is False def test_copy(self): msg = tutils.tdnsresp() - assert dns.Message.from_state(msg.get_state()) == msg + assert dns.DNSMessage.from_state(msg.get_state()) == msg copy = msg.copy() assert copy is not msg assert copy != msg diff --git a/test/mitmproxy/utils/test_strutils.py b/test/mitmproxy/utils/test_strutils.py index 5f76cd4a3..97e4181a5 100644 --- a/test/mitmproxy/utils/test_strutils.py +++ b/test/mitmproxy/utils/test_strutils.py @@ -55,6 +55,7 @@ def test_bytes_to_escaped_str(): assert strutils.bytes_to_escaped_str(b"\r\n\t") == "\\r\\n\\t" assert strutils.bytes_to_escaped_str(b"\r\n\t", True) == "\r\n\t" + assert strutils.bytes_to_escaped_str(b"\n", False) == r"\n" assert strutils.bytes_to_escaped_str(b"\n", True) == "\n" assert strutils.bytes_to_escaped_str(b"\\n", True) == "\\ \\ n".replace(" ", "") assert strutils.bytes_to_escaped_str(b"\\\n", True) == "\\ \\ \n".replace(" ", "") diff --git a/uv.lock b/uv.lock index 8f47282c8..4dff18553 100644 --- a/uv.lock +++ b/uv.lock @@ -796,7 +796,7 @@ requires-dist = [ { name = "pyopenssl", specifier = ">=22.1,<=25.0.0" }, { name = "pyparsing", specifier = ">=2.4.2,<=3.2.3" }, { name = "pyperclip", specifier = "<=1.9.0,>=1.9.0" }, - { name = "ruamel-yaml", specifier = ">=0.16,<=0.18.10" }, + { name = "ruamel-yaml", specifier = "<=0.18.10,>=0.18.10" }, { name = "sortedcontainers", specifier = ">=2.3,<=2.4.0" }, { name = "tornado", specifier = ">=6.4.1,<=6.4.2" }, { name = "typing-extensions", marker = "python_full_version < '3.13'", specifier = ">=4.13.2,<=4.14" }, diff --git a/web/src/js/components/FlowView/DnsMessages.tsx b/web/src/js/components/FlowView/DnsMessages.tsx index c3fc74e7a..a8ba6f979 100644 --- a/web/src/js/components/FlowView/DnsMessages.tsx +++ b/web/src/js/components/FlowView/DnsMessages.tsx @@ -63,7 +63,9 @@ const ResourceRecords: React.FC<{ {rr.type} {rr.class} {rr.ttl} - {rr.data} + + {JSON.stringify(rr.data).replace(/^"|"$/g, "")} + ))} diff --git a/web/src/js/flow.ts b/web/src/js/flow.ts index e1a5e6197..6f51002d6 100644 --- a/web/src/js/flow.ts +++ b/web/src/js/flow.ts @@ -132,7 +132,7 @@ export interface DNSResourceRecord { type: string; class: string; ttl: number; - data: string; + data: string | { [key: string | number]: string }; } export interface DNSMessage { @@ -150,7 +150,7 @@ export interface DNSMessage { authorities: DNSResourceRecord[]; additionals: DNSResourceRecord[]; size: number; - timestamp: number; + timestamp?: number; } export interface DNSFlow extends _Flow {