From 317f5b9dced719ceaef9d65ff427e64a98dfeffc Mon Sep 17 00:00:00 2001 From: Gaurav Jain <64748057+errorxyz@users.noreply.github.com> Date: Tue, 16 Jul 2024 21:01:11 +0530 Subject: [PATCH] Support all query types in DNS mode (#6975) * Use mitmproxy_rust's getaddrinfo to resolve domain names * Use mitmproxy.DnsResolver for A/AAAA queries and forward other queries to dns server * [autofix.ci] apply automated fixes * Add suggested changes * [autofix.ci] apply automated fixes * nits * lazy-load resolver * fix lookup of non A/AAAA records * bump required mitmproxy_rs version * [autofix.ci] apply automated fixes * Add tests * Update CHANGELOG * [autofix.ci] apply automated fixes * Fix tests * [autofix.ci] apply automated fixes * Fixup * Fixup * [autofix.ci] apply automated fixes --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Maximilian Hils --- CHANGELOG.md | 2 + mitmproxy/addons/dns_resolver.py | 222 +++++++++------------ pyproject.toml | 2 +- test/mitmproxy/addons/test_dns_resolver.py | 196 +++++++++--------- test/mitmproxy/addons/test_proxyserver.py | 21 +- web/src/js/ducks/_options_gen.ts | 4 + 6 files changed, 198 insertions(+), 249 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77dd9f37d..a4de351b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,8 @@ ([#7001](https://github.com/mitmproxy/mitmproxy/pull/7001), @errorxyz) * Add Host header to CONNECT requests. ([#7021](https://github.com/mitmproxy/mitmproxy/pull/7021), @petsneakers) +* Support all query types in DNS mode + ([#6975](https://github.com/mitmproxy/mitmproxy/pull/6975), @errorxyz) ## 12 June 2024: mitmproxy 10.3.1 diff --git a/mitmproxy/addons/dns_resolver.py b/mitmproxy/addons/dns_resolver.py index e46ee52b0..b362640c5 100644 --- a/mitmproxy/addons/dns_resolver.py +++ b/mitmproxy/addons/dns_resolver.py @@ -1,15 +1,15 @@ -import asyncio import ipaddress import socket -from collections.abc import Callable from collections.abc import Iterable +from collections.abc import Sequence +from functools import cached_property +import mitmproxy_rs + +from mitmproxy import ctx from mitmproxy import dns from mitmproxy.proxy import mode_specs -IP4_PTR_SUFFIX = ".in-addr.arpa" -IP6_PTR_SUFFIX = ".ip6.arpa" - class ResolveError(Exception): """Exception thrown by different resolve methods.""" @@ -19,129 +19,40 @@ class ResolveError(Exception): self.response_code = response_code -async def resolve_question_by_name( - question: dns.Question, - loop: asyncio.AbstractEventLoop, - family: socket.AddressFamily, - ip: Callable[[str], ipaddress.IPv4Address | ipaddress.IPv6Address], -) -> Iterable[dns.ResourceRecord]: - try: - addrinfos = await loop.getaddrinfo( - host=question.name, port=0, family=family, type=socket.SOCK_STREAM - ) - except socket.gaierror as e: - if e.errno == socket.EAI_NONAME: - raise ResolveError(dns.response_codes.NXDOMAIN) - else: - # NOTE might fail on Windows for IPv6 queries: - # https://stackoverflow.com/questions/66755681/getaddrinfo-c-on-windows-not-handling-ipv6-correctly-returning-error-code-1 - raise ResolveError(dns.response_codes.SERVFAIL) # pragma: no cover - return map( - lambda addrinfo: dns.ResourceRecord( - name=question.name, - type=question.type, - class_=question.class_, - ttl=dns.ResourceRecord.DEFAULT_TTL, - data=ip(addrinfo[4][0]).packed, - ), - addrinfos, - ) - - -async def resolve_question_by_addr( - question: dns.Question, - loop: asyncio.AbstractEventLoop, - suffix: str, - sockaddr: Callable[[list[str]], tuple[str, int] | tuple[str, int, int, int]], -) -> Iterable[dns.ResourceRecord]: - try: - addr = sockaddr(question.name[: -len(suffix)].split(".")[::-1]) - except ValueError: - raise ResolveError(dns.response_codes.FORMERR) - try: - name, _ = await loop.getnameinfo(addr, flags=socket.NI_NAMEREQD) - except socket.gaierror as e: - raise ResolveError( - dns.response_codes.NXDOMAIN - if e.errno == socket.EAI_NONAME - else dns.response_codes.SERVFAIL - ) - return [ - dns.ResourceRecord( - name=question.name, - type=question.type, - class_=question.class_, - ttl=dns.ResourceRecord.DEFAULT_TTL, - data=dns.domain_names.pack(name), - ) - ] - - -async def resolve_question( - question: dns.Question, loop: asyncio.AbstractEventLoop -) -> Iterable[dns.ResourceRecord]: - """Resolve the question into resource record(s), throwing ResolveError if an error condition occurs.""" - - if question.class_ != dns.classes.IN: - raise ResolveError(dns.response_codes.NOTIMP) - if question.type == dns.types.A: - return await resolve_question_by_name( - question, loop, socket.AddressFamily.AF_INET, ipaddress.IPv4Address - ) - elif question.type == dns.types.AAAA: - return await resolve_question_by_name( - question, loop, socket.AddressFamily.AF_INET6, ipaddress.IPv6Address - ) - elif question.type == dns.types.PTR: - name_lower = question.name.lower() - if name_lower.endswith(IP4_PTR_SUFFIX): - return await resolve_question_by_addr( - question=question, - loop=loop, - suffix=IP4_PTR_SUFFIX, - sockaddr=lambda x: (str(ipaddress.IPv4Address(".".join(x))), 0), - ) - elif name_lower.endswith(IP6_PTR_SUFFIX): - return await resolve_question_by_addr( - question=question, - loop=loop, - suffix=IP6_PTR_SUFFIX, - sockaddr=lambda x: ( - str(ipaddress.IPv6Address(bytes.fromhex("".join(x)))), - 0, - 0, - 0, - ), - ) - else: - raise ResolveError(dns.response_codes.FORMERR) - else: - raise ResolveError(dns.response_codes.NOTIMP) - - -async def resolve_message( - message: dns.Message, loop: asyncio.AbstractEventLoop -) -> dns.Message: - try: - if not message.query: - raise ResolveError( - dns.response_codes.REFUSED - ) # we cannot resolve an answer - if message.op_code != dns.op_codes.QUERY: - raise ResolveError( - dns.response_codes.NOTIMP - ) # inverse queries and others are not supported - rrs: list[dns.ResourceRecord] = [] - for question in message.questions: - rrs.extend(await resolve_question(question, loop)) - except ResolveError as e: - return message.fail(e.response_code) - else: - return message.succeed(rrs) - - class DnsResolver: + def load(self, loader): + loader.add_option( + "dns_use_hosts_file", + bool, + True, + "Use the hosts file for DNS lookups in regular DNS mode/wireguard mode.", + ) + + loader.add_option( + "dns_name_servers", + Sequence[str], + [], + "Name servers to use for lookups. Default: operating system's name servers", + ) + + def configure(self, updated): + if "dns_use_hosts_file" in updated or "dns_name_servers" in updated: + self.__dict__.pop("resolver", None) + self.__dict__.pop("name_servers", None) + + @cached_property + def resolver(self) -> mitmproxy_rs.DnsResolver: + return mitmproxy_rs.DnsResolver( + name_servers=self.name_servers, + use_hosts_file=ctx.options.dns_use_hosts_file, + ) + + @cached_property + def name_servers(self) -> list[str]: + return ctx.options.dns_name_servers or mitmproxy_rs.get_system_dns_servers() + async def dns_request(self, flow: dns.DNSFlow) -> None: + assert flow.request should_resolve = ( ( isinstance(flow.client_conn.proxy_mode, mode_specs.DnsMode) @@ -155,7 +66,60 @@ class DnsResolver: and not flow.error ) if should_resolve: - # TODO: We need to handle overly long responses here. - flow.response = await resolve_message( - flow.request, asyncio.get_running_loop() + all_ip_lookups = ( + flow.request.query + and flow.request.op_code == dns.op_codes.QUERY + and all( + q.type in (dns.types.A, dns.types.AAAA) + and q.class_ == dns.classes.IN + for q in flow.request.questions + ) ) + # We use `mitmproxy_rs.DnsResolver` if we need to use the hosts file to lookup hostnames(A/AAAA queries only) + # For other cases we forward it to the specified name server directly. + if all_ip_lookups and ctx.options.dns_use_hosts_file: + # TODO: We need to handle overly long responses here. + flow.response = await self.resolve_message(flow.request) + elif not flow.server_conn.address: + flow.server_conn.address = (self.name_servers[0], 53) + + async def resolve_message(self, message: dns.Message) -> dns.Message: + try: + rrs: list[dns.ResourceRecord] = [] + for question in message.questions: + rrs.extend(await self.resolve_question(question)) + except ResolveError as e: + return message.fail(e.response_code) + else: + return message.succeed(rrs) + + async def resolve_question( + self, question: dns.Question + ) -> Iterable[dns.ResourceRecord]: + assert question.type in (dns.types.A, dns.types.AAAA) + + try: + if question.type == dns.types.A: + addrinfos = await self.resolver.lookup_ipv4(question.name) + elif question.type == dns.types.AAAA: + addrinfos = await self.resolver.lookup_ipv6(question.name) + except socket.gaierror as e: + # We aren't exactly following the RFC here + # https://datatracker.ietf.org/doc/html/rfc2308#section-2 + if e.args[0] == "NXDOMAIN": + raise ResolveError(dns.response_codes.NXDOMAIN) + elif e.args[0] == "NOERROR": + addrinfos = [] + else: # pragma: no cover + raise ResolveError(dns.response_codes.SERVFAIL) + + return map( + lambda addrinfo: dns.ResourceRecord( + name=question.name, + type=question.type, + class_=question.class_, + ttl=dns.ResourceRecord.DEFAULT_TTL, + data=ipaddress.ip_address(addrinfo).packed, + ), + addrinfos, + ) diff --git a/pyproject.toml b/pyproject.toml index e5e59ea17..fbc5de6ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dependencies = [ "hyperframe>=6.0,<=6.0.1", "kaitaistruct>=0.10,<=0.10", "ldap3>=2.8,<=2.9.1", - "mitmproxy_rs>=0.6.0,<0.7", # relaxed upper bound here: we control this + "mitmproxy_rs>=0.6.1,<0.7", # relaxed upper bound here: we control this "msgpack>=1.0.0,<=1.0.8", "passlib>=1.6.5,<=1.7.4", "protobuf<=5.27.2,>=5.27.2", diff --git a/test/mitmproxy/addons/test_dns_resolver.py b/test/mitmproxy/addons/test_dns_resolver.py index a998ef264..97cc9d727 100644 --- a/test/mitmproxy/addons/test_dns_resolver.py +++ b/test/mitmproxy/addons/test_dns_resolver.py @@ -1,25 +1,17 @@ -import asyncio -import ipaddress import socket -from collections.abc import Callable -import pytest +import mitmproxy_rs from mitmproxy import dns from mitmproxy.addons import dns_resolver from mitmproxy.addons import proxyserver -from mitmproxy.connection import Address from mitmproxy.proxy.mode_specs import ProxyMode from mitmproxy.test import taddons from mitmproxy.test import tflow from mitmproxy.test import tutils -async def test_simple(monkeypatch): - monkeypatch.setattr( - dns_resolver, "resolve_message", lambda _, __: asyncio.sleep(0, "resp") - ) - +async def test_ignores_reverse_mode(): dr = dns_resolver.DnsResolver() with taddons.context(dr, proxyserver.Proxyserver()): f = tflow.tdnsflow() @@ -32,107 +24,99 @@ async def test_simple(monkeypatch): assert not f.response -class DummyLoop: - async def getnameinfo(self, socketaddr: Address, flags: int = 0): - assert flags == socket.NI_NAMEREQD - if socketaddr[0] in ("8.8.8.8", "2001:4860:4860::8888"): - return ("dns.google", "") - e = socket.gaierror() - e.errno = socket.EAI_NONAME - raise e +async def test_resolver(): + dr = dns_resolver.DnsResolver() + with taddons.context(dr) as tctx: + assert dr.name_servers == mitmproxy_rs.get_system_dns_servers() - async def getaddrinfo(self, host: str, port: int, *, family: int, type: int): - e = socket.gaierror() - e.errno = socket.EAI_NONAME - if family == socket.AF_INET: - if host == "dns.google": - return [(socket.AF_INET, type, None, None, ("8.8.8.8", port))] - elif family == socket.AF_INET6: - if host == "dns.google": - return [ - ( - socket.AF_INET6, - type, - None, - None, - ("2001:4860:4860::8888", port, None, None), - ) - ] - else: - e.errno = socket.EAI_FAMILY - raise e + tctx.options.dns_name_servers = ["1.1.1.1"] + assert dr.name_servers == ["1.1.1.1"] + + res_old = dr.resolver + tctx.options.dns_use_hosts_file = False + assert dr.resolver != res_old + + tctx.options.dns_name_servers = ["8.8.8.8"] + assert dr.name_servers == ["8.8.8.8"] -async def test_resolve(): - async def fail_with(question: dns.Question, code: int): - with pytest.raises(dns_resolver.ResolveError) as ex: - await dns_resolver.resolve_question(question, DummyLoop()) - assert ex.value.response_code == code +async def lookup_ipv4(name: str): + if name == "not.exists": + raise socket.gaierror("NXDOMAIN") + elif name == "no.records": + raise socket.gaierror("NOERROR") + return ["8.8.8.8"] - async def succeed_with( - question: dns.Question, check: Callable[[dns.ResourceRecord], bool] - ): - assert any( - map(check, await dns_resolver.resolve_question(question, DummyLoop())) + +async def test_dns_request(monkeypatch): + monkeypatch.setattr( + mitmproxy_rs.DnsResolver, "lookup_ipv4", lambda _, name: lookup_ipv4(name) + ) + + resolver = dns_resolver.DnsResolver() + with taddons.context(resolver) as tctx: + + async def process_questions(questions): + req = tutils.tdnsreq(questions=questions) + flow = tflow.tdnsflow(req=req) + flow.server_conn.address = None + await resolver.dns_request(flow) + return flow + + req = tutils.tdnsreq() + req.op_code = dns.op_codes.IQUERY + flow = tflow.tdnsflow(req=req) + flow.server_conn.address = None + await resolver.dns_request(flow) + assert flow.server_conn.address[0] == resolver.name_servers[0] + + req.query = False + req.op_code = dns.op_codes.QUERY + flow = tflow.tdnsflow(req=req) + flow.server_conn.address = None + await resolver.dns_request(flow) + assert flow.server_conn.address[0] == resolver.name_servers[0] + + flow = await process_questions( + [ + dns.Question("dns.google", dns.types.AAAA, dns.classes.IN), + dns.Question("dns.google", dns.types.NS, dns.classes.IN), + ] ) + assert flow.server_conn.address[0] == resolver.name_servers[0] - await fail_with( - dns.Question("dns.google", dns.types.A, dns.classes.CH), - dns.response_codes.NOTIMP, - ) - await fail_with( - dns.Question("not.exists", dns.types.A, dns.classes.IN), - dns.response_codes.NXDOMAIN, - ) - await fail_with( - dns.Question("dns.google", dns.types.SOA, dns.classes.IN), - dns.response_codes.NOTIMP, - ) - await fail_with( - dns.Question("totally.invalid", dns.types.PTR, dns.classes.IN), - dns.response_codes.FORMERR, - ) - await fail_with( - dns.Question("invalid.in-addr.arpa", dns.types.PTR, dns.classes.IN), - dns.response_codes.FORMERR, - ) - await fail_with( - dns.Question("0.0.0.1.in-addr.arpa", dns.types.PTR, dns.classes.IN), - dns.response_codes.NXDOMAIN, - ) + flow = await process_questions( + [ + dns.Question("dns.google", dns.types.AAAA, dns.classes.IN), + dns.Question("dns.google", dns.types.A, dns.classes.IN), + ] + ) + assert flow.server_conn.address is None + assert flow.response - await succeed_with( - dns.Question("dns.google", dns.types.A, dns.classes.IN), - lambda rr: rr.ipv4_address == ipaddress.IPv4Address("8.8.8.8"), - ) - await succeed_with( - dns.Question("dns.google", dns.types.AAAA, dns.classes.IN), - lambda rr: rr.ipv6_address == ipaddress.IPv6Address("2001:4860:4860::8888"), - ) - await succeed_with( - dns.Question("8.8.8.8.in-addr.arpa", dns.types.PTR, dns.classes.IN), - lambda rr: rr.domain_name == "dns.google", - ) - await succeed_with( - dns.Question( - "8.8.8.8.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.6.8.4.0.6.8.4.1.0.0.2.ip6.arpa", - dns.types.PTR, - dns.classes.IN, - ), - lambda rr: rr.domain_name == "dns.google", - ) + flow = tflow.tdnsflow() + await resolver.dns_request(flow) + assert flow.server_conn.address == ("address", 22) - req = tutils.tdnsreq() - req.query = False - assert ( - await dns_resolver.resolve_message(req, DummyLoop()) - ).response_code == dns.response_codes.REFUSED - req.query = True - req.op_code = dns.op_codes.IQUERY - assert ( - await dns_resolver.resolve_message(req, DummyLoop()) - ).response_code == dns.response_codes.NOTIMP - req.op_code = dns.op_codes.QUERY - resp = await dns_resolver.resolve_message(req, DummyLoop()) - assert resp.response_code == dns.response_codes.NOERROR - assert filter(lambda rr: str(rr.ipv4_address) == "8.8.8.8", resp.answers) + flow = await process_questions( + [ + dns.Question("not.exists", dns.types.A, dns.classes.IN), + ] + ) + assert flow.response.response_code == dns.response_codes.NXDOMAIN + + flow = await process_questions( + [ + dns.Question("no.records", dns.types.A, dns.classes.IN), + ] + ) + assert flow.response.response_code == dns.response_codes.NOERROR + assert not flow.response.answers + + tctx.options.dns_use_hosts_file = False + flow = await process_questions( + [ + dns.Question("dns.google", dns.types.A, dns.classes.IN), + ] + ) + assert flow.server_conn.address[0] == resolver.name_servers[0] diff --git a/test/mitmproxy/addons/test_proxyserver.py b/test/mitmproxy/addons/test_proxyserver.py index 0269dcd40..346167878 100644 --- a/test/mitmproxy/addons/test_proxyserver.py +++ b/test/mitmproxy/addons/test_proxyserver.py @@ -1,7 +1,6 @@ from __future__ import annotations import asyncio -import socket import ssl from collections.abc import AsyncGenerator from collections.abc import Callable @@ -266,22 +265,18 @@ async def test_shutdown_err(caplog_async) -> None: await _wait_for_connection_closes(ps) -class DummyResolver: - async def dns_request(self, flow: dns.DNSFlow) -> None: - flow.response = await dns_resolver.resolve_message(flow.request, self) - - async def getaddrinfo(self, host: str, port: int, *, family: int, type: int): - if family == socket.AF_INET and host == "dns.google": - return [(socket.AF_INET, type, None, None, ("8.8.8.8", port))] - e = socket.gaierror() - e.errno = socket.EAI_NONAME - raise e +async def lookup_ipv4(): + return await asyncio.sleep(0, ["8.8.8.8"]) -async def test_dns(caplog_async) -> None: +async def test_dns(caplog_async, monkeypatch) -> None: + monkeypatch.setattr( + mitmproxy_rs.DnsResolver, "lookup_ipv4", lambda _, __: lookup_ipv4() + ) + caplog_async.set_level("INFO") ps = Proxyserver() - with taddons.context(ps, DummyResolver()) as tctx: + with taddons.context(ps, dns_resolver.DnsResolver()) as tctx: tctx.configure( ps, mode=["dns@127.0.0.1:0"], diff --git a/web/src/js/ducks/_options_gen.ts b/web/src/js/ducks/_options_gen.ts index 10e39c957..19303c976 100644 --- a/web/src/js/ducks/_options_gen.ts +++ b/web/src/js/ducks/_options_gen.ts @@ -21,6 +21,8 @@ export interface OptionsState { connection_strategy: string; console_focus_follow: boolean; content_view_lines_cutoff: number; + dns_name_servers: string[]; + dns_use_hosts_file: boolean; export_preserve_original_ip: boolean; hardump: string; http2: boolean; @@ -119,6 +121,8 @@ export const defaultState: OptionsState = { connection_strategy: "eager", console_focus_follow: false, content_view_lines_cutoff: 512, + dns_name_servers: [], + dns_use_hosts_file: true, export_preserve_original_ip: false, hardump: "", http2: true,