Support all query types in DNS mode (#6975)

* Use mitmproxy_rust's getaddrinfo to resolve domain names

* Use mitmproxy.DnsResolver for A/AAAA queries and forward other queries to dns server

* [autofix.ci] apply automated fixes

* Add suggested changes

* [autofix.ci] apply automated fixes

* nits

* lazy-load resolver

* fix lookup of non A/AAAA records

* bump required mitmproxy_rs version

* [autofix.ci] apply automated fixes

* Add tests

* Update CHANGELOG

* [autofix.ci] apply automated fixes

* Fix tests

* [autofix.ci] apply automated fixes

* Fixup

* Fixup

* [autofix.ci] apply automated fixes

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Maximilian Hils <git@maximilianhils.com>
This commit is contained in:
Gaurav Jain 2024-07-16 21:01:11 +05:30 committed by GitHub
parent 9512e99819
commit 317f5b9dce
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 198 additions and 249 deletions

View file

@ -31,6 +31,8 @@
([#7001](https://github.com/mitmproxy/mitmproxy/pull/7001), @errorxyz)
* Add Host header to CONNECT requests.
([#7021](https://github.com/mitmproxy/mitmproxy/pull/7021), @petsneakers)
* Support all query types in DNS mode
([#6975](https://github.com/mitmproxy/mitmproxy/pull/6975), @errorxyz)
## 12 June 2024: mitmproxy 10.3.1

View file

@ -1,15 +1,15 @@
import asyncio
import ipaddress
import socket
from collections.abc import Callable
from collections.abc import Iterable
from collections.abc import Sequence
from functools import cached_property
import mitmproxy_rs
from mitmproxy import ctx
from mitmproxy import dns
from mitmproxy.proxy import mode_specs
IP4_PTR_SUFFIX = ".in-addr.arpa"
IP6_PTR_SUFFIX = ".ip6.arpa"
class ResolveError(Exception):
"""Exception thrown by different resolve methods."""
@ -19,129 +19,40 @@ class ResolveError(Exception):
self.response_code = response_code
async def resolve_question_by_name(
question: dns.Question,
loop: asyncio.AbstractEventLoop,
family: socket.AddressFamily,
ip: Callable[[str], ipaddress.IPv4Address | ipaddress.IPv6Address],
) -> Iterable[dns.ResourceRecord]:
try:
addrinfos = await loop.getaddrinfo(
host=question.name, port=0, family=family, type=socket.SOCK_STREAM
)
except socket.gaierror as e:
if e.errno == socket.EAI_NONAME:
raise ResolveError(dns.response_codes.NXDOMAIN)
else:
# NOTE might fail on Windows for IPv6 queries:
# https://stackoverflow.com/questions/66755681/getaddrinfo-c-on-windows-not-handling-ipv6-correctly-returning-error-code-1
raise ResolveError(dns.response_codes.SERVFAIL) # pragma: no cover
return map(
lambda addrinfo: dns.ResourceRecord(
name=question.name,
type=question.type,
class_=question.class_,
ttl=dns.ResourceRecord.DEFAULT_TTL,
data=ip(addrinfo[4][0]).packed,
),
addrinfos,
)
async def resolve_question_by_addr(
question: dns.Question,
loop: asyncio.AbstractEventLoop,
suffix: str,
sockaddr: Callable[[list[str]], tuple[str, int] | tuple[str, int, int, int]],
) -> Iterable[dns.ResourceRecord]:
try:
addr = sockaddr(question.name[: -len(suffix)].split(".")[::-1])
except ValueError:
raise ResolveError(dns.response_codes.FORMERR)
try:
name, _ = await loop.getnameinfo(addr, flags=socket.NI_NAMEREQD)
except socket.gaierror as e:
raise ResolveError(
dns.response_codes.NXDOMAIN
if e.errno == socket.EAI_NONAME
else dns.response_codes.SERVFAIL
)
return [
dns.ResourceRecord(
name=question.name,
type=question.type,
class_=question.class_,
ttl=dns.ResourceRecord.DEFAULT_TTL,
data=dns.domain_names.pack(name),
)
]
async def resolve_question(
question: dns.Question, loop: asyncio.AbstractEventLoop
) -> Iterable[dns.ResourceRecord]:
"""Resolve the question into resource record(s), throwing ResolveError if an error condition occurs."""
if question.class_ != dns.classes.IN:
raise ResolveError(dns.response_codes.NOTIMP)
if question.type == dns.types.A:
return await resolve_question_by_name(
question, loop, socket.AddressFamily.AF_INET, ipaddress.IPv4Address
)
elif question.type == dns.types.AAAA:
return await resolve_question_by_name(
question, loop, socket.AddressFamily.AF_INET6, ipaddress.IPv6Address
)
elif question.type == dns.types.PTR:
name_lower = question.name.lower()
if name_lower.endswith(IP4_PTR_SUFFIX):
return await resolve_question_by_addr(
question=question,
loop=loop,
suffix=IP4_PTR_SUFFIX,
sockaddr=lambda x: (str(ipaddress.IPv4Address(".".join(x))), 0),
)
elif name_lower.endswith(IP6_PTR_SUFFIX):
return await resolve_question_by_addr(
question=question,
loop=loop,
suffix=IP6_PTR_SUFFIX,
sockaddr=lambda x: (
str(ipaddress.IPv6Address(bytes.fromhex("".join(x)))),
0,
0,
0,
),
)
else:
raise ResolveError(dns.response_codes.FORMERR)
else:
raise ResolveError(dns.response_codes.NOTIMP)
async def resolve_message(
message: dns.Message, loop: asyncio.AbstractEventLoop
) -> dns.Message:
try:
if not message.query:
raise ResolveError(
dns.response_codes.REFUSED
) # we cannot resolve an answer
if message.op_code != dns.op_codes.QUERY:
raise ResolveError(
dns.response_codes.NOTIMP
) # inverse queries and others are not supported
rrs: list[dns.ResourceRecord] = []
for question in message.questions:
rrs.extend(await resolve_question(question, loop))
except ResolveError as e:
return message.fail(e.response_code)
else:
return message.succeed(rrs)
class DnsResolver:
def load(self, loader):
loader.add_option(
"dns_use_hosts_file",
bool,
True,
"Use the hosts file for DNS lookups in regular DNS mode/wireguard mode.",
)
loader.add_option(
"dns_name_servers",
Sequence[str],
[],
"Name servers to use for lookups. Default: operating system's name servers",
)
def configure(self, updated):
if "dns_use_hosts_file" in updated or "dns_name_servers" in updated:
self.__dict__.pop("resolver", None)
self.__dict__.pop("name_servers", None)
@cached_property
def resolver(self) -> mitmproxy_rs.DnsResolver:
return mitmproxy_rs.DnsResolver(
name_servers=self.name_servers,
use_hosts_file=ctx.options.dns_use_hosts_file,
)
@cached_property
def name_servers(self) -> list[str]:
return ctx.options.dns_name_servers or mitmproxy_rs.get_system_dns_servers()
async def dns_request(self, flow: dns.DNSFlow) -> None:
assert flow.request
should_resolve = (
(
isinstance(flow.client_conn.proxy_mode, mode_specs.DnsMode)
@ -155,7 +66,60 @@ class DnsResolver:
and not flow.error
)
if should_resolve:
# TODO: We need to handle overly long responses here.
flow.response = await resolve_message(
flow.request, asyncio.get_running_loop()
all_ip_lookups = (
flow.request.query
and flow.request.op_code == dns.op_codes.QUERY
and all(
q.type in (dns.types.A, dns.types.AAAA)
and q.class_ == dns.classes.IN
for q in flow.request.questions
)
)
# We use `mitmproxy_rs.DnsResolver` if we need to use the hosts file to lookup hostnames(A/AAAA queries only)
# For other cases we forward it to the specified name server directly.
if all_ip_lookups and ctx.options.dns_use_hosts_file:
# TODO: We need to handle overly long responses here.
flow.response = await self.resolve_message(flow.request)
elif not flow.server_conn.address:
flow.server_conn.address = (self.name_servers[0], 53)
async def resolve_message(self, message: dns.Message) -> dns.Message:
try:
rrs: list[dns.ResourceRecord] = []
for question in message.questions:
rrs.extend(await self.resolve_question(question))
except ResolveError as e:
return message.fail(e.response_code)
else:
return message.succeed(rrs)
async def resolve_question(
self, question: dns.Question
) -> Iterable[dns.ResourceRecord]:
assert question.type in (dns.types.A, dns.types.AAAA)
try:
if question.type == dns.types.A:
addrinfos = await self.resolver.lookup_ipv4(question.name)
elif question.type == dns.types.AAAA:
addrinfos = await self.resolver.lookup_ipv6(question.name)
except socket.gaierror as e:
# We aren't exactly following the RFC here
# https://datatracker.ietf.org/doc/html/rfc2308#section-2
if e.args[0] == "NXDOMAIN":
raise ResolveError(dns.response_codes.NXDOMAIN)
elif e.args[0] == "NOERROR":
addrinfos = []
else: # pragma: no cover
raise ResolveError(dns.response_codes.SERVFAIL)
return map(
lambda addrinfo: dns.ResourceRecord(
name=question.name,
type=question.type,
class_=question.class_,
ttl=dns.ResourceRecord.DEFAULT_TTL,
data=ipaddress.ip_address(addrinfo).packed,
),
addrinfos,
)

View file

@ -42,7 +42,7 @@ dependencies = [
"hyperframe>=6.0,<=6.0.1",
"kaitaistruct>=0.10,<=0.10",
"ldap3>=2.8,<=2.9.1",
"mitmproxy_rs>=0.6.0,<0.7", # relaxed upper bound here: we control this
"mitmproxy_rs>=0.6.1,<0.7", # relaxed upper bound here: we control this
"msgpack>=1.0.0,<=1.0.8",
"passlib>=1.6.5,<=1.7.4",
"protobuf<=5.27.2,>=5.27.2",

View file

@ -1,25 +1,17 @@
import asyncio
import ipaddress
import socket
from collections.abc import Callable
import pytest
import mitmproxy_rs
from mitmproxy import dns
from mitmproxy.addons import dns_resolver
from mitmproxy.addons import proxyserver
from mitmproxy.connection import Address
from mitmproxy.proxy.mode_specs import ProxyMode
from mitmproxy.test import taddons
from mitmproxy.test import tflow
from mitmproxy.test import tutils
async def test_simple(monkeypatch):
monkeypatch.setattr(
dns_resolver, "resolve_message", lambda _, __: asyncio.sleep(0, "resp")
)
async def test_ignores_reverse_mode():
dr = dns_resolver.DnsResolver()
with taddons.context(dr, proxyserver.Proxyserver()):
f = tflow.tdnsflow()
@ -32,107 +24,99 @@ async def test_simple(monkeypatch):
assert not f.response
class DummyLoop:
async def getnameinfo(self, socketaddr: Address, flags: int = 0):
assert flags == socket.NI_NAMEREQD
if socketaddr[0] in ("8.8.8.8", "2001:4860:4860::8888"):
return ("dns.google", "")
e = socket.gaierror()
e.errno = socket.EAI_NONAME
raise e
async def test_resolver():
dr = dns_resolver.DnsResolver()
with taddons.context(dr) as tctx:
assert dr.name_servers == mitmproxy_rs.get_system_dns_servers()
async def getaddrinfo(self, host: str, port: int, *, family: int, type: int):
e = socket.gaierror()
e.errno = socket.EAI_NONAME
if family == socket.AF_INET:
if host == "dns.google":
return [(socket.AF_INET, type, None, None, ("8.8.8.8", port))]
elif family == socket.AF_INET6:
if host == "dns.google":
return [
(
socket.AF_INET6,
type,
None,
None,
("2001:4860:4860::8888", port, None, None),
)
]
else:
e.errno = socket.EAI_FAMILY
raise e
tctx.options.dns_name_servers = ["1.1.1.1"]
assert dr.name_servers == ["1.1.1.1"]
res_old = dr.resolver
tctx.options.dns_use_hosts_file = False
assert dr.resolver != res_old
tctx.options.dns_name_servers = ["8.8.8.8"]
assert dr.name_servers == ["8.8.8.8"]
async def test_resolve():
async def fail_with(question: dns.Question, code: int):
with pytest.raises(dns_resolver.ResolveError) as ex:
await dns_resolver.resolve_question(question, DummyLoop())
assert ex.value.response_code == code
async def lookup_ipv4(name: str):
if name == "not.exists":
raise socket.gaierror("NXDOMAIN")
elif name == "no.records":
raise socket.gaierror("NOERROR")
return ["8.8.8.8"]
async def succeed_with(
question: dns.Question, check: Callable[[dns.ResourceRecord], bool]
):
assert any(
map(check, await dns_resolver.resolve_question(question, DummyLoop()))
async def test_dns_request(monkeypatch):
monkeypatch.setattr(
mitmproxy_rs.DnsResolver, "lookup_ipv4", lambda _, name: lookup_ipv4(name)
)
resolver = dns_resolver.DnsResolver()
with taddons.context(resolver) as tctx:
async def process_questions(questions):
req = tutils.tdnsreq(questions=questions)
flow = tflow.tdnsflow(req=req)
flow.server_conn.address = None
await resolver.dns_request(flow)
return flow
req = tutils.tdnsreq()
req.op_code = dns.op_codes.IQUERY
flow = tflow.tdnsflow(req=req)
flow.server_conn.address = None
await resolver.dns_request(flow)
assert flow.server_conn.address[0] == resolver.name_servers[0]
req.query = False
req.op_code = dns.op_codes.QUERY
flow = tflow.tdnsflow(req=req)
flow.server_conn.address = None
await resolver.dns_request(flow)
assert flow.server_conn.address[0] == resolver.name_servers[0]
flow = await process_questions(
[
dns.Question("dns.google", dns.types.AAAA, dns.classes.IN),
dns.Question("dns.google", dns.types.NS, dns.classes.IN),
]
)
assert flow.server_conn.address[0] == resolver.name_servers[0]
await fail_with(
dns.Question("dns.google", dns.types.A, dns.classes.CH),
dns.response_codes.NOTIMP,
)
await fail_with(
dns.Question("not.exists", dns.types.A, dns.classes.IN),
dns.response_codes.NXDOMAIN,
)
await fail_with(
dns.Question("dns.google", dns.types.SOA, dns.classes.IN),
dns.response_codes.NOTIMP,
)
await fail_with(
dns.Question("totally.invalid", dns.types.PTR, dns.classes.IN),
dns.response_codes.FORMERR,
)
await fail_with(
dns.Question("invalid.in-addr.arpa", dns.types.PTR, dns.classes.IN),
dns.response_codes.FORMERR,
)
await fail_with(
dns.Question("0.0.0.1.in-addr.arpa", dns.types.PTR, dns.classes.IN),
dns.response_codes.NXDOMAIN,
)
flow = await process_questions(
[
dns.Question("dns.google", dns.types.AAAA, dns.classes.IN),
dns.Question("dns.google", dns.types.A, dns.classes.IN),
]
)
assert flow.server_conn.address is None
assert flow.response
await succeed_with(
dns.Question("dns.google", dns.types.A, dns.classes.IN),
lambda rr: rr.ipv4_address == ipaddress.IPv4Address("8.8.8.8"),
)
await succeed_with(
dns.Question("dns.google", dns.types.AAAA, dns.classes.IN),
lambda rr: rr.ipv6_address == ipaddress.IPv6Address("2001:4860:4860::8888"),
)
await succeed_with(
dns.Question("8.8.8.8.in-addr.arpa", dns.types.PTR, dns.classes.IN),
lambda rr: rr.domain_name == "dns.google",
)
await succeed_with(
dns.Question(
"8.8.8.8.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.6.8.4.0.6.8.4.1.0.0.2.ip6.arpa",
dns.types.PTR,
dns.classes.IN,
),
lambda rr: rr.domain_name == "dns.google",
)
flow = tflow.tdnsflow()
await resolver.dns_request(flow)
assert flow.server_conn.address == ("address", 22)
req = tutils.tdnsreq()
req.query = False
assert (
await dns_resolver.resolve_message(req, DummyLoop())
).response_code == dns.response_codes.REFUSED
req.query = True
req.op_code = dns.op_codes.IQUERY
assert (
await dns_resolver.resolve_message(req, DummyLoop())
).response_code == dns.response_codes.NOTIMP
req.op_code = dns.op_codes.QUERY
resp = await dns_resolver.resolve_message(req, DummyLoop())
assert resp.response_code == dns.response_codes.NOERROR
assert filter(lambda rr: str(rr.ipv4_address) == "8.8.8.8", resp.answers)
flow = await process_questions(
[
dns.Question("not.exists", dns.types.A, dns.classes.IN),
]
)
assert flow.response.response_code == dns.response_codes.NXDOMAIN
flow = await process_questions(
[
dns.Question("no.records", dns.types.A, dns.classes.IN),
]
)
assert flow.response.response_code == dns.response_codes.NOERROR
assert not flow.response.answers
tctx.options.dns_use_hosts_file = False
flow = await process_questions(
[
dns.Question("dns.google", dns.types.A, dns.classes.IN),
]
)
assert flow.server_conn.address[0] == resolver.name_servers[0]

View file

@ -1,7 +1,6 @@
from __future__ import annotations
import asyncio
import socket
import ssl
from collections.abc import AsyncGenerator
from collections.abc import Callable
@ -266,22 +265,18 @@ async def test_shutdown_err(caplog_async) -> None:
await _wait_for_connection_closes(ps)
class DummyResolver:
async def dns_request(self, flow: dns.DNSFlow) -> None:
flow.response = await dns_resolver.resolve_message(flow.request, self)
async def getaddrinfo(self, host: str, port: int, *, family: int, type: int):
if family == socket.AF_INET and host == "dns.google":
return [(socket.AF_INET, type, None, None, ("8.8.8.8", port))]
e = socket.gaierror()
e.errno = socket.EAI_NONAME
raise e
async def lookup_ipv4():
return await asyncio.sleep(0, ["8.8.8.8"])
async def test_dns(caplog_async) -> None:
async def test_dns(caplog_async, monkeypatch) -> None:
monkeypatch.setattr(
mitmproxy_rs.DnsResolver, "lookup_ipv4", lambda _, __: lookup_ipv4()
)
caplog_async.set_level("INFO")
ps = Proxyserver()
with taddons.context(ps, DummyResolver()) as tctx:
with taddons.context(ps, dns_resolver.DnsResolver()) as tctx:
tctx.configure(
ps,
mode=["dns@127.0.0.1:0"],

View file

@ -21,6 +21,8 @@ export interface OptionsState {
connection_strategy: string;
console_focus_follow: boolean;
content_view_lines_cutoff: number;
dns_name_servers: string[];
dns_use_hosts_file: boolean;
export_preserve_original_ip: boolean;
hardump: string;
http2: boolean;
@ -119,6 +121,8 @@ export const defaultState: OptionsState = {
connection_strategy: "eager",
console_focus_follow: false,
content_view_lines_cutoff: 512,
dns_name_servers: [],
dns_use_hosts_file: true,
export_preserve_original_ip: false,
hardump: "",
http2: true,