Send http/1 request and response trailer headers

CHANGELOG

@@ -16,6 +16,7 @@ Unreleased: mitmproxy next
     * Fix file unlinking before external viewer finishes loading (@wchasekelley)
     * Add --cert-passphrase command line argument (@mirosyn)
     * Add interactive tutorials to the documentation (@mplattner)
+    * Add support for sending (but not parsing) HTTP Trailers to the HTTP/1 protocol (@bburky)
 
     * --- TODO: add new PRs above this line ---
 

docs/src/content/concepts-protocols.md

@@ -16,7 +16,7 @@ menu:
 HTTP/1.0 and HTTP/1.1 support in mitmproxy is based on our custom HTTP stack,
 which takes care of all semantics and on-the-wire parsing/serialization tasks.
 
-mitmproxy currently does not support HTTP trailers - but if you want to send
+mitmproxy currently does not parsing HTTP trailers - but if you want to send
 us a PR, we promise to take look!
 
 ## HTTP/2

examples/addons/http-trailers.py

@@ -15,14 +15,29 @@ def request(flow: http.HTTPFlow):
     if flow.request.trailers:
         print("HTTP Trailers detected! Request contains:", flow.request.trailers)
 
+    if flow.request.path == "/inject_trailers":
+        if not flow.request.is_http2:
+            # HTTP 1.0 requires transfer-encoding: chunked to send trailers
+            flow.request.headers["transfer-encoding"] = "chunked"
+
+        flow.request.headers["trailer"] = "x-my-injected-trailer-header"
+        flow.request.trailers = Headers([
+            (b"x-my-injected-trailer-header", b"foobar")
+        ])
+        print("Injected a new request trailer...", flow.request.headers["trailer"])
+
 
 def response(flow: http.HTTPFlow):
     if flow.response.trailers:
         print("HTTP Trailers detected! Response contains:", flow.response.trailers)
 
     if flow.request.path == "/inject_trailers":
+        if not flow.response.is_http2:
+            # HTTP 1.0 requires transfer-encoding: chunked to send trailers
+            flow.response.headers["transfer-encoding"] = "chunked"
+
         flow.response.headers["trailer"] = "x-my-injected-trailer-header"
         flow.response.trailers = Headers([
             (b"x-my-injected-trailer-header", b"foobar")
         ])
-        print("Injected a new trailer...", flow.response.headers["trailer"])
+        print("Injected a new response trailer...", flow.response.headers["trailer"])

mitmproxy/net/http/http1/assemble.py

@@ -5,7 +5,7 @@ def assemble_request(request):
     if request.data.content is None:
         raise exceptions.HttpException("Cannot assemble flow with missing content")
     head = assemble_request_head(request)
-    body = b"".join(assemble_body(request.data.headers, [request.data.content]))
+    body = b"".join(assemble_body(request.data.headers, [request.data.content], request.data.trailers))
     return head + body
 
 
@@ -19,7 +19,7 @@ def assemble_response(response):
     if response.data.content is None:
         raise exceptions.HttpException("Cannot assemble flow with missing content")
     head = assemble_response_head(response)
-    body = b"".join(assemble_body(response.data.headers, [response.data.content]))
+    body = b"".join(assemble_body(response.data.headers, [response.data.content], response.data.trailers))
     return head + body
 
 
@@ -29,13 +29,18 @@ def assemble_response_head(response):
     return b"%s\r\n%s\r\n" % (first_line, headers)
 
 
-def assemble_body(headers, body_chunks):
+def assemble_body(headers, body_chunks, trailers):
     if "chunked" in headers.get("transfer-encoding", "").lower():
         for chunk in body_chunks:
             if chunk:
                 yield b"%x\r\n%s\r\n" % (len(chunk), chunk)
-        yield b"0\r\n\r\n"
+        if trailers:
+            yield b"0\r\n%s\r\n" % (trailers,)
+        else:
+            yield b"0\r\n\r\n"
     else:
+        if trailers:
+            raise exceptions.HttpException("Sending HTTP/1 trailer headers requires transfer-encoding: chunked")
         for chunk in body_chunks:
             yield chunk
 

mitmproxy/proxy/protocol/http1.py

@@ -32,17 +32,15 @@ class Http1Layer(httpbase._HttpTransmissionLayer):
         self.server_conn.wfile.flush()
 
     def send_request_body(self, request, chunks):
-        for chunk in http1.assemble_body(request.headers, chunks):
+        for chunk in http1.assemble_body(request.headers, chunks, request.trailers):
             self.server_conn.wfile.write(chunk)
             self.server_conn.wfile.flush()
 
     def send_request_trailers(self, request):
-        if "Trailer" in request.headers:
-            # TODO: not implemented yet
-            self.log("HTTP/1 request trailer headers are not implemented yet!", "warn")
+        # HTTP/1 request trailer headers are sent in the body
+        pass
 
     def send_request(self, request):
-        # TODO: this does not yet support request trailers
         self.server_conn.wfile.write(http1.assemble_request(request))
         self.server_conn.wfile.flush()
 
@@ -58,6 +56,7 @@ class Http1Layer(httpbase._HttpTransmissionLayer):
         )
 
     def read_response_trailers(self, request, response):
+        # Trailers should actually be parsed unconditionally, the "Trailer" header is optional
         if "Trailer" in response.headers:
             # TODO: not implemented yet
             self.log("HTTP/1 trailer headers are not implemented yet!", "warn")
@@ -69,15 +68,13 @@ class Http1Layer(httpbase._HttpTransmissionLayer):
         self.client_conn.wfile.flush()
 
     def send_response_body(self, response, chunks):
-        for chunk in http1.assemble_body(response.headers, chunks):
+        for chunk in http1.assemble_body(response.headers, chunks, response.trailers):
             self.client_conn.wfile.write(chunk)
             self.client_conn.wfile.flush()
 
     def send_response_trailers(self, response):
-        if "Trailer" in response.headers:
-            # TODO: not implemented yet
-            self.log("HTTP/1 trailer headers are not implemented yet!", "warn")
-        return
+        # HTTP/1 response trailer headers are sent in the body
+        pass
 
     def check_close_connection(self, flow):
         request_close = http1.connection_close(

test/mitmproxy/net/http/http1/test_assemble.py

@@ -52,15 +52,18 @@ def test_assemble_response_head():
 
 
 def test_assemble_body():
-    c = list(assemble_body(Headers(), [b"body"]))
+    c = list(assemble_body(Headers(), [b"body"], Headers()))
     assert c == [b"body"]
 
-    c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a", b""]))
+    c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a", b""], Headers()))
     assert c == [b"a\r\n123456789a\r\n", b"0\r\n\r\n"]
 
-    c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a"]))
+    c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a"], Headers()))
     assert c == [b"a\r\n123456789a\r\n", b"0\r\n\r\n"]
 
+    c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a"], Headers(trailer="trailer")))
+    assert c == [b"a\r\n123456789a\r\n", b"0\r\ntrailer: trailer\r\n\r\n"]
+
 
 def test_assemble_request_line():
     assert _assemble_request_line(treq().data) == b"GET /path HTTP/1.1"