mirror of
https://github.com/vee1e/mitmproxy.git
synced 2026-09-02 18:57:22 +00:00
dfeddcf4ad
* Bump install-pinned/black
Bumps [install-pinned/black](https://github.com/install-pinned/black) from 9101a4d68e870eaaaae21c412d1d879b93c9afcb to 13c8a20eb904ba800c87f0b34ccfd932ac2ff81d.
- [Release notes](https://github.com/install-pinned/black/releases)
- [Commits](9101a4d68e...13c8a20eb9)
---
updated-dependencies:
- dependency-name: install-pinned/black
dependency-type: direct:production
...
Signed-off-by: dependabot[bot] <support@github.com>
* [autofix.ci] apply automated fixes
---------
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
33 lines
1.5 KiB
Python
33 lines
1.5 KiB
Python
# This script determines if request is an HTML webpage and if so seeks out
|
|
# relative links (<a href="./about.html">) and expands them to absolute links
|
|
# In practice this can be used to front an indexing spider that may not have the capability to expand relative page links.
|
|
# Usage: mitmdump -s link_expander.py or mitmproxy -s link_expander.py
|
|
import re
|
|
from urllib.parse import urljoin
|
|
|
|
|
|
def response(flow):
|
|
if (
|
|
"Content-Type" in flow.response.headers
|
|
and flow.response.headers["Content-Type"].find("text/html") != -1
|
|
):
|
|
pageUrl = flow.request.url
|
|
pageText = flow.response.text
|
|
pattern = (
|
|
r"<a\s+(?:[^>]*?\s+)?href=(?P<delimiter>[\"'])"
|
|
r"(?P<link>(?!https?:\/\/|ftps?:\/\/|\/\/|#|javascript:|mailto:).*?)(?P=delimiter)"
|
|
)
|
|
rel_matcher = re.compile(pattern, flags=re.IGNORECASE)
|
|
rel_matches = rel_matcher.finditer(pageText)
|
|
map_dict = {}
|
|
for match_num, match in enumerate(rel_matches):
|
|
(delimiter, rel_link) = match.group("delimiter", "link")
|
|
abs_link = urljoin(pageUrl, rel_link)
|
|
map_dict["{0}{1}{0}".format(delimiter, rel_link)] = "{0}{1}{0}".format(
|
|
delimiter, abs_link
|
|
)
|
|
for map in map_dict.items():
|
|
pageText = pageText.replace(*map)
|
|
# Uncomment the following to print the expansion mapping
|
|
# print("{0} -> {1}".format(*map))
|
|
flow.response.text = pageText
|