From 397fb8ece51cfbb1a55e81040d485e6be00ff56c Mon Sep 17 00:00:00 2001 From: Miroslav Date: Sat, 9 Jun 2018 15:51:17 +0300 Subject: [PATCH] Various fixes. Tests --- mitmproxy/command.py | 21 +++++----- mitmproxy/language/lexer.py | 14 +++---- mitmproxy/language/parser.py | 3 +- .../tools/console/commander/commander.py | 33 +++------------- setup.py | 2 +- test/mitmproxy/test_command.py | 38 ++++++++++++++----- .../mitmproxy/tools/console/test_commander.py | 5 --- .../tools/console/test_defaultkeys.py | 2 +- 8 files changed, 55 insertions(+), 63 deletions(-) diff --git a/mitmproxy/command.py b/mitmproxy/command.py index 2fe325ffa..79373299a 100644 --- a/mitmproxy/command.py +++ b/mitmproxy/command.py @@ -162,17 +162,17 @@ class CommandManager(mitmproxy.types._CommandBase): params: typing.List[type] = [] typ: typing.Type = None for i, part in enumerate(parts): - if (i == 0 and not part.isspace()) or i == 1: - typ = mitmproxy.types.Cmd - if part in self.commands: - params.extend(self.commands[part].paramtypes) - elif params and not part.isspace(): - typ = params.pop(0) - if typ == mitmproxy.types.Cmd and params and params[0] == mitmproxy.types.Arg: + typ = mitmproxy.types.Unknown + if not part.isspace(): + if i == 0 or (i == 1 and parts[0].isspace()): + typ = mitmproxy.types.Cmd if part in self.commands: - params[:] = self.commands[part].paramtypes - else: - typ = mitmproxy.types.Unknown + params.extend(self.commands[part].paramtypes) + elif params: + typ = params.pop(0) + if typ == mitmproxy.types.Cmd and params and params[0] == mitmproxy.types.Arg: + if part in self.commands: + params[:] = self.commands[part].paramtypes to = mitmproxy.types.CommandTypes.get(typ, None) valid = False @@ -191,6 +191,7 @@ class CommandManager(mitmproxy.types._CommandBase): valid=valid, ) ) + remhelp: typing.List[str] = [] for x in params: remt = mitmproxy.types.CommandTypes.get(x, None) diff --git a/mitmproxy/language/lexer.py b/mitmproxy/language/lexer.py index d3bf42241..4e394c9f0 100644 --- a/mitmproxy/language/lexer.py +++ b/mitmproxy/language/lexer.py @@ -1,4 +1,3 @@ -import re import typing import ply.lex as lex @@ -13,17 +12,14 @@ class CommandLanguageLexer: "PLAIN_STR", "QUOTED_STR" ) - states = ( ("interactive", "inclusive"), ) - special_symbols = re.escape("") # Symbols to ignore in PLAIN_STR. For example: ,'" - plain_str = fr"[^{special_symbols}\s]+" - def __init__(self, oneword_commands: typing.Sequence[str]): self.oneword_commands = dict.fromkeys(oneword_commands, "COMMAND") + # Main(INITIAL) state t_ignore_WHITESPACE = r"\s+" def t_COMMAND(self, t): @@ -37,8 +33,8 @@ class CommandLanguageLexer: """ return t - @lex.TOKEN(plain_str) def t_PLAIN_STR(self, t): + r"""[^\(\)\s]+""" t.type = self.oneword_commands.get(t.value, "PLAIN_STR") return t @@ -46,6 +42,7 @@ class CommandLanguageLexer: t.lexer.skip(1) raise exceptions.CommandError(f"Illegal character '{t.value[0]}'") + # Interactive state t_interactive_WHITESPACE = r"\s+" def build(self, **kwargs): @@ -59,7 +56,8 @@ def create_lexer(cmdstr: str, oneword_commands: typing.Sequence[str]) -> lex.Lex return command_lexer.lexer -def get_tokens(cmdstr: str) -> typing.List[str]: +def get_tokens(cmdstr: str, state="interactive") -> typing.List[str]: lexer = create_lexer(cmdstr, []) - lexer.begin("interactive") + # Switching to the state with meaningful white spaces + lexer.begin(state) return [token.value for token in lexer] diff --git a/mitmproxy/language/parser.py b/mitmproxy/language/parser.py index 9764fb9b3..44fe6e940 100644 --- a/mitmproxy/language/parser.py +++ b/mitmproxy/language/parser.py @@ -8,7 +8,8 @@ from mitmproxy.language.lexer import CommandLanguageLexer class CommandLanguageParser: - tokens = CommandLanguageLexer.tokens # it is always required + # the list of possible tokens is always required + tokens = CommandLanguageLexer.tokens def __init__(self, command_manager): self.return_value = None diff --git a/mitmproxy/tools/console/commander/commander.py b/mitmproxy/tools/console/commander/commander.py index 89a00633e..fc7996c00 100644 --- a/mitmproxy/tools/console/commander/commander.py +++ b/mitmproxy/tools/console/commander/commander.py @@ -50,7 +50,7 @@ CompletionState = typing.NamedTuple( class CommandBuffer: def __init__(self, master: mitmproxy.master.Master, start: str = "") -> None: self.master = master - self.text = self.flatten(start) + self.text = start # Cursor is always within the range [0:len(buffer)]. self._cursor = len(self.text) self.completion: CompletionState = None @@ -68,21 +68,6 @@ class CommandBuffer: else: self._cursor = x - def maybequote(self, value): - if " " in value and not value.startswith("\""): - return "\"%s\"" % value - return value - - def parse_quoted(self, txt): - parts, remhelp = self.master.commands.parse_partial(txt) - for i, p in enumerate(parts): - parts[i] = mitmproxy.command.ParseResult( - value = p.value, - type = p.type, - valid = p.valid - ) - return parts, remhelp - def render(self): """ This function is somewhat tricky - in order to make the cursor @@ -90,7 +75,7 @@ class CommandBuffer: character-for-character offset match in the rendered output, up to the cursor. Beyond that, we can add stuff. """ - parts, remhelp = self.parse_quoted(self.text) + parts, remhelp = self.master.commands.parse_partial(self.text) ret = [] for p in parts: @@ -109,17 +94,10 @@ class CommandBuffer: ret.append(("text", " ")) if remhelp: - # ret.append(("text", " ")) for v in remhelp: ret.append(("commander_hint", "%s " % v)) - # ret.append(("text", " ")) return ret - def flatten(self, txt): - parts, _ = self.parse_quoted(txt) - ret = [x.value for x in parts] - return "".join(ret) - def left(self) -> None: self.cursor = self.cursor - 1 @@ -142,14 +120,13 @@ class CommandBuffer: if self.completion: nxt = self.completion.completer.cycle() buf = "".join([i.value for i in self.completion.parse[:-1]]) + nxt - # buf = buf.strip() - self.text = self.flatten(buf) + self.text = buf self.cursor = len(self.text) def backspace(self) -> None: if self.cursor == 0: return - self.text = self.flatten(self.text[:self.cursor - 1] + self.text[self.cursor:]) + self.text = self.text[:self.cursor - 1] + self.text[self.cursor:] self.cursor = self.cursor - 1 self.completion = None @@ -157,7 +134,7 @@ class CommandBuffer: """ Inserts text at the cursor. """ - self.text = self.flatten(self.text[:self.cursor] + k + self.text[self.cursor:]) + self.text = self.text[:self.cursor] + k + self.text[self.cursor:] self.cursor += 1 self.completion = None diff --git a/setup.py b/setup.py index 7f9265fca..10223ec7f 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ setup( "kaitaistruct>=0.7,<0.9", "ldap3>=2.5,<2.6", "passlib>=1.6.5, <1.8", - "ply>=3.4, <3.12", + "ply>=3.6, <3.12", "pyasn1>=0.3.1,<0.5", "pyOpenSSL>=17.5,<18.1", "pyparsing>=2.1.3, <2.3", diff --git a/test/mitmproxy/test_command.py b/test/mitmproxy/test_command.py index 681b10f09..20ef75735 100644 --- a/test/mitmproxy/test_command.py +++ b/test/mitmproxy/test_command.py @@ -118,6 +118,9 @@ class TestCommand: command.ParseResult( value = "foo", type = mitmproxy.types.Cmd, valid = False ), + command.ParseResult( + value = " ", type = mitmproxy.types.Unknown, valid = False + ), command.ParseResult( value = "bar", type = mitmproxy.types.Unknown, valid = False ) @@ -128,6 +131,9 @@ class TestCommand: "cmd1 'bar", [ command.ParseResult(value = "cmd1", type = mitmproxy.types.Cmd, valid = True), + command.ParseResult( + value = " ", type = mitmproxy.types.Unknown, valid = False + ), command.ParseResult(value = "'bar", type = str, valid = True) ], [], @@ -142,10 +148,19 @@ class TestCommand: [command.ParseResult(value = "", type = mitmproxy.types.Cmd, valid = False)], [] ], + [ + " ", + [ + command.ParseResult(value = " ", type=mitmproxy.types.Unknown, valid=False), + command.ParseResult(value = "", type=mitmproxy.types.Cmd, valid=False) + ], + [] + ], [ "cmd3 1", [ command.ParseResult(value = "cmd3", type = mitmproxy.types.Cmd, valid = True), + command.ParseResult(value = " ", type=mitmproxy.types.Unknown, valid=False), command.ParseResult(value = "1", type = int, valid = True), ], [] @@ -154,6 +169,7 @@ class TestCommand: "cmd3 ", [ command.ParseResult(value = "cmd3", type = mitmproxy.types.Cmd, valid = True), + command.ParseResult(value = " ", type=mitmproxy.types.Unknown, valid=False), command.ParseResult(value = "", type = int, valid = False), ], [] @@ -164,6 +180,7 @@ class TestCommand: command.ParseResult( value = "subcommand", type = mitmproxy.types.Cmd, valid = True, ), + command.ParseResult(value = " ", type=mitmproxy.types.Unknown, valid=False), command.ParseResult(value = "", type = mitmproxy.types.Cmd, valid = False), ], ["arg"], @@ -172,7 +189,9 @@ class TestCommand: "subcommand cmd3 ", [ command.ParseResult(value = "subcommand", type = mitmproxy.types.Cmd, valid = True), + command.ParseResult(value = " ", type=mitmproxy.types.Unknown, valid=False), command.ParseResult(value = "cmd3", type = mitmproxy.types.Cmd, valid = True), + command.ParseResult(value = " ", type=mitmproxy.types.Unknown, valid=False), command.ParseResult(value = "", type = int, valid = False), ], [] @@ -188,6 +207,7 @@ class TestCommand: "cmd4 ", [ command.ParseResult(value = "cmd4", type = mitmproxy.types.Cmd, valid = True), + command.ParseResult(value = " ", type=mitmproxy.types.Unknown, valid=False), command.ParseResult(value = "", type = int, valid = False), ], ["str", "path"] @@ -196,14 +216,7 @@ class TestCommand: "cmd4 1", [ command.ParseResult(value = "cmd4", type = mitmproxy.types.Cmd, valid = True), - command.ParseResult(value = "1", type = int, valid = True), - ], - ["str", "path"] - ], - [ - "cmd4 1", - [ - command.ParseResult(value = "cmd4", type = mitmproxy.types.Cmd, valid = True), + command.ParseResult(value=" ", type=mitmproxy.types.Unknown, valid=False), command.ParseResult(value = "1", type = int, valid = True), ], ["str", "path"] @@ -219,6 +232,7 @@ class TestCommand: "flow ", [ command.ParseResult(value = "flow", type = mitmproxy.types.Cmd, valid = True), + command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False), command.ParseResult(value = "", type = flow.Flow, valid = False), ], ["str"] @@ -227,6 +241,7 @@ class TestCommand: "flow x", [ command.ParseResult(value = "flow", type = mitmproxy.types.Cmd, valid = True), + command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False), command.ParseResult(value = "x", type = flow.Flow, valid = False), ], ["str"] @@ -235,7 +250,9 @@ class TestCommand: "flow x ", [ command.ParseResult(value = "flow", type = mitmproxy.types.Cmd, valid = True), + command.ParseResult(value=" ", type=mitmproxy.types.Unknown, valid=False), command.ParseResult(value = "x", type = flow.Flow, valid = False), + command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False), command.ParseResult(value = "", type = str, valid = True), ], [] @@ -244,7 +261,9 @@ class TestCommand: "flow \"one two", [ command.ParseResult(value = "flow", type = mitmproxy.types.Cmd, valid = True), + command.ParseResult(value=" ", type=mitmproxy.types.Unknown, valid=False), command.ParseResult(value = "\"one", type = flow.Flow, valid = False), + command.ParseResult(value=" ", type=mitmproxy.types.Unknown, valid=False), command.ParseResult(value="two", type=str, valid=True), ], [] @@ -253,7 +272,8 @@ class TestCommand: "flow \"one two\"", [ command.ParseResult(value = "flow", type = mitmproxy.types.Cmd, valid = True), - command.ParseResult(value = "one two", type = flow.Flow, valid = False), + command.ParseResult(value=" ", type=mitmproxy.types.Unknown, valid=False), + command.ParseResult(value = "\"one two\"", type = flow.Flow, valid = False), ], ["str"] ], diff --git a/test/mitmproxy/tools/console/test_commander.py b/test/mitmproxy/tools/console/test_commander.py index 2a96995d5..8b6132a73 100644 --- a/test/mitmproxy/tools/console/test_commander.py +++ b/test/mitmproxy/tools/console/test_commander.py @@ -91,8 +91,3 @@ class TestCommandBuffer: cb = commander.CommandBuffer(tctx.master) cb.text = "foo" assert cb.render() - - def test_flatten(self): - with taddons.context() as tctx: - cb = commander.CommandBuffer(tctx.master) - assert cb.flatten("foo bar") == "foo bar" diff --git a/test/mitmproxy/tools/console/test_defaultkeys.py b/test/mitmproxy/tools/console/test_defaultkeys.py index ff2ee94e6..f60174aa7 100644 --- a/test/mitmproxy/tools/console/test_defaultkeys.py +++ b/test/mitmproxy/tools/console/test_defaultkeys.py @@ -16,7 +16,7 @@ async def test_commands_exist(): await m.load_flow(tflow()) for binding in km.bindings: - cmd, *args = lexer.get_tokens(binding.command) + cmd, *args = lexer.get_tokens(binding.command, state="INITIAL") assert cmd in m.commands.commands cmd_obj = m.commands.commands[cmd]