Going into mergeable state.

Some typing fixes since typing module doesn't fully support recursive types.
a few typos were fixed
test coverage, new tests for the new code
new syntax feature - assignment for 'set' command.

test/mitmproxy/language/test_traversal.py

@@ -0,0 +1,46 @@
+import typing
+import asyncio
+
+from mitmproxy import command
+from mitmproxy.test import taddons
+from mitmproxy.language import lexer, parser, traversal
+
+import pytest
+
+
+class TAddon:
+    @command.command("cmd1")
+    def cmd1(self, foo: typing.Sequence[str]) -> str:
+        return " ".join(foo)
+
+    @command.command("cmd2")
+    def cmd2(self, foo: str) -> str:
+        return foo
+
+    @command.command("cmd3")
+    async def cmd3(self, foo: str) -> str:
+        await asyncio.sleep(0.01)
+        return foo
+
+
+@pytest.mark.asyncio
+async def test_execute_parsed_line():
+    test_commands = ["""join.cmd1 [str.cmd2(abc)
+                        str.cmd2(strasync.cmd3("def"))]""",
+                     "[1 2 3]", "str.cmd2 abc | strasync.cmd3()"]
+    results = ["abc def", ['1', '2', '3'], "abc"]
+
+    with taddons.context() as tctx:
+        cm = command.CommandManager(tctx.master)
+        a = TAddon()
+        cm.add("join.cmd1", a.cmd1)
+        cm.add("str.cmd2", a.cmd2)
+        cm.add("strasync.cmd3", a.cmd3)
+
+        command_parser = parser.create_parser(cm)
+        for cmd, exp_res in zip(test_commands, results):
+            lxr = lexer.create_lexer(cmd, cm.oneword_commands)
+            parsed = command_parser.parse(lxr, async_exec=True)
+
+            result = await traversal.execute_parsed_line(parsed)
+            assert result == exp_res

test/mitmproxy/test_command.py

@@ -1,11 +1,15 @@
 import typing
 import inspect
+import asyncio
+from unittest import mock
+
+import mitmproxy.types
 from mitmproxy import command
 from mitmproxy import flow
 from mitmproxy import exceptions
 from mitmproxy.test import tflow
 from mitmproxy.test import taddons
-import mitmproxy.types
+
 import io
 import pytest
 
@@ -36,6 +40,15 @@ class TAddon:
     def cmd6(self, pipe_value: str) -> str:
         return pipe_value
 
+    @command.command("cmd7")
+    async def cmd7(self, foo: str) -> str:
+        await asyncio.sleep(0.01)
+        return foo
+
+    @command.command("cmd8")
+    async def cmd8(self, foo: str) -> str:
+        return 99
+
     @command.command("subcommand")
     def subcommand(self, cmd: mitmproxy.types.Cmd, *args: mitmproxy.types.Arg) -> str:
         return "ok"
@@ -44,6 +57,10 @@ class TAddon:
     def empty(self) -> None:
         pass
 
+    @command.command("empty")
+    async def asyncempty(self) -> None:
+        pass
+
     @command.command("varargs")
     def varargs(self, one: str, *var: str) -> typing.Sequence[str]:
         return list(var)
@@ -82,6 +99,35 @@ class TypeErrAddon:
         pass
 
 
+class TestAsyncExecutionManager:
+    def test_add_command(self):
+        aem = command.AsyncExecutionManager()
+        dummy_command = command.RunningCommand("addon.command", mock.Mock())
+        aem.add_command(dummy_command)
+        assert aem.running_cmds == {1: dummy_command}
+
+    def test_stop_command(self):
+        aem = command.AsyncExecutionManager()
+        dummy_command = command.RunningCommand("addon.command", mock.Mock())
+        aem.add_command(dummy_command)
+        with pytest.raises(ValueError, match="There is not the command"):
+            aem.stop_command(100)
+
+        aem.stop_command(1)
+        assert aem.running_cmds == {}
+
+    def test_get_runnings(self):
+        aem = command.AsyncExecutionManager()
+        expected_res = []
+        for i in range(3):
+            cmd = f"addon.command{i}"
+            dummy = command.RunningCommand(cmd, mock.Mock())
+            aem.add_command(dummy)
+            expected_res.append((i + 1, cmd))
+
+        assert aem.get_running() == expected_res
+
+
 class TestCommand:
     def test_typecheck(self):
         with taddons.context(loadcore=False) as tctx:
@@ -115,9 +161,25 @@ class TestCommand:
             with pytest.raises(exceptions.CommandError):
                 c.call(["foo"])
 
+            with pytest.raises(exceptions.CommandError, match="unexpected data"):
+                c.call([123])
+
             c = command.Command(cm, "cmd.three", a.cmd3)
             assert c.call(["1"]) == 1
 
+    @pytest.mark.asyncio
+    async def test_async_call(self):
+        with taddons.context() as tctx:
+            cm = command.CommandManager(tctx.master)
+            a = TAddon()
+
+            c = command.Command(cm, "async.empty", a.asyncempty)
+            await c.async_call([])
+
+            c = command.Command(cm, "asynccmd.two", a.cmd8)
+            with pytest.raises(exceptions.CommandError, match="unexpected data"):
+                await c.async_call(["foo"])
+
     def test_parse_partial(self):
         tests = [
             [
@@ -301,6 +363,7 @@ def test_simple():
         c.add("one.two", a.cmd1)
         c.add("array.command", a.cmd5)
         c.add("pipe.command", a.cmd6)
+        c.add("strasync.command", a.cmd7)
 
         assert c.commands["one.two"].help == "cmd1 help"
         assert(c.execute("one.two foo") == "ret foo")
@@ -320,6 +383,8 @@ def test_simple():
             c.execute("")
         with pytest.raises(exceptions.CommandError, match="argument mismatch"):
             c.execute("one.two too many args")
+        with pytest.raises(exceptions.ExecutionError, match="sync executor"):
+            c.execute("strasync.command abc")
         with pytest.raises(exceptions.CommandError, match="Unknown"):
             c.call("nonexistent")
 
@@ -331,6 +396,19 @@ def test_simple():
         assert fp.getvalue()
 
 
+@pytest.mark.asyncio
+async def test_async_execute():
+    with taddons.context() as tctx:
+        c = command.CommandManager(tctx.master)
+        a = TAddon()
+        c.add("strasync.command", a.cmd7)
+
+        c.async_execute("strasync.command abc")
+        assert c.async_manager.get_running() == [(1, "strasync.command abc")]
+        assert "abc" == await c.async_manager.running_cmds[1].task
+        assert c.async_manager.get_running() == []
+
+
 def test_typename():
     assert command.typename(str) == "str"
     assert command.typename(typing.Sequence[flow.Flow]) == "[flow]"

test/mitmproxy/tools/console/test_defaultkeys.py

@@ -2,7 +2,7 @@ from mitmproxy.test.tflow import tflow
 from mitmproxy.tools.console import defaultkeys
 from mitmproxy.tools.console import keymap
 from mitmproxy.tools.console import master
-from mitmproxy.language import lexer
+from mitmproxy.language import lexer, parser
 
 import pytest
 
@@ -15,12 +15,11 @@ async def test_commands_exist():
     m = master.ConsoleMaster(None)
     await m.load_flow(tflow())
 
-    for binding in km.bindings:
-        cmd, *args = lexer.get_tokens(binding.command, state="INITIAL")
-        assert cmd in m.commands.commands
+    command_parser = parser.create_parser(m.commands)
 
-        cmd_obj = m.commands.commands[cmd]
+    for binding in km.bindings:
+        lxr = lexer.create_lexer(binding.command, m.commands.oneword_commands)
         try:
-            cmd_obj.prepare_args(args)
+            command_parser.parse(lxr, async_exec=True)
         except Exception as e:
-            raise ValueError("Invalid command: {}".format(binding.command)) from e
+            raise ValueError(f"Invalid command: '{binding.command}'") from e