diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7be4cdc9..0681c15d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,8 +35,10 @@ jobs: python-version: 3.8 - if: matrix.os == 'ubuntu-18.04' run: sudo apt-get install -y libyaml-dev + - name: Upgrade pip, setuptools + run: pip install --upgrade pip setuptools - name: Install PyInstaller - run: pip install 'pyinstaller==4.2' + run: pip install 'pyinstaller==4.10' - name: Install capa run: pip install -e . - name: Build standalone executable diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 34eda0e0..91478bfe 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,7 +48,7 @@ jobs: - name: Checkout capa with submodules uses: actions/checkout@v2 with: - submodules: true + submodules: recursive - name: Set up Python 3.8 uses: actions/setup-python@v2 with: @@ -78,7 +78,7 @@ jobs: - name: Checkout capa with submodules uses: actions/checkout@v2 with: - submodules: true + submodules: recursive - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 84b6b3d1..466b05f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,15 +11,17 @@ ### Breaking Changes -- instruction scope and operand feature are new and are not backwards compatible with older versions of capa -- Python 3.7 is now the minimum supported Python version #866 @williballenthin + - instruction scope and operand feature are new and are not backwards compatible with older versions of capa + - remove /x32 and /x64 flavors of number and operand features #932 @williballenthin + - the tool now accepts multiple paths to rules, and JSON doc updated accordingly @williballenthin -### New Rules (4) +### New Rules (5) - data-manipulation/encryption/aes/manually-build-aes-constants huynh.t.nhan@gmail.com - nursery/get-process-image-filename michael.hunhoff@mandiant.com - compiler/v/compiled-with-v jakub.jozwiak@mandiant.com - compiler/zig/compiled-with-zig jakub.jozwiak@mandiant.com +- anti-analysis/packer/huan/packed-with-huan jakub.jozwiak@mandiant.com - ### Bug Fixes diff --git a/capa/features/common.py b/capa/features/common.py index bff1138c..ed2b288e 100644 --- a/capa/features/common.py +++ b/capa/features/common.py @@ -98,33 +98,23 @@ class Result: class Feature(abc.ABC): - def __init__(self, value: Union[str, int, bytes], bitness=None, description=None): + def __init__(self, value: Union[str, int, bytes], description=None): """ Args: value (any): the value of the feature, such as the number or string. - bitness (str): one of the VALID_BITNESS values, or None. - When None, then the feature applies to any bitness. - Modifies the feature name from `feature` to `feature/bitness`, like `offset/x32`. description (str): a human-readable description that explains the feature value. """ super(Feature, self).__init__() - - if bitness is not None: - if bitness not in VALID_BITNESS: - raise ValueError("bitness '%s' must be one of %s" % (bitness, VALID_BITNESS)) - self.name = self.__class__.__name__.lower() + "/" + bitness - else: - self.name = self.__class__.__name__.lower() + self.name = self.__class__.__name__.lower() self.value = value - self.bitness = bitness self.description = description def __hash__(self): - return hash((self.name, self.value, self.bitness)) + return hash((self.name, self.value)) def __eq__(self, other): - return self.name == other.name and self.value == other.value and self.bitness == other.bitness + return self.name == other.name and self.value == other.value def get_value_str(self) -> str: """ @@ -153,10 +143,7 @@ class Feature(abc.ABC): return Result(self in ctx, self, [], locations=ctx.get(self, [])) def freeze_serialize(self): - if self.bitness is not None: - return (self.__class__.__name__, [self.value, {"bitness": self.bitness}]) - else: - return (self.__class__.__name__, [self.value]) + return (self.__class__.__name__, [self.value]) @classmethod def freeze_deserialize(cls, args): @@ -400,13 +387,6 @@ class Bytes(Feature): return cls(*[codecs.decode(x, "hex") for x in args]) -# identifiers for supported bitness names that tweak a feature -# for example, offset/x32 -BITNESS_X32 = "x32" -BITNESS_X64 = "x64" -VALID_BITNESS = (BITNESS_X32, BITNESS_X64) - - # other candidates here: https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#machine-types ARCH_I386 = "i386" ARCH_AMD64 = "amd64" diff --git a/capa/features/extractors/dnfile_.py b/capa/features/extractors/dnfile_.py index 715e8a5f..af9befcc 100644 --- a/capa/features/extractors/dnfile_.py +++ b/capa/features/extractors/dnfile_.py @@ -67,6 +67,9 @@ class DnfileFeatureExtractor(FeatureExtractor): return 0x0 def get_entry_point(self) -> int: + # self.pe.net.Flags.CLT_NATIVE_ENTRYPOINT + # True: native EP: Token + # False: managed EP: RVA return self.pe.net.struct.EntryPointTokenOrRva def extract_global_features(self): @@ -78,6 +81,9 @@ class DnfileFeatureExtractor(FeatureExtractor): def is_dotnet_file(self) -> bool: return bool(self.pe.net) + def is_mixed_mode(self) -> bool: + return not bool(self.pe.net.Flags.CLR_ILONLY) + def get_runtime_version(self) -> Tuple[int, int]: return self.pe.net.struct.MajorRuntimeVersion, self.pe.net.struct.MinorRuntimeVersion diff --git a/capa/features/extractors/helpers.py b/capa/features/extractors/helpers.py index f6d91333..8593a5bb 100644 --- a/capa/features/extractors/helpers.py +++ b/capa/features/extractors/helpers.py @@ -29,8 +29,7 @@ def is_aw_function(symbol: str) -> bool: if symbol[-1] not in ("A", "W"): return False - # second to last character should be lowercase letter - return "a" <= symbol[-2] <= "z" or "0" <= symbol[-2] <= "9" + return True def is_ordinal(symbol: str) -> bool: diff --git a/capa/features/extractors/ida/insn.py b/capa/features/extractors/ida/insn.py index 306bb954..15942c78 100644 --- a/capa/features/extractors/ida/insn.py +++ b/capa/features/extractors/ida/insn.py @@ -13,39 +13,13 @@ import idautils import capa.features.extractors.helpers import capa.features.extractors.ida.helpers from capa.features.insn import API, Number, Offset, Mnemonic, OperandNumber, OperandOffset -from capa.features.common import ( - BITNESS_X32, - BITNESS_X64, - MAX_BYTES_FEATURE_SIZE, - THUNK_CHAIN_DEPTH_DELTA, - Bytes, - String, - Characteristic, -) +from capa.features.common import MAX_BYTES_FEATURE_SIZE, THUNK_CHAIN_DEPTH_DELTA, Bytes, String, Characteristic # security cookie checks may perform non-zeroing XORs, these are expected within a certain # byte range within the first and returning basic blocks, this helps to reduce FP features SECURITY_COOKIE_BYTES_DELTA = 0x40 -def get_bitness(ctx): - """ - fetch the BITNESS_* constant for the currently open workspace. - - via Tamir Bahar/@tmr232 - https://reverseengineering.stackexchange.com/a/11398/17194 - """ - if "bitness" not in ctx: - info = idaapi.get_inf_structure() - if info.is_64bit(): - ctx["bitness"] = BITNESS_X64 - elif info.is_32bit(): - ctx["bitness"] = BITNESS_X32 - else: - raise ValueError("unexpected bitness") - return ctx["bitness"] - - def get_imports(ctx): if "imports_cache" not in ctx: ctx["imports_cache"] = capa.features.extractors.ida.helpers.get_file_imports() @@ -159,7 +133,6 @@ def extract_insn_number_features(f, bb, insn): const = op.addr yield Number(const), insn.ea - yield Number(const, bitness=get_bitness(f.ctx)), insn.ea yield OperandNumber(i, const), insn.ea @@ -234,7 +207,6 @@ def extract_insn_offset_features(f, bb, insn): op_off = capa.features.extractors.helpers.twos_complement(op_off, 32) yield Offset(op_off), insn.ea - yield Offset(op_off, bitness=get_bitness(f.ctx)), insn.ea yield OperandOffset(i, op_off), insn.ea diff --git a/capa/features/extractors/smda/insn.py b/capa/features/extractors/smda/insn.py index e4b92132..309389e8 100644 --- a/capa/features/extractors/smda/insn.py +++ b/capa/features/extractors/smda/insn.py @@ -6,15 +6,7 @@ from smda.common.SmdaReport import SmdaReport import capa.features.extractors.helpers from capa.features.insn import API, Number, Offset, Mnemonic -from capa.features.common import ( - BITNESS_X32, - BITNESS_X64, - MAX_BYTES_FEATURE_SIZE, - THUNK_CHAIN_DEPTH_DELTA, - Bytes, - String, - Characteristic, -) +from capa.features.common import MAX_BYTES_FEATURE_SIZE, THUNK_CHAIN_DEPTH_DELTA, Bytes, String, Characteristic # security cookie checks may perform non-zeroing XORs, these are expected within a certain # byte range within the first and returning basic blocks, this helps to reduce FP features @@ -23,16 +15,6 @@ PATTERN_HEXNUM = re.compile(r"[+\-] (?P0x[a-fA-F0-9]+)") PATTERN_SINGLENUM = re.compile(r"[+\-] (?P[0-9])") -def get_bitness(smda_report): - if smda_report.architecture == "intel": - if smda_report.bitness == 32: - return BITNESS_X32 - elif smda_report.bitness == 64: - return BITNESS_X64 - else: - raise NotImplementedError - - def extract_insn_api_features(f, bb, insn): """parse API features from the given instruction.""" if insn.offset in f.apirefs: @@ -89,7 +71,6 @@ def extract_insn_number_features(f, bb, insn): value = int(operand, 16) & ((1 << f.smda_report.bitness) - 1) yield Number(value), insn.offset - yield Number(value, bitness=get_bitness(f.smda_report)), insn.offset except: continue @@ -232,7 +213,6 @@ def extract_insn_offset_features(f, bb, insn): number = int(number_int.group("num")) number = -1 * number if number_int.group().startswith("-") else number yield Offset(number), insn.offset - yield Offset(number, bitness=get_bitness(f.smda_report)), insn.offset def is_security_cookie(f, bb, insn): diff --git a/capa/features/extractors/viv/insn.py b/capa/features/extractors/viv/insn.py index 3ae065db..e721606f 100644 --- a/capa/features/extractors/viv/insn.py +++ b/capa/features/extractors/viv/insn.py @@ -18,15 +18,7 @@ import envi.archs.amd64.disasm import capa.features.extractors.helpers import capa.features.extractors.viv.helpers from capa.features.insn import API, Number, Offset, Mnemonic, OperandNumber, OperandOffset -from capa.features.common import ( - BITNESS_X32, - BITNESS_X64, - MAX_BYTES_FEATURE_SIZE, - THUNK_CHAIN_DEPTH_DELTA, - Bytes, - String, - Characteristic, -) +from capa.features.common import MAX_BYTES_FEATURE_SIZE, THUNK_CHAIN_DEPTH_DELTA, Bytes, String, Characteristic from capa.features.extractors.viv.indirect_calls import NotFoundError, resolve_indirect_call # security cookie checks may perform non-zeroing XORs, these are expected within a certain @@ -34,14 +26,6 @@ from capa.features.extractors.viv.indirect_calls import NotFoundError, resolve_i SECURITY_COOKIE_BYTES_DELTA = 0x40 -def get_bitness(vw): - bitness = vw.getMeta("Architecture") - if bitness == "i386": - return BITNESS_X32 - elif bitness == "amd64": - return BITNESS_X64 - - def interface_extract_instruction_XXX(f, bb, insn): """ parse features from the given instruction. @@ -553,7 +537,6 @@ def extract_op_number_features(f, bb, insn, i, oper): return yield Number(v), insn.va - yield Number(v, bitness=get_bitness(f.vw)), insn.va yield OperandNumber(i, v), insn.va @@ -582,7 +565,6 @@ def extract_op_offset_features(f, bb, insn, i, oper): v = oper.disp yield Offset(v), insn.va - yield Offset(v, bitness=get_bitness(f.vw)), insn.va yield OperandOffset(i, v), insn.va # like: [esi + ecx + 16384] @@ -594,7 +576,6 @@ def extract_op_offset_features(f, bb, insn, i, oper): v = oper.disp yield Offset(v), insn.va - yield Offset(v, bitness=get_bitness(f.vw)), insn.va yield OperandOffset(i, v), insn.va diff --git a/capa/features/insn.py b/capa/features/insn.py index 957450a3..28622b05 100644 --- a/capa/features/insn.py +++ b/capa/features/insn.py @@ -22,16 +22,16 @@ class API(Feature): class Number(Feature): - def __init__(self, value: int, bitness=None, description=None): - super(Number, self).__init__(value, bitness=bitness, description=description) + def __init__(self, value: int, description=None): + super(Number, self).__init__(value, description=description) def get_value_str(self): return capa.render.utils.hex(self.value) class Offset(Feature): - def __init__(self, value: int, bitness=None, description=None): - super(Offset, self).__init__(value, bitness=bitness, description=description) + def __init__(self, value: int, description=None): + super(Offset, self).__init__(value, description=description) def get_value_str(self): return capa.render.utils.hex(self.value) @@ -42,7 +42,11 @@ class Mnemonic(Feature): super(Mnemonic, self).__init__(value, description=description) -MAX_OPERAND_INDEX = 3 +# max number of operands to consider for a given instrucion. +# since we only support Intel and .NET, we can assume this is 3 +# which covers cases up to e.g. "vinserti128 ymm0,ymm0,ymm5,1" +MAX_OPERAND_COUNT = 4 +MAX_OPERAND_INDEX = MAX_OPERAND_COUNT - 1 class _Operand(Feature, abc.ABC): @@ -53,7 +57,7 @@ class _Operand(Feature, abc.ABC): self.index = index def __hash__(self): - return hash((self.name, self.value, self.bitness)) + return hash((self.name, self.value)) def __eq__(self, other): return super().__eq__(other) and self.index == other.index @@ -64,7 +68,7 @@ class _Operand(Feature, abc.ABC): class OperandNumber(_Operand): # cached names so we don't do extra string formatting every ctor - NAMES = ["operand[%d].number" % i for i in range(MAX_OPERAND_INDEX)] + NAMES = ["operand[%d].number" % i for i in range(MAX_OPERAND_COUNT)] # operand[i].number: 0x12 def __init__(self, index: int, value: int, description=None): @@ -78,7 +82,7 @@ class OperandNumber(_Operand): class OperandOffset(_Operand): # cached names so we don't do extra string formatting every ctor - NAMES = ["operand[%d].offset" % i for i in range(MAX_OPERAND_INDEX)] + NAMES = ["operand[%d].offset" % i for i in range(MAX_OPERAND_COUNT)] # operand[i].offset: 0x12 def __init__(self, index: int, value: int, description=None): diff --git a/capa/ida/plugin/README.md b/capa/ida/plugin/README.md index 4dc29182..4900dbb3 100644 --- a/capa/ida/plugin/README.md +++ b/capa/ida/plugin/README.md @@ -34,19 +34,18 @@ For more information on the FLARE team's open-source framework, capa, check out ### Requirements -capa explorer supports Python versions >= 3.6.x and the following IDA Pro versions: +capa explorer supports Python versions >= 3.7.x and the following IDA Pro versions: * IDA 7.4 * IDA 7.5 * IDA 7.6 (caveat below) * IDA 7.7 -capa explorer is however limited to the Python versions supported by your IDA installation (which may not include all Python versions >= 3.6.x). Based on our testing the following matrix shows the Python versions supported +capa explorer is however limited to the Python versions supported by your IDA installation (which may not include all Python versions >= 3.7.x). Based on our testing the following matrix shows the Python versions supported by each supported IDA version: | | IDA 7.4 | IDA 7.5 | IDA 7.6 | | --- | --- | --- | --- | -| Python 3.6.x | Yes | Yes | Yes | | Python 3.7.x | Yes | Yes | Yes | | Python 3.8.x | Partial (see below) | Yes | Yes | | Python 3.9.x | No | Partial (see below) | Yes | diff --git a/capa/ida/plugin/model.py b/capa/ida/plugin/model.py index 01161375..bf2267b9 100644 --- a/capa/ida/plugin/model.py +++ b/capa/ida/plugin/model.py @@ -581,10 +581,6 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel): "mnemonic", "number", "offset", - "number/x32", - "number/x64", - "offset/x32", - "offset/x64", ): # display instruction preview return CapaExplorerInstructionViewItem(parent, display, location) diff --git a/capa/main.py b/capa/main.py index 7499afc3..5b306c7c 100644 --- a/capa/main.py +++ b/capa/main.py @@ -571,32 +571,33 @@ def is_nursery_rule_path(path: str) -> bool: return "nursery" in path -def get_rules(rule_path: str, disable_progress=False) -> List[Rule]: - if not os.path.exists(rule_path): - raise IOError("rule path %s does not exist or cannot be accessed" % rule_path) +def get_rules(rule_paths: List[str], disable_progress=False) -> List[Rule]: + rule_file_paths = [] + for rule_path in rule_paths: + if not os.path.exists(rule_path): + raise IOError("rule path %s does not exist or cannot be accessed" % rule_path) - rule_paths = [] - if os.path.isfile(rule_path): - rule_paths.append(rule_path) - elif os.path.isdir(rule_path): - logger.debug("reading rules from directory %s", rule_path) - for root, dirs, files in os.walk(rule_path): - if ".github" in root: - # the .github directory contains CI config in capa-rules - # this includes some .yml files - # these are not rules - continue - - for file in files: - if not file.endswith(".yml"): - if not (file.startswith(".git") or file.endswith((".git", ".md", ".txt"))): - # expect to see .git* files, readme.md, format.md, and maybe a .git directory - # other things maybe are rules, but are mis-named. - logger.warning("skipping non-.yml file: %s", file) + if os.path.isfile(rule_path): + rule_file_paths.append(rule_path) + elif os.path.isdir(rule_path): + logger.debug("reading rules from directory %s", rule_path) + for root, dirs, files in os.walk(rule_path): + if ".github" in root: + # the .github directory contains CI config in capa-rules + # this includes some .yml files + # these are not rules continue - rule_path = os.path.join(root, file) - rule_paths.append(rule_path) + for file in files: + if not file.endswith(".yml"): + if not (file.startswith(".git") or file.endswith((".git", ".md", ".txt"))): + # expect to see .git* files, readme.md, format.md, and maybe a .git directory + # other things maybe are rules, but are mis-named. + logger.warning("skipping non-.yml file: %s", file) + continue + + rule_path = os.path.join(root, file) + rule_file_paths.append(rule_path) rules = [] # type: List[Rule] @@ -606,14 +607,14 @@ def get_rules(rule_path: str, disable_progress=False) -> List[Rule]: # to disable progress completely pbar = lambda s, *args, **kwargs: s - for rule_path in pbar(list(rule_paths), desc="loading ", unit=" rules"): + for rule_file_path in pbar(list(rule_file_paths), desc="loading ", unit=" rules"): try: - rule = capa.rules.Rule.from_yaml_file(rule_path) + rule = capa.rules.Rule.from_yaml_file(rule_file_path) except capa.rules.InvalidRule: raise else: - rule.meta["capa/path"] = rule_path - if is_nursery_rule_path(rule_path): + rule.meta["capa/path"] = rule_file_path + if is_nursery_rule_path(rule_file_path): rule.meta["capa/nursery"] = True rules.append(rule) @@ -662,8 +663,8 @@ def collect_metadata(argv, sample_path, rules_path, extractor): sha1.update(buf) sha256.update(buf) - if rules_path != RULES_PATH_DEFAULT_STRING: - rules_path = os.path.abspath(os.path.normpath(rules_path)) + if rules_path != [RULES_PATH_DEFAULT_STRING]: + rules_path = [os.path.abspath(os.path.normpath(r)) for r in rules_path] format_ = get_format(sample_path) arch = get_arch(sample_path) @@ -827,7 +828,8 @@ def install_common_args(parser, wanted=None): "-r", "--rules", type=str, - default=RULES_PATH_DEFAULT_STRING, + default=[RULES_PATH_DEFAULT_STRING], + action="append", help="path to rule file or directory, use embedded rules by default", ) @@ -868,7 +870,7 @@ def handle_common_args(args): # disable vivisect-related logging, it's verbose and not relevant for capa users set_vivisect_log_level(logging.CRITICAL) - # Since Python 3.8 cp65001 is an alias to utf_8, but not for Pyhton < 3.8 + # Since Python 3.8 cp65001 is an alias to utf_8, but not for Python < 3.8 # TODO: remove this code when only supporting Python 3.8+ # https://stackoverflow.com/a/3259271/87207 import codecs @@ -889,7 +891,9 @@ def handle_common_args(args): raise RuntimeError("unexpected --color value: " + args.color) if hasattr(args, "rules"): - if args.rules == RULES_PATH_DEFAULT_STRING: + rules_paths: List[str] = [] + + if args.rules == [RULES_PATH_DEFAULT_STRING]: logger.debug("-" * 80) logger.debug(" Using default embedded rules.") logger.debug(" To provide your own rules, use the form `capa.exe -r ./path/to/rules/ /path/to/mal.exe`.") @@ -897,9 +901,9 @@ def handle_common_args(args): logger.debug(" https://github.com/mandiant/capa-rules") logger.debug("-" * 80) - rules_path = os.path.join(get_default_root(), "rules") + default_rule_path = os.path.join(get_default_root(), "rules") - if not os.path.exists(rules_path): + if not os.path.exists(default_rule_path): # when a users installs capa via pip, # this pulls down just the source code - not the default rules. # i'm not sure the default rules should even be written to the library directory, @@ -907,11 +911,18 @@ def handle_common_args(args): logger.error("default embedded rules not found! (maybe you installed capa as a library?)") logger.error("provide your own rule set via the `-r` option.") return E_MISSING_RULES - else: - rules_path = args.rules - logger.debug("using rules path: %s", rules_path) - args.rules = rules_path + rules_paths.append(default_rule_path) + else: + rules_paths = args.rules + + if RULES_PATH_DEFAULT_STRING in rules_paths: + rules_paths.remove(RULES_PATH_DEFAULT_STRING) + + for rule_path in rules_paths: + logger.debug("using rules path: %s", rule_path) + + args.rules = rules_paths if hasattr(args, "signatures"): if args.signatures == SIGNATURES_PATH_DEFAULT_STRING: @@ -931,8 +942,8 @@ def handle_common_args(args): def main(argv=None): - if sys.version_info < (3, 6): - raise UnsupportedRuntimeError("This version of capa can only be used with Python 3.6+") + if sys.version_info < (3, 7): + raise UnsupportedRuntimeError("This version of capa can only be used with Python 3.7+") if argv is None: argv = sys.argv[1:] diff --git a/capa/render/verbose.py b/capa/render/verbose.py index 9e92fc97..9cf99fb1 100644 --- a/capa/render/verbose.py +++ b/capa/render/verbose.py @@ -62,7 +62,7 @@ def render_meta(ostream, doc): ("arch", doc["meta"]["analysis"]["arch"]), ("extractor", doc["meta"]["analysis"]["extractor"]), ("base address", hex(doc["meta"]["analysis"]["base_address"])), - ("rules", doc["meta"]["analysis"]["rules"]), + ("rules", "\n".join(doc["meta"]["analysis"]["rules"])), ("function count", len(doc["meta"]["analysis"]["feature_counts"]["functions"])), ("library function count", len(doc["meta"]["analysis"]["library_functions"])), ( @@ -71,6 +71,7 @@ def render_meta(ostream, doc): + sum(doc["meta"]["analysis"]["feature_counts"]["functions"].values()), ), ] + ostream.writeln(tabulate.tabulate(rows, tablefmt="plain")) diff --git a/capa/render/vverbose.py b/capa/render/vverbose.py index 14269a9f..f93a6045 100644 --- a/capa/render/vverbose.py +++ b/capa/render/vverbose.py @@ -39,7 +39,13 @@ def render_locations(ostream, match): def render_statement(ostream, match, statement, indent=0): ostream.write(" " * indent) if statement["type"] in ("and", "or", "optional", "not", "subscope"): - ostream.write(statement["type"]) + if statement["type"] == "subscope": + # emit `basic block:` + # rather than `subscope:` + ostream.write(statement["subscope"]) + else: + # emit `and:` + ostream.write(statement["type"]) ostream.write(":") if statement.get("description"): ostream.write(" = %s" % statement["description"]) @@ -196,7 +202,6 @@ def render_rules(ostream, doc): author michael.hunhoff@mandiant.com scope function mbc Anti-Behavioral Analysis::Detect Debugger::OutputDebugString - examples Practical Malware Analysis Lab 16-02.exe_:0x401020 function @ 0x10004706 and: api: kernel32.SetLastError @ 0x100047C2 @@ -209,7 +214,17 @@ def render_rules(ostream, doc): functions_by_bb[bb] = function had_match = False - for rule in rutils.capability_rules(doc): + + for (_, _, rule) in sorted( + map(lambda rule: (rule["meta"].get("namespace", ""), rule["meta"]["name"], rule), doc["rules"].values()) + ): + # default scope hides things like lib rules, malware-category rules, etc. + # but in vverbose mode, we really want to show everything. + # + # still ignore subscope rules because they're stitched into the final document. + if rule["meta"].get("capa/subscope"): + continue + count = len(rule["matches"]) if count == 1: capability = rutils.bold(rule["meta"]["name"]) @@ -224,6 +239,13 @@ def render_rules(ostream, doc): if key == "name" or key not in rule["meta"]: continue + if key == "examples": + # I can't think of a reason that an analyst would pivot to the concrete example + # directly from the capa output. + # the more likely flow is to review the rule and go from there. + # so, don't make the output messy by showing the examples. + continue + v = rule["meta"][key] if not v: continue diff --git a/capa/rules.py b/capa/rules.py index d776df4e..f42b56a4 100644 --- a/capa/rules.py +++ b/capa/rules.py @@ -257,20 +257,8 @@ def parse_feature(key: str): return capa.features.common.Bytes elif key == "number": return capa.features.insn.Number - elif key.startswith("number/"): - bitness = key.partition("/")[2] - # the other handlers here return constructors for features, - # and we want to as well, - # however, we need to preconfigure one of the arguments (`bitness`). - # so, instead we return a partially-applied function that - # provides `bitness` to the feature constructor. - # it forwards any other arguments provided to the closure along to the constructor. - return functools.partial(capa.features.insn.Number, bitness=bitness) elif key == "offset": return capa.features.insn.Offset - elif key.startswith("offset/"): - bitness = key.partition("/")[2] - return functools.partial(capa.features.insn.Offset, bitness=bitness) elif key == "mnemonic": return capa.features.insn.Mnemonic elif key == "basic blocks": diff --git a/rules b/rules index 82308c41..f8a03a30 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 82308c4109c97688e8a2e3f346520b064fb868e7 +Subproject commit f8a03a3014c9e7fa97cfd5b681cfe089d6083de0 diff --git a/scripts/capa2yara.py b/scripts/capa2yara.py index bdc4521a..8206217f 100644 --- a/scripts/capa2yara.py +++ b/scripts/capa2yara.py @@ -43,7 +43,7 @@ import capa.rules import capa.engine import capa.features import capa.features.insn -from capa.features.common import BITNESS_X32, BITNESS_X64, String +from capa.features.common import String logger = logging.getLogger("capa2yara") @@ -703,7 +703,7 @@ def main(argv=None): logging.getLogger("capa2yara").setLevel(level) try: - rules = capa.main.get_rules(args.rules, disable_progress=True) + rules = capa.main.get_rules([args.rules], disable_progress=True) namespaces = capa.rules.index_rules_by_namespace(list(rules)) rules = capa.rules.RuleSet(rules) logger.info("successfully loaded %s rules (including subscope rules which will be ignored)", len(rules)) diff --git a/scripts/capa_as_library.py b/scripts/capa_as_library.py index 885ef03b..bd860af5 100644 --- a/scripts/capa_as_library.py +++ b/scripts/capa_as_library.py @@ -17,7 +17,7 @@ from capa.engine import * RULES_PATH = "/tmp/capa/rules/" # load rules from disk -rules = capa.rules.RuleSet(capa.main.get_rules(RULES_PATH, disable_progress=True)) +rules = capa.rules.RuleSet(capa.main.get_rules([RULES_PATH], disable_progress=True)) # == Render ddictionary helpers def render_meta(doc, ostream): diff --git a/scripts/lint.py b/scripts/lint.py index f4ba39ed..553e9f2e 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -162,10 +162,10 @@ class MissingScope(Lint): class InvalidScope(Lint): name = "invalid scope" - recommendation = "Use only file, function, or basic block rule scopes" + recommendation = "Use only file, function, basic block, or instruction rule scopes" def check_rule(self, ctx: Context, rule: Rule): - return rule.meta.get("scope") not in ("file", "function", "basic block") + return rule.meta.get("scope") not in ("file", "function", "basic block", "instruction") class MissingAuthor(Lint): @@ -963,7 +963,7 @@ def main(argv=None): parser = argparse.ArgumentParser(description="Lint capa rules.") capa.main.install_common_args(parser, wanted={"tag"}) - parser.add_argument("rules", type=str, help="Path to rules") + parser.add_argument("rules", type=str, action="append", help="Path to rules") parser.add_argument("--samples", type=str, default=samples_path, help="Path to samples") parser.add_argument( "--thorough", diff --git a/setup.py b/setup.py index f07ae6c9..64e5f03e 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ setuptools.setup( install_requires=requirements, extras_require={ "dev": [ - "pytest==7.0.1", + "pytest==7.1.1", "pytest-sugar==0.9.4", "pytest-instafail==0.4.2", "pytest-cov==3.0.0", diff --git a/tests/data b/tests/data index d2ce5d84..12c64af2 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit d2ce5d846ba6af3193edecc1fadbb2056d1ddde9 +Subproject commit 12c64af268337d6213b603e00cb9df908c779ff9 diff --git a/tests/fixtures.py b/tests/fixtures.py index c236d43f..18400ca7 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -31,14 +31,13 @@ from capa.features.common import ( ARCH_AMD64, FORMAT_ELF, OS_WINDOWS, - BITNESS_X32, - BITNESS_X64, FORMAT_DOTNET, Arch, Format, ) CD = os.path.dirname(__file__) +DNFILE_TESTFILES = "dnfile-testfiles" @contextlib.contextmanager @@ -234,6 +233,8 @@ def get_data_path_by_name(name): return os.path.join(CD, "data", "946a99f36a46d335dec080d9a4371940.dll_") elif name.startswith("b9f5b"): return os.path.join(CD, "data", "b9f5bd514485fb06da39beff051b9fdc.exe_") + elif name.startswith("mixed-mode-64"): + return os.path.join(CD, "data", DNFILE_TESTFILES, "mixed-mode", "ModuleCode", "bin", "ModuleCode_amd64.exe") else: raise ValueError("unexpected sample fixture: %s" % name) @@ -443,10 +444,6 @@ FEATURE_PRESENCE_TESTS = sorted( # insn/number: stack adjustments ("mimikatz", "function=0x40105D", capa.features.insn.Number(0xC), False), ("mimikatz", "function=0x40105D", capa.features.insn.Number(0x10), False), - # insn/number: bitness flavors - ("mimikatz", "function=0x40105D", capa.features.insn.Number(0xFF), True), - ("mimikatz", "function=0x40105D", capa.features.insn.Number(0xFF, bitness=BITNESS_X32), True), - ("mimikatz", "function=0x40105D", capa.features.insn.Number(0xFF, bitness=BITNESS_X64), False), # insn/number: negative ("mimikatz", "function=0x401553", capa.features.insn.Number(0xFFFFFFFF), True), ("mimikatz", "function=0x43e543", capa.features.insn.Number(0xFFFFFFF0), True), @@ -462,10 +459,6 @@ FEATURE_PRESENCE_TESTS = sorted( # insn/offset: negative ("mimikatz", "function=0x4011FB", capa.features.insn.Offset(-0x1), True), ("mimikatz", "function=0x4011FB", capa.features.insn.Offset(-0x2), True), - # insn/offset: bitness flavors - ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0x0), True), - ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0x0, bitness=BITNESS_X32), True), - ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0x0, bitness=BITNESS_X64), False), # insn/api ("mimikatz", "function=0x403BAC", capa.features.insn.API("advapi32.CryptAcquireContextW"), True), ("mimikatz", "function=0x403BAC", capa.features.insn.API("advapi32.CryptAcquireContext"), True), @@ -599,6 +592,8 @@ FEATURE_PRESENCE_TESTS_DOTNET = sorted( [ ("b9f5b", "file", Arch(ARCH_I386), True), ("b9f5b", "file", Arch(ARCH_AMD64), False), + ("mixed-mode-64", "file", Arch(ARCH_AMD64), True), + ("mixed-mode-64", "file", Arch(ARCH_I386), False), ("b9f5b", "file", OS(OS_ANY), True), ("b9f5b", "file", Format(FORMAT_DOTNET), True), ], @@ -722,5 +717,10 @@ def pingtaest_extractor(): @pytest.fixture -def b9f5b_extractor(): +def b9f5b_dnfile_extractor(): return get_dnfile_extractor(get_data_path_by_name("b9f5b")) + + +@pytest.fixture +def mixed_mode_64_dnfile_extractor(): + return get_dnfile_extractor(get_data_path_by_name("mixed-mode-64")) diff --git a/tests/test_dotnet_features.py b/tests/test_dotnet_features.py index 10b5183f..b6714419 100644 --- a/tests/test_dotnet_features.py +++ b/tests/test_dotnet_features.py @@ -6,15 +6,11 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. -# b9f5bd514485fb06da39beff051b9fdc - import pytest import fixtures from fixtures import * from fixtures import parametrize -import capa.features.file - @parametrize( "sample,scope,feature,expected", @@ -26,14 +22,16 @@ def test_dnfile_features(sample, scope, feature, expected): @parametrize( - "function,expected", + "extractor,function,expected", [ - ("is_dotnet_file", True), - ("get_entry_point", 0x6000007), - ("get_runtime_version", (2, 5)), - ("get_meta_version_string", "v2.0.50727"), + ("b9f5b_dnfile_extractor", "is_dotnet_file", True), + ("b9f5b_dnfile_extractor", "is_mixed_mode", False), + ("mixed_mode_64_dnfile_extractor", "is_mixed_mode", True), + ("b9f5b_dnfile_extractor", "get_entry_point", 0x6000007), + ("b9f5b_dnfile_extractor", "get_runtime_version", (2, 5)), + ("b9f5b_dnfile_extractor", "get_meta_version_string", "v2.0.50727"), ], ) -def test_dnfile_extractor(b9f5b_extractor, function, expected): - func = getattr(b9f5b_extractor, function) - assert func() == expected +def test_dnfile_extractor(request, extractor, function, expected): + extractor_function = getattr(request.getfixturevalue(extractor), function) + assert extractor_function() == expected diff --git a/tests/test_render.py b/tests/test_render.py index c6b181e8..313b0029 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -9,14 +9,10 @@ import capa.render.result_document def test_render_number(): assert str(capa.features.insn.Number(1)) == "number(0x1)" - assert str(capa.features.insn.Number(1, bitness=capa.features.common.BITNESS_X32)) == "number/x32(0x1)" - assert str(capa.features.insn.Number(1, bitness=capa.features.common.BITNESS_X64)) == "number/x64(0x1)" def test_render_offset(): assert str(capa.features.insn.Offset(1)) == "offset(0x1)" - assert str(capa.features.insn.Offset(1, bitness=capa.features.common.BITNESS_X32)) == "offset/x32(0x1)" - assert str(capa.features.insn.Offset(1, bitness=capa.features.common.BITNESS_X64)) == "offset/x64(0x1)" def test_render_meta_attack(): diff --git a/tests/test_rules.py b/tests/test_rules.py index 90b3b02a..68afea7d 100644 --- a/tests/test_rules.py +++ b/tests/test_rules.py @@ -23,8 +23,6 @@ from capa.features.common import ( ARCH_AMD64, FORMAT_ELF, OS_WINDOWS, - BITNESS_X32, - BITNESS_X64, Arch, Format, String, @@ -110,8 +108,6 @@ def test_rule_descriptions(): - description: and description - and: - description: and description - - offset/x64: 0x50 = offset/x64 description - - offset/x64: 0x30 = offset/x64 description """ ) r = capa.rules.Rule.from_yaml(rule) @@ -531,39 +527,6 @@ def test_invalid_number(): ) -def test_number_bitness(): - r = capa.rules.Rule.from_yaml( - textwrap.dedent( - """ - rule: - meta: - name: test rule - features: - - number/x32: 2 - """ - ) - ) - assert r.evaluate({Number(2, bitness=BITNESS_X32): {1}}) == True - - assert r.evaluate({Number(2): {1}}) == False - assert r.evaluate({Number(2, bitness=BITNESS_X64): {1}}) == False - - -def test_number_bitness_symbol(): - r = capa.rules.Rule.from_yaml( - textwrap.dedent( - """ - rule: - meta: - name: test rule - features: - - number/x32: 2 = some constant - """ - ) - ) - assert r.evaluate({Number(2, bitness=BITNESS_X32, description="some constant"): {1}}) == True - - def test_offset_symbol(): rule = textwrap.dedent( """ @@ -609,39 +572,6 @@ def test_count_offset_symbol(): assert r.evaluate({Offset(0x100, description="symbol name"): {1, 2, 3}}) == True -def test_offset_bitness(): - r = capa.rules.Rule.from_yaml( - textwrap.dedent( - """ - rule: - meta: - name: test rule - features: - - offset/x32: 2 - """ - ) - ) - assert r.evaluate({Offset(2, bitness=BITNESS_X32): {1}}) == True - - assert r.evaluate({Offset(2): {1}}) == False - assert r.evaluate({Offset(2, bitness=BITNESS_X64): {1}}) == False - - -def test_offset_bitness_symbol(): - r = capa.rules.Rule.from_yaml( - textwrap.dedent( - """ - rule: - meta: - name: test rule - features: - - offset/x32: 2 = some constant - """ - ) - ) - assert r.evaluate({Offset(2, bitness=BITNESS_X32, description="some constant"): {1}}) == True - - def test_invalid_offset(): with pytest.raises(capa.rules.InvalidRule): r = capa.rules.Rule.from_yaml(