From 58e94a35cbaa384307410ef846b5965868b051e2 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Mon, 9 Jan 2023 10:51:08 +0100 Subject: [PATCH 01/30] features: string: better __str__ embedded whitespace --- CHANGELOG.md | 1 + capa/features/common.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d76f33d..031002d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,7 @@ - dotnet: address unhandled exceptions with improved type checking #1230 @mike-hunhoff - fix import-to-ida script formatting #1208 @williballenthin - render: fix verbose rendering of scopes #1263 @williballenthin +- show-features: better render strings with embedded whitespace #1267 @williballenthin ### capa explorer IDA Pro plugin - fix: display instruction items #1154 @mr-tz diff --git a/capa/features/common.py b/capa/features/common.py index 5d30f10b..782094d8 100644 --- a/capa/features/common.py +++ b/capa/features/common.py @@ -179,6 +179,9 @@ class String(Feature): def __init__(self, value: str, description=None): super().__init__(value, description=description) + def get_value_str(self) -> str: + return escape_string(self.value) + class Class(Feature): def __init__(self, value: str, description=None): @@ -232,9 +235,12 @@ class Substring(String): else: return Result(False, _MatchedSubstring(self, {}), []) + def get_value_str(self) -> str: + return escape_string(self.value) + def __str__(self): assert isinstance(self.value, str) - return "substring(%s)" % self.value + return "substring(%s)" % escape_string(self.value) class _MatchedSubstring(Substring): From e34fdfae1ab8efccc89a14b4a3d3fc78a7caa3c1 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Mon, 9 Jan 2023 13:01:41 +0100 Subject: [PATCH 02/30] mypy --- capa/features/common.py | 2 ++ capa/features/extractors/common.py | 1 + 2 files changed, 3 insertions(+) diff --git a/capa/features/common.py b/capa/features/common.py index 782094d8..cf2c02f3 100644 --- a/capa/features/common.py +++ b/capa/features/common.py @@ -180,6 +180,7 @@ class String(Feature): super().__init__(value, description=description) def get_value_str(self) -> str: + assert isinstance(self.value, str) return escape_string(self.value) @@ -236,6 +237,7 @@ class Substring(String): return Result(False, _MatchedSubstring(self, {}), []) def get_value_str(self) -> str: + assert isinstance(self.value, str) return escape_string(self.value) def __str__(self): diff --git a/capa/features/extractors/common.py b/capa/features/extractors/common.py index 585c0040..d72fcefd 100644 --- a/capa/features/extractors/common.py +++ b/capa/features/extractors/common.py @@ -9,6 +9,7 @@ import pefile import capa.features import capa.features.extractors.elf import capa.features.extractors.pefile +import capa.features.extractors.strings from capa.features.common import OS, FORMAT_PE, FORMAT_ELF, OS_WINDOWS, FORMAT_FREEZE, Arch, Format, String, Feature from capa.features.freeze import is_freeze from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress From 2a61e357de7dcba5c616bd1edd6d5f443568acb2 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Mon, 9 Jan 2023 13:08:27 +0000 Subject: [PATCH 03/30] Sync capa rules submodule --- CHANGELOG.md | 2 +- README.md | 2 +- rules | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d76f33d..1d4c0c7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ ### Breaking Changes - remove SMDA backend #1062 @williballenthin -### New Rules (48) +### New Rules (49) - collection/use-dotnet-library-sharpclipboard @johnk3r - data-manipulation/encryption/aes/use-dotnet-library-encryptdecryptutils @johnk3r diff --git a/README.md b/README.md index 699f885f..32a1628c 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa) [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases) -[![Number of rules](https://img.shields.io/badge/rules-747-blue.svg)](https://github.com/mandiant/capa-rules) +[![Number of rules](https://img.shields.io/badge/rules-746-blue.svg)](https://github.com/mandiant/capa-rules) [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) diff --git a/rules b/rules index 4c93fad4..dc81cb49 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 4c93fad4aaed8249767a9b65dee51eaa12d0ff37 +Subproject commit dc81cb4904fb4a7cbb7c6526939231e1ff7a2d14 From 85dd065f91cc48c9684809a062623cdedde44e11 Mon Sep 17 00:00:00 2001 From: Moritz Date: Mon, 9 Jan 2023 14:14:08 +0100 Subject: [PATCH 04/30] only show first lib match to reduce vverbose output noise (#1266) * only show first lib match to reduce vverbose output noise * improve rendering and wording --- CHANGELOG.md | 1 + capa/render/utils.py | 4 ++++ capa/render/vverbose.py | 17 ++++++++++++++--- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d4c0c7d..2470af58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - dotnet: emit namespace/class features for ldvirtftn/ldftn instructions #1241 @mike-hunhoff - dotnet: emit namespace/class features for type references #1242 @mike-hunhoff - dotnet: extract dotnet and pe format #1187 @mr-tz +- don't render all library rule matches in vverbose output #1174 @mr-tz ### Breaking Changes - remove SMDA backend #1062 @williballenthin diff --git a/capa/render/utils.py b/capa/render/utils.py index 2cf480c9..96abadcd 100644 --- a/capa/render/utils.py +++ b/capa/render/utils.py @@ -24,6 +24,10 @@ def bold2(s: str) -> str: return termcolor.colored(s, "green") +def warn(s: str) -> str: + return termcolor.colored(s, "yellow") + + def format_parts_id(data: Union[rd.AttackSpec, rd.MBCSpec]): """ format canonical representation of ATT&CK/MBC parts and ID diff --git a/capa/render/vverbose.py b/capa/render/vverbose.py index 5950275a..74be65da 100644 --- a/capa/render/vverbose.py +++ b/capa/render/vverbose.py @@ -285,17 +285,24 @@ def render_rules(ostream, doc: rd.ResultDocument): if rule.meta.is_subscope_rule: continue + lib_info = "" count = len(rule.matches) if count == 1: - capability = rutils.bold(rule.meta.name) + if rule.meta.lib: + lib_info = " (library rule)" + capability = "%s%s" % (rutils.bold(rule.meta.name), lib_info) else: - capability = "%s (%d matches)" % (rutils.bold(rule.meta.name), count) + if rule.meta.lib: + lib_info = ", only showing first match of library rule" + capability = "%s (%d matches%s)" % (rutils.bold(rule.meta.name), count, lib_info) ostream.writeln(capability) had_match = True rows = [] - rows.append(("namespace", rule.meta.namespace)) + if not rule.meta.lib: + # library rules should not have a namespace + rows.append(("namespace", rule.meta.namespace)) if rule.meta.maec.analysis_conclusion or rule.meta.maec.analysis_conclusion_ov: rows.append( @@ -355,6 +362,10 @@ def render_rules(ostream, doc: rd.ResultDocument): ostream.write("\n") render_match(ostream, match, indent=1) + if rule.meta.lib: + # only show first match + break + ostream.write("\n") if not had_match: From 810e2d70d3a7c1d19caad07a9146e33783cebf47 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Mon, 9 Jan 2023 13:38:25 +0000 Subject: [PATCH 05/30] Sync capa rules submodule --- CHANGELOG.md | 5 ++++- README.md | 2 +- rules | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2470af58..0f5e5614 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ ### Breaking Changes - remove SMDA backend #1062 @williballenthin -### New Rules (49) +### New Rules (52) - collection/use-dotnet-library-sharpclipboard @johnk3r - data-manipulation/encryption/aes/use-dotnet-library-encryptdecryptutils @johnk3r @@ -66,6 +66,9 @@ - compiler/nuitka/compiled-with-nuitka @williballenthin - nursery/authenticate-data-with-md5-mac william.ballenthin@mandiant.com - nursery/resolve-function-by-djb2-hash still@teamt5.org +- host-interaction/mutex/create-semaphore-on-linux @ramen0x3f +- host-interaction/mutex/lock-semaphore-on-linux @ramen0x3f +- host-interaction/mutex/unlock-semaphore-on-linux @ramen0x3f - ### Bug Fixes diff --git a/README.md b/README.md index 32a1628c..09569bf0 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa) [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases) -[![Number of rules](https://img.shields.io/badge/rules-746-blue.svg)](https://github.com/mandiant/capa-rules) +[![Number of rules](https://img.shields.io/badge/rules-749-blue.svg)](https://github.com/mandiant/capa-rules) [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) diff --git a/rules b/rules index dc81cb49..cf41d4fe 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit dc81cb4904fb4a7cbb7c6526939231e1ff7a2d14 +Subproject commit cf41d4fe4ad6e807b26a3669fa3eca364009502a From 5f31bdbb3e76ab23557114abdbfbd3034a48dc4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 14:03:44 +0000 Subject: [PATCH 06/30] build(deps): bump networkx from 2.5.1 to 3.0 Bumps [networkx](https://github.com/networkx/networkx) from 2.5.1 to 3.0. - [Release notes](https://github.com/networkx/networkx/releases) - [Commits](https://github.com/networkx/networkx/compare/networkx-2.5.1...networkx-3.0) --- updated-dependencies: - dependency-name: networkx dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a52f8f5b..46a4d35e 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ requirements = [ "ida-settings==2.1.0", "viv-utils[flirt]==0.7.7", "halo==0.0.31", - "networkx==2.5.1", + "networkx==3.0", "ruamel.yaml==0.17.21", "vivisect==1.0.8", "pefile==2022.5.30", From ef86160d884047465e2c5bda1848a895821bf21b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 14:03:48 +0000 Subject: [PATCH 07/30] build(deps): bump termcolor from 2.1.1 to 2.2.0 Bumps [termcolor](https://github.com/termcolor/termcolor) from 2.1.1 to 2.2.0. - [Release notes](https://github.com/termcolor/termcolor/releases) - [Changelog](https://github.com/termcolor/termcolor/blob/main/CHANGES.md) - [Commits](https://github.com/termcolor/termcolor/compare/2.1.1...2.2.0) --- updated-dependencies: - dependency-name: termcolor dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a52f8f5b..92225ed3 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ requirements = [ "pyyaml==6.0", "tabulate==0.9.0", "colorama==0.4.5", - "termcolor==2.1.1", + "termcolor==2.2.0", "wcwidth==0.2.5", "ida-settings==2.1.0", "viv-utils[flirt]==0.7.7", From 37a4dbf8220ec146060c20ea8510c429dfc68716 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Mon, 9 Jan 2023 15:53:03 +0000 Subject: [PATCH 08/30] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index cf41d4fe..9a514c76 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit cf41d4fe4ad6e807b26a3669fa3eca364009502a +Subproject commit 9a514c7620377ded2ede17b3c6b11276e5d8e1eb From b6fd95c7b8cb292dfbd8fd289a8527d8b68d70c5 Mon Sep 17 00:00:00 2001 From: Moritz Date: Tue, 10 Jan 2023 13:14:23 +0100 Subject: [PATCH 09/30] use positive error return code numbers (#1274) --- CHANGELOG.md | 3 ++- capa/main.py | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d59c32c9..c98cdd44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,8 @@ - don't render all library rule matches in vverbose output #1174 @mr-tz ### Breaking Changes -- remove SMDA backend #1062 @williballenthin +- remove SMDA backend #1062 @williballenthin +- error return codes are now positive numbers #1269 @mr-tz ### New Rules (52) diff --git a/capa/main.py b/capa/main.py index b3a654a4..6262b7d0 100644 --- a/capa/main.py +++ b/capa/main.py @@ -73,16 +73,16 @@ SIGNATURES_PATH_DEFAULT_STRING = "(embedded signatures)" BACKEND_VIV = "vivisect" BACKEND_DOTNET = "dotnet" -E_MISSING_RULES = -10 -E_MISSING_FILE = -11 -E_INVALID_RULE = -12 -E_CORRUPT_FILE = -13 -E_FILE_LIMITATION = -14 -E_INVALID_SIG = -15 -E_INVALID_FILE_TYPE = -16 -E_INVALID_FILE_ARCH = -17 -E_INVALID_FILE_OS = -18 -E_UNSUPPORTED_IDA_VERSION = -19 +E_MISSING_RULES = 10 +E_MISSING_FILE = 11 +E_INVALID_RULE = 12 +E_CORRUPT_FILE = 13 +E_FILE_LIMITATION = 14 +E_INVALID_SIG = 15 +E_INVALID_FILE_TYPE = 16 +E_INVALID_FILE_ARCH = 17 +E_INVALID_FILE_OS = 18 +E_UNSUPPORTED_IDA_VERSION = 19 logger = logging.getLogger("capa") From 05879dc02a6d4f4a3971f5231e8fe746bc1715e8 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Wed, 11 Jan 2023 10:49:04 +0100 Subject: [PATCH 10/30] Revert "build(deps): bump networkx from 2.5.1 to 3.0" --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 010c227a..92225ed3 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ requirements = [ "ida-settings==2.1.0", "viv-utils[flirt]==0.7.7", "halo==0.0.31", - "networkx==3.0", + "networkx==2.5.1", "ruamel.yaml==0.17.21", "vivisect==1.0.8", "pefile==2022.5.30", From 5fb9de775f627dd9f1c94e8fd67a2338c87b4627 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Wed, 11 Jan 2023 10:50:55 +0100 Subject: [PATCH 11/30] setup: document networkx dep version pin --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 92225ed3..d94dbf38 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ requirements = [ "ida-settings==2.1.0", "viv-utils[flirt]==0.7.7", "halo==0.0.31", - "networkx==2.5.1", + "networkx==2.5.1", # newer versions no longer support py3.7. "ruamel.yaml==0.17.21", "vivisect==1.0.8", "pefile==2022.5.30", From a273ad31d4e5fc6d556d99e10a1b9915a373e2a8 Mon Sep 17 00:00:00 2001 From: Moritz Date: Wed, 11 Jan 2023 17:17:04 +0100 Subject: [PATCH 12/30] make read consistent with file object behavior (#1281) --- CHANGELOG.md | 1 + capa/ida/helpers.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c98cdd44..86be5431 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,6 +94,7 @@ - generator: refactor caching and matching #1251 @mike-hunhoff - fix: improve exception handling to prevent IDA from locking up when errors occur #1262 @mike-hunhoff - verify rule metadata using Pydantic #1167 @mr-tz +- extractor: make read consistent with file object behavior #1254 @mr-tz ### Development diff --git a/capa/ida/helpers.py b/capa/ida/helpers.py index d1ef3093..2d12e931 100644 --- a/capa/ida/helpers.py +++ b/capa/ida/helpers.py @@ -181,11 +181,13 @@ class IDAIO: def read(self, size): ea = ida_loader.get_fileregion_ea(self.offset) if ea == idc.BADADDR: - # best guess, such as if file is mapped at address 0x0. - ea = self.offset + logger.debug("cannot read 0x%x bytes at 0x%x (ea: BADADDR)", size, self.offset) + return b"" logger.debug("reading 0x%x bytes at 0x%x (ea: 0x%x)", size, self.offset, ea) - return ida_bytes.get_bytes(ea, size) + + # get_bytes returns None on error, for consistency with read always return bytes + return ida_bytes.get_bytes(ea, size) or b"" def close(self): return From 4b9a844c92a200438b116d2886602eeac383e24c Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Thu, 12 Jan 2023 10:38:26 +0100 Subject: [PATCH 13/30] rules: catch invalid YAML exception --- capa/rules.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/capa/rules.py b/capa/rules.py index c4d2ad77..0eece7f9 100644 --- a/capa/rules.py +++ b/capa/rules.py @@ -27,6 +27,7 @@ except ImportError: from typing import Any, Set, Dict, List, Tuple, Union, Iterator import yaml +import yaml.parser import pydantic import ruamel.yaml @@ -836,6 +837,8 @@ class Rule: raise InvalidRuleWithPath(path, str(e)) from e except pydantic.ValidationError as e: raise InvalidRuleWithPath(path, str(e)) from e + except yaml.parser.ParserError as e: + raise InvalidRuleWithPath(path, str(e)) from e def to_yaml(self) -> str: # reformat the yaml document with a common style. From 9a9f72f07a4c28982c23be093780dd3a0e45ba83 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Thu, 12 Jan 2023 10:38:52 +0100 Subject: [PATCH 14/30] pep8 --- capa/rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capa/rules.py b/capa/rules.py index 0eece7f9..ac344334 100644 --- a/capa/rules.py +++ b/capa/rules.py @@ -27,9 +27,9 @@ except ImportError: from typing import Any, Set, Dict, List, Tuple, Union, Iterator import yaml -import yaml.parser import pydantic import ruamel.yaml +import yaml.parser import capa.perf import capa.engine as ceng From ac36b9d32869945e150609cf1bbd0d5167c5bf35 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Thu, 12 Jan 2023 10:39:36 +0100 Subject: [PATCH 15/30] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d76f33d..450cf2b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,7 @@ - dotnet: address unhandled exceptions with improved type checking #1230 @mike-hunhoff - fix import-to-ida script formatting #1208 @williballenthin - render: fix verbose rendering of scopes #1263 @williballenthin +- rules: better detect invalid rules #1282 @williballenthin ### capa explorer IDA Pro plugin - fix: display instruction items #1154 @mr-tz From 3571f3557820256b768c51ea225c1e95a4f64710 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Thu, 12 Jan 2023 11:57:41 +0000 Subject: [PATCH 16/30] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index 9a514c76..b99c2471 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 9a514c7620377ded2ede17b3c6b11276e5d8e1eb +Subproject commit b99c247153fb6fc15d85f4fad8b90cc8ce98191d From 12f8588c031cea35480ee8c7274e9d9c91096287 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Thu, 12 Jan 2023 12:59:01 +0000 Subject: [PATCH 17/30] Sync capa-testfiles submodule --- tests/data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data b/tests/data index b06307a6..45e1d0cc 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit b06307a693e3b91fe72d197acdeb0b273a720b0c +Subproject commit 45e1d0ccfa7c3a02853e035d3d5c249178c83c2e From 2ab057a24d08a06bd1b32b46068b74b50f19b087 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Thu, 12 Jan 2023 13:15:35 +0000 Subject: [PATCH 18/30] Sync capa rules submodule --- CHANGELOG.md | 4 +++- README.md | 2 +- rules | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6804286..59330ffa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ - remove SMDA backend #1062 @williballenthin - error return codes are now positive numbers #1269 @mr-tz -### New Rules (52) +### New Rules (54) - collection/use-dotnet-library-sharpclipboard @johnk3r - data-manipulation/encryption/aes/use-dotnet-library-encryptdecryptutils @johnk3r @@ -70,6 +70,8 @@ - host-interaction/mutex/create-semaphore-on-linux @ramen0x3f - host-interaction/mutex/lock-semaphore-on-linux @ramen0x3f - host-interaction/mutex/unlock-semaphore-on-linux @ramen0x3f +- data-manipulation/hashing/sha384/hash-data-using-sha384 william.ballenthin@mandiant.com +- data-manipulation/hashing/sha512/hash-data-using-sha512 william.ballenthin@mandiant.com - ### Bug Fixes diff --git a/README.md b/README.md index 09569bf0..29dfb6be 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa) [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases) -[![Number of rules](https://img.shields.io/badge/rules-749-blue.svg)](https://github.com/mandiant/capa-rules) +[![Number of rules](https://img.shields.io/badge/rules-751-blue.svg)](https://github.com/mandiant/capa-rules) [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) diff --git a/rules b/rules index b99c2471..8e5faa50 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit b99c247153fb6fc15d85f4fad8b90cc8ce98191d +Subproject commit 8e5faa50104c068689a3271d97c1f6be428cf433 From cb1ef965d0e7d0b38475ed1f1529b3f979dd62d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jan 2023 14:03:54 +0000 Subject: [PATCH 19/30] build(deps): bump wcwidth from 0.2.5 to 0.2.6 Bumps [wcwidth](https://github.com/jquast/wcwidth) from 0.2.5 to 0.2.6. - [Release notes](https://github.com/jquast/wcwidth/releases) - [Commits](https://github.com/jquast/wcwidth/compare/0.2.5...0.2.6) --- updated-dependencies: - dependency-name: wcwidth dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d94dbf38..5f33582f 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ requirements = [ "tabulate==0.9.0", "colorama==0.4.5", "termcolor==2.2.0", - "wcwidth==0.2.5", + "wcwidth==0.2.6", "ida-settings==2.1.0", "viv-utils[flirt]==0.7.7", "halo==0.0.31", From 35e6df6f6b9be33ec10cea7f5b1ce38db879bca6 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Wed, 18 Jan 2023 15:10:43 +0000 Subject: [PATCH 20/30] Sync capa rules submodule --- CHANGELOG.md | 4 +++- README.md | 2 +- rules | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59330ffa..8b7e0984 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ - remove SMDA backend #1062 @williballenthin - error return codes are now positive numbers #1269 @mr-tz -### New Rules (54) +### New Rules (56) - collection/use-dotnet-library-sharpclipboard @johnk3r - data-manipulation/encryption/aes/use-dotnet-library-encryptdecryptutils @johnk3r @@ -72,6 +72,8 @@ - host-interaction/mutex/unlock-semaphore-on-linux @ramen0x3f - data-manipulation/hashing/sha384/hash-data-using-sha384 william.ballenthin@mandiant.com - data-manipulation/hashing/sha512/hash-data-using-sha512 william.ballenthin@mandiant.com +- nursery/decode-data-using-url-encoding michael.hunhoff@mandiant.com +- nursery/manipulate-user-privileges michael.hunhoff@mandiant.com - ### Bug Fixes diff --git a/README.md b/README.md index 29dfb6be..6fb2320b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa) [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases) -[![Number of rules](https://img.shields.io/badge/rules-751-blue.svg)](https://github.com/mandiant/capa-rules) +[![Number of rules](https://img.shields.io/badge/rules-753-blue.svg)](https://github.com/mandiant/capa-rules) [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) diff --git a/rules b/rules index 8e5faa50..0218bd52 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 8e5faa50104c068689a3271d97c1f6be428cf433 +Subproject commit 0218bd52744ea99e3c4e9bf646f544adad2ee795 From b8de9625ee6b568c641b8d40e9a2bbed8e63703a Mon Sep 17 00:00:00 2001 From: Moritz Date: Thu, 19 Jan 2023 11:56:13 +0100 Subject: [PATCH 21/30] fix: don't extract invalid calls from features (#1285) --- CHANGELOG.md | 1 + capa/features/extractors/viv/insn.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b7e0984..0c7d6455 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,7 @@ - render: fix verbose rendering of scopes #1263 @williballenthin - rules: better detect invalid rules #1282 @williballenthin - show-features: better render strings with embedded whitespace #1267 @williballenthin +- extractor: guard against invalid "calls from" features #1177 @mr-tz ### capa explorer IDA Pro plugin - fix: display instruction items #1154 @mr-tz diff --git a/capa/features/extractors/viv/insn.py b/capa/features/extractors/viv/insn.py index 738c69a7..ca05bee5 100644 --- a/capa/features/extractors/viv/insn.py +++ b/capa/features/extractors/viv/insn.py @@ -493,7 +493,8 @@ def extract_function_calls_from(fh: FunctionHandle, bb, ih: InsnHandle) -> Itera if isinstance(insn.opers[0], envi.archs.i386.disasm.i386ImmMemOper): oper = insn.opers[0] target = oper.getOperAddr(insn) - yield Characteristic("calls from"), AbsoluteVirtualAddress(target) + if target >= 0: + yield Characteristic("calls from"), AbsoluteVirtualAddress(target) # call via thunk on x86, # see 9324d1a8ae37a36ae560c37448c9705a at 0x407985 @@ -509,7 +510,8 @@ def extract_function_calls_from(fh: FunctionHandle, bb, ih: InsnHandle) -> Itera elif isinstance(insn.opers[0], envi.archs.amd64.disasm.Amd64RipRelOper): op = insn.opers[0] target = op.getOperAddr(insn) - yield Characteristic("calls from"), AbsoluteVirtualAddress(target) + if target >= 0: + yield Characteristic("calls from"), AbsoluteVirtualAddress(target) if target and target == f.va: # if we found a jump target and it's the function address From efb4c9d540b26835ec4cce6323d5522eadd0d497 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Thu, 19 Jan 2023 10:58:26 +0000 Subject: [PATCH 22/30] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index 0218bd52..f0e0befa 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 0218bd52744ea99e3c4e9bf646f544adad2ee795 +Subproject commit f0e0befa154a4e7abeb6e511c914e347f4b0bc22 From 2b07ec925cd66bd6059f118d6e7349238cde9d69 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Thu, 19 Jan 2023 11:23:42 +0000 Subject: [PATCH 23/30] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index f0e0befa..28e9c066 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit f0e0befa154a4e7abeb6e511c914e347f4b0bc22 +Subproject commit 28e9c066ad99c49f87bcc66cfc98b486955ab909 From 5513d4ca431ff5a74d6b3f4375d0f5a5c0917a77 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Thu, 19 Jan 2023 13:02:53 +0100 Subject: [PATCH 24/30] viv: insn: string: handle viv bug around substrings (#1273) * viv: insn: string: handle viv bug around substrings closes #1271 * use minimum string length 4 * update overlapping string test and fixup vivisect elf analysis missing function Co-authored-by: Moritz --- CHANGELOG.md | 1 + capa/features/extractors/viv/insn.py | 16 ++++++++++++---- tests/fixtures.py | 7 +++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c7d6455..d88b1f4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,7 @@ - render: fix verbose rendering of scopes #1263 @williballenthin - rules: better detect invalid rules #1282 @williballenthin - show-features: better render strings with embedded whitespace #1267 @williballenthin +- handle vivisect bug around strings at instruction level, use min length 4 #1271 @williballenthin @mr-tz - extractor: guard against invalid "calls from" features #1177 @mr-tz ### capa explorer IDA Pro plugin diff --git a/capa/features/extractors/viv/insn.py b/capa/features/extractors/viv/insn.py index ca05bee5..1b36e348 100644 --- a/capa/features/extractors/viv/insn.py +++ b/capa/features/extractors/viv/insn.py @@ -281,7 +281,12 @@ def read_string(vw, offset: int) -> str: pass else: if alen > 0: - return read_memory(vw, offset, alen).decode("utf-8") + buf = read_memory(vw, offset, alen) + if b"\x00" in buf: + # account for bug #1271. + # remove when vivisect is fixed. + buf = buf.partition(b"\x00")[0] + return buf.decode("utf-8") try: ulen = vw.detectUnicode(offset) @@ -300,7 +305,9 @@ def read_string(vw, offset: int) -> str: # vivisect seems to mis-detect the end unicode strings # off by two, too short ulen += 2 - return read_memory(vw, offset, ulen).decode("utf-16") + # partition to account for bug #1271. + # remove when vivisect is fixed. + return read_memory(vw, offset, ulen).decode("utf-16").partition("\x00")[0] raise ValueError("not a string", offset) @@ -665,11 +672,12 @@ def extract_op_string_features( for v in derefs(f.vw, v): try: - s = read_string(f.vw, v) + s = read_string(f.vw, v).rstrip("\x00") except ValueError: continue else: - yield String(s.rstrip("\x00")), ih.address + if len(s) > 4: + yield String(s), ih.address def extract_operand_features(f: FunctionHandle, bb, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]: diff --git a/tests/fixtures.py b/tests/fixtures.py index f6686c4b..734f64a4 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -118,6 +118,9 @@ def fixup_viv(path, extractor): if "3b13b" in path: # vivisect only recognizes calling thunk function at 0x10001573 extractor.vw.makeFunction(0x10006860) + if "294b8d" in path: + # see vivisect/#561 + extractor.vw.makeFunction(0x404970) @lru_cache(maxsize=1) @@ -277,6 +280,8 @@ def get_data_path_by_name(name): return os.path.join(CD, "data", "b5f0524e69b3a3cf636c7ac366ca57bf5e3a8fdc8a9f01caf196c611a7918a87.elf_") elif name.startswith("bf7a9c"): return os.path.join(CD, "data", "bf7a9c8bdfa6d47e01ad2b056264acc3fd90cf43fe0ed8deec93ab46b47d76cb.elf_") + elif name.startswith("294b8d"): + return os.path.join(CD, "data", "294b8db1f2702b60fb2e42fdc50c2cee6a5046112da9a5703a548a4fa50477bc.elf_") else: raise ValueError("unexpected sample fixture: %s" % name) @@ -627,6 +632,8 @@ FEATURE_PRESENCE_TESTS = sorted( ("mimikatz", "function=0x40105D", capa.features.common.String("ACR > "), True), ("mimikatz", "function=0x40105D", capa.features.common.String("nope"), False), ("773290...", "function=0x140001140", capa.features.common.String(r"%s:\\OfficePackagesForWDAG"), True), + # overlapping string, see #1271 + ("294b8d...", "function=0x404970,bb=0x404970,insn=0x40499F", capa.features.common.String("\r\n\x00:ht"), False), # insn/regex ("pma16-01", "function=0x4021B0", capa.features.common.Regex("HTTP/1.0"), True), ("pma16-01", "function=0x402F40", capa.features.common.Regex("www.practicalmalwareanalysis.com"), True), From 26662e99de08ccb1399c328359bdb074a4bb0094 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Thu, 19 Jan 2023 12:11:19 +0000 Subject: [PATCH 25/30] Sync capa rules submodule --- CHANGELOG.md | 3 ++- README.md | 2 +- rules | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d88b1f4d..7963661c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ - remove SMDA backend #1062 @williballenthin - error return codes are now positive numbers #1269 @mr-tz -### New Rules (56) +### New Rules (57) - collection/use-dotnet-library-sharpclipboard @johnk3r - data-manipulation/encryption/aes/use-dotnet-library-encryptdecryptutils @johnk3r @@ -74,6 +74,7 @@ - data-manipulation/hashing/sha512/hash-data-using-sha512 william.ballenthin@mandiant.com - nursery/decode-data-using-url-encoding michael.hunhoff@mandiant.com - nursery/manipulate-user-privileges michael.hunhoff@mandiant.com +- lib/get-os-version @mr-tz - ### Bug Fixes diff --git a/README.md b/README.md index 6fb2320b..14d1a8a5 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa) [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases) -[![Number of rules](https://img.shields.io/badge/rules-753-blue.svg)](https://github.com/mandiant/capa-rules) +[![Number of rules](https://img.shields.io/badge/rules-754-blue.svg)](https://github.com/mandiant/capa-rules) [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) diff --git a/rules b/rules index 28e9c066..6e5d63b4 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 28e9c066ad99c49f87bcc66cfc98b486955ab909 +Subproject commit 6e5d63b4b70ff270a3e6615c05590ea6059642d7 From 0fb3be359f62fba4e8df3007628ba8c6274ad623 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Thu, 19 Jan 2023 12:12:41 +0000 Subject: [PATCH 26/30] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index 6e5d63b4..2a997f5d 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 6e5d63b4b70ff270a3e6615c05590ea6059642d7 +Subproject commit 2a997f5d0ceb9acbdd149b64c3eafa6abbe8adfa From fa0ddba43686c78849fe3ac1c41b20f3c96c8829 Mon Sep 17 00:00:00 2001 From: Moritz Date: Thu, 19 Jan 2023 13:31:00 +0100 Subject: [PATCH 27/30] add format to global features and code refactors (#1284) * refactor: get format handling * add format to global features --- CHANGELOG.md | 1 + capa/features/extractors/dnfile/extractor.py | 3 ++- capa/features/extractors/ida/extractor.py | 1 + capa/features/extractors/viv/extractor.py | 1 + capa/helpers.py | 8 ++++++- capa/main.py | 22 +++++++++----------- scripts/lint.py | 6 +----- scripts/show-capabilities-by-function.py | 2 +- tests/fixtures.py | 8 +++++++ 9 files changed, 32 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7963661c..3ba52ca1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,6 +91,7 @@ - show-features: better render strings with embedded whitespace #1267 @williballenthin - handle vivisect bug around strings at instruction level, use min length 4 #1271 @williballenthin @mr-tz - extractor: guard against invalid "calls from" features #1177 @mr-tz +- extractor: add format to global features #1258 @mr-tz ### capa explorer IDA Pro plugin - fix: display instruction items #1154 @mr-tz diff --git a/capa/features/extractors/dnfile/extractor.py b/capa/features/extractors/dnfile/extractor.py index bd4b9c9e..9d7c6c4c 100644 --- a/capa/features/extractors/dnfile/extractor.py +++ b/capa/features/extractors/dnfile/extractor.py @@ -8,13 +8,13 @@ from __future__ import annotations -from enum import Enum from typing import Dict, List, Tuple, Union, Iterator, Optional import dnfile from dncil.cil.opcode import OpCodes import capa.features.extractors +import capa.features.extractors.dotnetfile import capa.features.extractors.dnfile.file import capa.features.extractors.dnfile.insn import capa.features.extractors.dnfile.function @@ -78,6 +78,7 @@ class DnfileFeatureExtractor(FeatureExtractor): # pre-compute these because we'll yield them at *every* scope. self.global_features: List[Tuple[Feature, Address]] = [] + self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_format()) self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_os(pe=self.pe)) self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_arch(pe=self.pe)) diff --git a/capa/features/extractors/ida/extractor.py b/capa/features/extractors/ida/extractor.py index 1a587fa6..0d44ba9e 100644 --- a/capa/features/extractors/ida/extractor.py +++ b/capa/features/extractors/ida/extractor.py @@ -25,6 +25,7 @@ class IdaFeatureExtractor(FeatureExtractor): def __init__(self): super().__init__() self.global_features: List[Tuple[Feature, Address]] = [] + self.global_features.extend(capa.features.extractors.ida.file.extract_file_format()) self.global_features.extend(capa.features.extractors.ida.global_.extract_os()) self.global_features.extend(capa.features.extractors.ida.global_.extract_arch()) diff --git a/capa/features/extractors/viv/extractor.py b/capa/features/extractors/viv/extractor.py index 53683f66..e2d0fb1e 100644 --- a/capa/features/extractors/viv/extractor.py +++ b/capa/features/extractors/viv/extractor.py @@ -34,6 +34,7 @@ class VivisectFeatureExtractor(FeatureExtractor): # pre-compute these because we'll yield them at *every* scope. self.global_features: List[Tuple[Feature, Address]] = [] + self.global_features.extend(capa.features.extractors.viv.file.extract_file_format(self.buf)) self.global_features.extend(capa.features.extractors.common.extract_os(self.buf)) self.global_features.extend(capa.features.extractors.viv.global_.extract_arch(self.vw)) diff --git a/capa/helpers.py b/capa/helpers.py index 2e44fc6c..a2edc812 100644 --- a/capa/helpers.py +++ b/capa/helpers.py @@ -10,7 +10,7 @@ import logging from typing import NoReturn from capa.exceptions import UnsupportedFormatError -from capa.features.common import FORMAT_SC32, FORMAT_SC64, FORMAT_UNKNOWN +from capa.features.common import FORMAT_PE, FORMAT_SC32, FORMAT_SC64, FORMAT_DOTNET, FORMAT_UNKNOWN, Format EXTENSIONS_SHELLCODE_32 = ("sc32", "raw32") EXTENSIONS_SHELLCODE_64 = ("sc64", "raw64") @@ -68,11 +68,17 @@ def get_auto_format(path: str) -> str: def get_format(sample: str) -> str: # imported locally to avoid import cycle from capa.features.extractors.common import extract_format + from capa.features.extractors.dnfile_ import DnfileFeatureExtractor with open(sample, "rb") as f: buf = f.read() for feature, _ in extract_format(buf): + if feature == Format(FORMAT_PE): + dnfile_extractor = DnfileFeatureExtractor(sample) + if dnfile_extractor.is_dotnet_file(): + feature = Format(FORMAT_DOTNET) + assert isinstance(feature.value, str) return feature.value diff --git a/capa/main.py b/capa/main.py index 6262b7d0..1741fad9 100644 --- a/capa/main.py +++ b/capa/main.py @@ -20,7 +20,7 @@ import textwrap import itertools import contextlib import collections -from typing import Any, Dict, List, Tuple, Optional +from typing import Any, Dict, List, Tuple import halo import tqdm @@ -535,12 +535,12 @@ def get_extractor( def get_file_extractors(sample: str, format_: str) -> List[FeatureExtractor]: file_extractors: List[FeatureExtractor] = list() - if format_ == capa.features.extractors.common.FORMAT_PE: + if format_ == FORMAT_PE: file_extractors.append(capa.features.extractors.pefile.PefileFeatureExtractor(sample)) - dnfile_extractor = capa.features.extractors.dnfile_.DnfileFeatureExtractor(sample) - if dnfile_extractor.is_dotnet_file(): - file_extractors.append(dnfile_extractor) + elif format_ == FORMAT_DOTNET: + file_extractors.append(capa.features.extractors.pefile.PefileFeatureExtractor(sample)) + file_extractors.append(capa.features.extractors.dnfile_.DnfileFeatureExtractor(sample)) elif format_ == capa.features.extractors.common.FORMAT_ELF: file_extractors.append(capa.features.extractors.elffile.ElfFeatureExtractor(sample)) @@ -646,7 +646,6 @@ def collect_metadata( sample_path: str, rules_path: List[str], extractor: capa.features.extractors.base_extractor.FeatureExtractor, - format_: Optional[str] = None, ): md5 = hashlib.md5() sha1 = hashlib.sha1() @@ -662,8 +661,7 @@ def collect_metadata( if rules_path != [RULES_PATH_DEFAULT_STRING]: rules_path = [os.path.abspath(os.path.normpath(r)) for r in rules_path] - if format_ is None: - format_ = get_format(sample_path) + format_ = get_format(sample_path) arch = get_arch(sample_path) os_ = get_os(sample_path) @@ -996,6 +994,9 @@ def main(argv=None): if format_ == FORMAT_AUTO: try: format_ = get_auto_format(args.sample) + except PEFormatError as e: + logger.error("Input file '%s' is not a valid PE file: %s", args.sample, str(e)) + return E_CORRUPT_FILE except UnsupportedFormatError: log_unsupported_format_error() return E_INVALID_FILE_TYPE @@ -1058,9 +1059,6 @@ def main(argv=None): logger.error("Input file '%s' is not a valid ELF file: %s", args.sample, str(e)) return E_CORRUPT_FILE - if isinstance(file_extractor, capa.features.extractors.dnfile_.DnfileFeatureExtractor): - format_ = FORMAT_DOTNET - # file limitations that rely on non-file scope won't be detected here. # nor on FunctionName features, because pefile doesn't support this. if has_file_limitation(rules, pure_file_capabilities): @@ -1100,7 +1098,7 @@ def main(argv=None): log_unsupported_os_error() return E_INVALID_FILE_OS - meta = collect_metadata(argv, args.sample, args.rules, extractor, format_=format_) + meta = collect_metadata(argv, args.sample, args.rules, extractor) capabilities, counts = find_capabilities(rules, extractor, disable_progress=args.quiet) meta["analysis"].update(counts) diff --git a/scripts/lint.py b/scripts/lint.py index ed6e8b17..767cd0a8 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -307,11 +307,7 @@ def get_sample_capabilities(ctx: Context, path: Path) -> Set[str]: elif nice_path.endswith(capa.helpers.EXTENSIONS_SHELLCODE_64): format_ = "sc64" else: - format_ = "auto" - if not nice_path.endswith(capa.helpers.EXTENSIONS_ELF): - dnfile_extractor = capa.features.extractors.dnfile_.DnfileFeatureExtractor(nice_path) - if dnfile_extractor.is_dotnet_file(): - format_ = FORMAT_DOTNET + format_ = capa.main.get_auto_format(nice_path) logger.debug("analyzing sample: %s", nice_path) extractor = capa.main.get_extractor(nice_path, format_, "", DEFAULT_SIGNATURES, False, disable_progress=True) diff --git a/scripts/show-capabilities-by-function.py b/scripts/show-capabilities-by-function.py index 7f0b13b6..f4303fef 100644 --- a/scripts/show-capabilities-by-function.py +++ b/scripts/show-capabilities-by-function.py @@ -175,7 +175,7 @@ def main(argv=None): capa.helpers.log_unsupported_runtime_error() return -1 - meta = capa.main.collect_metadata(argv, args.sample, args.rules, extractor, format_=format_) + meta = capa.main.collect_metadata(argv, args.sample, args.rules, extractor) capabilities, counts = capa.main.find_capabilities(rules, extractor) meta["analysis"].update(counts) meta["analysis"]["layout"] = capa.main.compute_layout(rules, extractor, capabilities) diff --git a/tests/fixtures.py b/tests/fixtures.py index 734f64a4..ae336f56 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -689,14 +689,22 @@ FEATURE_PRESENCE_TESTS = sorted( # os & format & arch ("pma16-01", "file", OS(OS_WINDOWS), True), ("pma16-01", "file", OS(OS_LINUX), False), + ("mimikatz", "file", OS(OS_WINDOWS), True), ("pma16-01", "function=0x404356", OS(OS_WINDOWS), True), ("pma16-01", "function=0x404356,bb=0x4043B9", OS(OS_WINDOWS), True), + ("mimikatz", "function=0x40105D", OS(OS_WINDOWS), True), ("pma16-01", "file", Arch(ARCH_I386), True), ("pma16-01", "file", Arch(ARCH_AMD64), False), + ("mimikatz", "file", Arch(ARCH_I386), True), ("pma16-01", "function=0x404356", Arch(ARCH_I386), True), ("pma16-01", "function=0x404356,bb=0x4043B9", Arch(ARCH_I386), True), + ("mimikatz", "function=0x40105D", Arch(ARCH_I386), True), ("pma16-01", "file", Format(FORMAT_PE), True), ("pma16-01", "file", Format(FORMAT_ELF), False), + ("mimikatz", "file", Format(FORMAT_PE), True), + # format is also a global feature + ("pma16-01", "function=0x404356", Format(FORMAT_PE), True), + ("mimikatz", "function=0x456BB9", Format(FORMAT_PE), True), # elf support ("7351f.elf", "file", OS(OS_LINUX), True), ("7351f.elf", "file", OS(OS_WINDOWS), False), From b10196cdac6b4794de221b46fe2336098c3cf5cd Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Fri, 20 Jan 2023 11:12:04 +0000 Subject: [PATCH 28/30] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index 2a997f5d..54396145 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 2a997f5d0ceb9acbdd149b64c3eafa6abbe8adfa +Subproject commit 543961457214f201346960072d530ffc400ca94c From 67717761bda73220f19f5439bc069d47d401724b Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Fri, 20 Jan 2023 12:15:02 +0000 Subject: [PATCH 29/30] Sync capa rules submodule --- CHANGELOG.md | 4 +++- README.md | 2 +- rules | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ba52ca1..2cb4ba91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ - remove SMDA backend #1062 @williballenthin - error return codes are now positive numbers #1269 @mr-tz -### New Rules (57) +### New Rules (60) - collection/use-dotnet-library-sharpclipboard @johnk3r - data-manipulation/encryption/aes/use-dotnet-library-encryptdecryptutils @johnk3r @@ -75,6 +75,8 @@ - nursery/decode-data-using-url-encoding michael.hunhoff@mandiant.com - nursery/manipulate-user-privileges michael.hunhoff@mandiant.com - lib/get-os-version @mr-tz +- nursery/decrypt-data-using-tea william.ballenthin@mandiant.com +- nursery/encrypt-data-using-tea william.ballenthin@mandiant.com - ### Bug Fixes diff --git a/README.md b/README.md index 14d1a8a5..f980eab1 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa) [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases) -[![Number of rules](https://img.shields.io/badge/rules-754-blue.svg)](https://github.com/mandiant/capa-rules) +[![Number of rules](https://img.shields.io/badge/rules-757-blue.svg)](https://github.com/mandiant/capa-rules) [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) diff --git a/rules b/rules index 54396145..650a05af 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 543961457214f201346960072d530ffc400ca94c +Subproject commit 650a05af504aad79b4932b815261c5642611530a From 49c18bd83db1e5c1d3288a739456eaf12a92362e Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Fri, 20 Jan 2023 12:15:23 +0000 Subject: [PATCH 30/30] Sync capa rules submodule --- CHANGELOG.md | 3 ++- rules | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cb4ba91..96cbdcc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ - remove SMDA backend #1062 @williballenthin - error return codes are now positive numbers #1269 @mr-tz -### New Rules (60) +### New Rules (61) - collection/use-dotnet-library-sharpclipboard @johnk3r - data-manipulation/encryption/aes/use-dotnet-library-encryptdecryptutils @johnk3r @@ -77,6 +77,7 @@ - lib/get-os-version @mr-tz - nursery/decrypt-data-using-tea william.ballenthin@mandiant.com - nursery/encrypt-data-using-tea william.ballenthin@mandiant.com +- nursery/hash-data-using-whirlpool william.ballenthin@mandiant.com - ### Bug Fixes diff --git a/rules b/rules index 650a05af..09956929 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 650a05af504aad79b4932b815261c5642611530a +Subproject commit 09956929b5e38a2397cab767da0dc3a0e0513a90