From 7c4d4d02a81701548d58fe85721c8206c658dd2f Mon Sep 17 00:00:00 2001 From: lakshit verma Date: Wed, 19 Aug 2026 04:50:45 +0530 Subject: [PATCH] fix(tags): detect unpulled Git LFS pointers in all loaders Only gp.StringHashDatabase checked for a Git LFS pointer; the oss, winapi, and expert loaders failed with a confusing BadGzipFile/msgspec error on a fresh clone without git lfs pull. A shared ensure_not_lfs_pointer() helper now raises the actionable message, and gp's loader is refactored onto it. --- floss/tags/__init__.py | 20 +++++++++++++ floss/tags/expert.py | 3 +- floss/tags/gp.py | 6 ++-- floss/tags/oss.py | 3 +- floss/tags/winapi.py | 4 ++- tests/test_tags_lfs.py | 66 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 95 insertions(+), 7 deletions(-) create mode 100644 tests/test_tags_lfs.py diff --git a/floss/tags/__init__.py b/floss/tags/__init__.py index f26a6c1..7a22ef1 100644 --- a/floss/tags/__init__.py +++ b/floss/tags/__init__.py @@ -29,6 +29,26 @@ def data_root() -> pathlib.Path: return pathlib.Path(__file__).resolve().parent / "data" +# the first line of a Git LFS pointer file; used to detect unpulled databases +LFS_POINTER_PREFIX = b"version https://git-lfs.github.com/" + + +def ensure_not_lfs_pointer(path: pathlib.Path) -> None: + """Raise a clear error when a tag database file is an unpulled Git LFS pointer. + + Without ``git lfs pull`` the LFS-tracked database files are tiny text + pointers, which the loaders otherwise fail on with confusing gzip/msgspec + errors. + """ + try: + with path.open("rb") as f: + head = f.read(len(LFS_POINTER_PREFIX)) + except OSError: + return + if head == LFS_POINTER_PREFIX: + raise ValueError(f"Git LFS pointer detected in {path.name}; please run `git lfs pull`") + + from floss.tags.engine import ( Tagger, load_databases, diff --git a/floss/tags/expert.py b/floss/tags/expert.py index 3159e06..8efe790 100644 --- a/floss/tags/expert.py +++ b/floss/tags/expert.py @@ -27,7 +27,7 @@ from dataclasses import dataclass import msgspec -from floss.tags import data_root +from floss.tags import data_root, ensure_not_lfs_pointer class ExpertRule(msgspec.Struct): @@ -78,6 +78,7 @@ class ExpertStringDatabase: substring_rules: List[ExpertRule] = [] regex_rules: List[Tuple[ExpertRule, re.Pattern]] = [] + ensure_not_lfs_pointer(path) decoder = msgspec.json.Decoder(type=ExpertRule) buf = path.read_bytes() for line in buf.split(b"\n"): diff --git a/floss/tags/gp.py b/floss/tags/gp.py index a671cc2..a145dff 100644 --- a/floss/tags/gp.py +++ b/floss/tags/gp.py @@ -28,7 +28,7 @@ from dataclasses import dataclass import msgspec -from floss.tags import data_root +from floss.tags import data_root, ensure_not_lfs_pointer Encoding = Literal["ascii"] | Literal["utf-16le"] | Literal["unknown"] # header | gap | overlay @@ -135,11 +135,9 @@ class StringHashDatabase: def from_file(cls, path: pathlib.Path) -> "StringHashDatabase": string_hashes: Set[bytes] = set() + ensure_not_lfs_pointer(path) buf = path.read_bytes() - if buf.startswith(b"version https://git-lfs.github.com/"): - raise ValueError(f"Git LFS pointer detected in {path.name}; please run `git lfs pull`") - for i in range(0, len(buf), 8): string_hashes.add(buf[i : i + 8]) diff --git a/floss/tags/oss.py b/floss/tags/oss.py index 74b6f90..7f75b98 100644 --- a/floss/tags/oss.py +++ b/floss/tags/oss.py @@ -25,7 +25,7 @@ from dataclasses import dataclass import msgspec -from floss.tags import data_root +from floss.tags import data_root, ensure_not_lfs_pointer class OpenSourceString(msgspec.Struct): @@ -47,6 +47,7 @@ class OpenSourceStringDatabase: @classmethod def from_file(cls, path: pathlib.Path) -> "OpenSourceStringDatabase": metadata_by_string: Dict[str, OpenSourceString] = {} + ensure_not_lfs_pointer(path) decoder = msgspec.json.Decoder(type=OpenSourceString) for line in gzip.decompress(path.read_bytes()).split(b"\n"): if not line: diff --git a/floss/tags/winapi.py b/floss/tags/winapi.py index 70c1118..20d9d2f 100644 --- a/floss/tags/winapi.py +++ b/floss/tags/winapi.py @@ -23,7 +23,7 @@ import pathlib from typing import Set, Sequence from dataclasses import dataclass -from floss.tags import data_root +from floss.tags import data_root, ensure_not_lfs_pointer @dataclass @@ -39,11 +39,13 @@ class WindowsApiStringDatabase: dll_names: Set[str] = set() api_names: Set[str] = set() + ensure_not_lfs_pointer(path / "dlls.txt.gz") for line in gzip.decompress((path / "dlls.txt.gz").read_bytes()).decode("utf-8").splitlines(): if not line: continue dll_names.add(line) + ensure_not_lfs_pointer(path / "apis.txt.gz") for line in gzip.decompress((path / "apis.txt.gz").read_bytes()).decode("utf-8").splitlines(): if not line: continue diff --git a/tests/test_tags_lfs.py b/tests/test_tags_lfs.py new file mode 100644 index 0000000..82e0f72 --- /dev/null +++ b/tests/test_tags_lfs.py @@ -0,0 +1,66 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +import floss.tags.gp +import floss.tags.oss +import floss.tags.expert +import floss.tags.winapi +from floss.tags import ensure_not_lfs_pointer + +LFS_POINTER = b"version https://git-lfs.github.com/spec/v1\noid sha256:0" * 4 + b"\nsize 123\n" + + +def test_ensure_not_lfs_pointer_raises(tmp_path): + path = tmp_path / "db.bin" + path.write_bytes(LFS_POINTER) + with pytest.raises(ValueError, match="Git LFS pointer detected"): + ensure_not_lfs_pointer(path) + + +def test_ensure_not_lfs_pointer_ok(tmp_path): + path = tmp_path / "db.bin" + path.write_bytes(b"not an lfs pointer") + ensure_not_lfs_pointer(path) + + +def test_oss_loader_rejects_lfs_pointer(tmp_path): + path = tmp_path / "oss.jsonl.gz" + path.write_bytes(LFS_POINTER) + with pytest.raises(ValueError, match="Git LFS pointer detected"): + floss.tags.oss.OpenSourceStringDatabase.from_file(path) + + +def test_expert_loader_rejects_lfs_pointer(tmp_path): + path = tmp_path / "expert.jsonl.gz" + path.write_bytes(LFS_POINTER) + with pytest.raises(ValueError, match="Git LFS pointer detected"): + floss.tags.expert.ExpertStringDatabase.from_file(path) + + +def test_winapi_loader_rejects_lfs_pointer(tmp_path): + path = tmp_path / "winapi" + path.mkdir() + (path / "dlls.txt.gz").write_bytes(LFS_POINTER) + (path / "apis.txt.gz").write_bytes(LFS_POINTER) + with pytest.raises(ValueError, match="Git LFS pointer detected"): + floss.tags.winapi.WindowsApiStringDatabase.from_dir(path) + + +def test_gp_loader_rejects_lfs_pointer(tmp_path): + path = tmp_path / "hashes.bin" + path.write_bytes(LFS_POINTER) + with pytest.raises(ValueError, match="Git LFS pointer detected"): + floss.tags.gp.StringHashDatabase.from_file(path)