diff --git a/internal/apex/apex.go b/internal/apex/apex.go index 9a75dfb..de14772 100644 --- a/internal/apex/apex.go +++ b/internal/apex/apex.go @@ -15,7 +15,7 @@ var specialUseSuffixes = []string{ } func Normalize(input string) (string, error) { - s := strings.ToLower(strings.TrimSpace(input)) + s := strings.TrimSpace(input) s = strings.TrimSuffix(s, ".") if s == "" { return "", fmt.Errorf("empty name") @@ -41,6 +41,40 @@ func Normalize(input string) (string, error) { return s, nil } +var ingestProfile = idna.New(idna.MapForLookup()) + +// Canonical prepares an ingest-side hostname for storage: unicode names are +// mapped to punycode (with case folding) so stored records match what search +// clients ask for. Names the IDNA lookup profile rejects but that are plain +// ASCII with common service characters (e.g. DKIM-style underscores) are +// kept, since they are useful recon data. +func Canonical(host string) (string, bool) { + s := strings.TrimPrefix(host, "*.") + s = strings.TrimSuffix(s, ".") + if s == "" || len(s) > 253 { + return "", false + } + if ascii, err := ingestProfile.ToASCII(s); err == nil { + s = ascii + } else { + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case c >= 'a' && c <= 'z', c >= '0' && c <= '9': + case c == '.' || c == '-' || c == '_': + default: + return "", false + } + } + } + for _, label := range strings.Split(s, ".") { + if len(label) == 0 || len(label) > 63 { + return "", false + } + } + return s, true +} + func ValidateApex(normalized string) (string, error) { etld1, err := publicsuffix.EffectiveTLDPlusOne(normalized) if err != nil { diff --git a/internal/apex/apex_test.go b/internal/apex/apex_test.go index c2fd1aa..eafaf5e 100644 --- a/internal/apex/apex_test.go +++ b/internal/apex/apex_test.go @@ -1,6 +1,9 @@ package apex -import "testing" +import ( + "strings" + "testing" +) func TestNormalize(t *testing.T) { cases := []struct { @@ -64,3 +67,25 @@ func TestNormalizeRejectsControlChars(t *testing.T) { } } } + +func TestCanonical(t *testing.T) { + cases := []struct { + in string + want string + ok bool + }{ + {"example.com", "example.com", true}, + {"bücher.example.com", "xn--bcher-kva.example.com", true}, + {"_dmarc.example.com", "_dmarc.example.com", true}, + {"ab\x00cd.example.com", "", false}, + {"a..b.com", "", false}, + {strings.Repeat("a", 64) + ".com", "", false}, + {"*.example.com", "example.com", true}, + } + for _, c := range cases { + got, ok := Canonical(c.in) + if ok != c.ok || (ok && got != c.want) { + t.Errorf("Canonical(%q) = %q, %v; want %q, %v", c.in, got, ok, c.want, c.ok) + } + } +} diff --git a/internal/rfc6962/leaf.go b/internal/rfc6962/leaf.go index bc79b8d..911fd18 100644 --- a/internal/rfc6962/leaf.go +++ b/internal/rfc6962/leaf.go @@ -135,7 +135,7 @@ func dedupeLower(in []string) []string { s = lowerASCII(s) s = trimTrailingDot(s) s = strings.TrimPrefix(s, "*.") - if s == "" || !printableASCII(s) { + if s == "" || hasControlByte(s) { continue } if _, ok := seen[s]; ok { @@ -147,14 +147,15 @@ func dedupeLower(in []string) []string { return out } -// printableASCII rejects control characters, spaces, and non-ASCII bytes. -// Hostnames must never contain them, and the store uses 0x00 as its key -// separator, so a NUL byte from a hostile certificate would corrupt keys. -func printableASCII(s string) bool { +// hasControlByte rejects control characters early. Hostnames must never +// contain them, and the store uses 0x00 as its key separator, so a NUL byte +// from a hostile certificate would corrupt keys. Full charset validation +// happens in apex.Canonical. +func hasControlByte(s string) bool { for i := 0; i < len(s); i++ { - if s[i] < 0x21 || s[i] > 0x7e { - return false + if s[i] < 0x20 || s[i] == 0x7f { + return true } } - return true + return false } diff --git a/internal/tailer/tailer.go b/internal/tailer/tailer.go index 187ed94..e50d019 100644 --- a/internal/tailer/tailer.go +++ b/internal/tailer/tailer.go @@ -174,11 +174,15 @@ func (t *Tailer) fetchRange(ctx context.Context, client *rfc6962.Client, logID s continue } for _, name := range leaf.Names { - a, ok := apex.ApexOf(name) + c, ok := apex.Canonical(name) if !ok { continue } - if err := t.Store.Ingest(store.Record{Apex: a, Sub: name, FirstSeen: fs, Source: sourceCT}); err != nil { + a, ok := apex.ApexOf(c) + if !ok { + continue + } + if err := t.Store.Ingest(store.Record{Apex: a, Sub: c, FirstSeen: fs, Source: sourceCT}); err != nil { return total, err } }