apex,tailer: normalize ingest names to punycode

Certificate SANs went into the store after only lowercasing, so a
unicode name like 'buecher.example.com' was stored as raw UTF-8 and
could never be found by search, which runs IDNA. Ingest now maps
names through a permissive IDNA profile (keeping DKIM-style
underscores that the strict lookup profile rejects), enforces RFC
1035 label lengths, and stores the canonical form. Normalize also no
longer lowercases before IDNA mapping, which produced wrong punycode
for decomposed unicode input.

internal/apex/apex.go

@@ -15,7 +15,7 @@ var specialUseSuffixes = []string{
 }
 
 func Normalize(input string) (string, error) {
-	s := strings.ToLower(strings.TrimSpace(input))
+	s := strings.TrimSpace(input)
 	s = strings.TrimSuffix(s, ".")
 	if s == "" {
 		return "", fmt.Errorf("empty name")
@@ -41,6 +41,40 @@ func Normalize(input string) (string, error) {
 	return s, nil
 }
 
+var ingestProfile = idna.New(idna.MapForLookup())
+
+// Canonical prepares an ingest-side hostname for storage: unicode names are
+// mapped to punycode (with case folding) so stored records match what search
+// clients ask for. Names the IDNA lookup profile rejects but that are plain
+// ASCII with common service characters (e.g. DKIM-style underscores) are
+// kept, since they are useful recon data.
+func Canonical(host string) (string, bool) {
+	s := strings.TrimPrefix(host, "*.")
+	s = strings.TrimSuffix(s, ".")
+	if s == "" || len(s) > 253 {
+		return "", false
+	}
+	if ascii, err := ingestProfile.ToASCII(s); err == nil {
+		s = ascii
+	} else {
+		for i := 0; i < len(s); i++ {
+			c := s[i]
+			switch {
+			case c >= 'a' && c <= 'z', c >= '0' && c <= '9':
+			case c == '.' || c == '-' || c == '_':
+			default:
+				return "", false
+			}
+		}
+	}
+	for _, label := range strings.Split(s, ".") {
+		if len(label) == 0 || len(label) > 63 {
+			return "", false
+		}
+	}
+	return s, true
+}
+
 func ValidateApex(normalized string) (string, error) {
 	etld1, err := publicsuffix.EffectiveTLDPlusOne(normalized)
 	if err != nil {

internal/apex/apex_test.go

@@ -1,6 +1,9 @@
 package apex
 
-import "testing"
+import (
+	"strings"
+	"testing"
+)
 
 func TestNormalize(t *testing.T) {
 	cases := []struct {
@@ -64,3 +67,25 @@ func TestNormalizeRejectsControlChars(t *testing.T) {
 		}
 	}
 }
+
+func TestCanonical(t *testing.T) {
+	cases := []struct {
+		in   string
+		want string
+		ok   bool
+	}{
+		{"example.com", "example.com", true},
+		{"bücher.example.com", "xn--bcher-kva.example.com", true},
+		{"_dmarc.example.com", "_dmarc.example.com", true},
+		{"ab\x00cd.example.com", "", false},
+		{"a..b.com", "", false},
+		{strings.Repeat("a", 64) + ".com", "", false},
+		{"*.example.com", "example.com", true},
+	}
+	for _, c := range cases {
+		got, ok := Canonical(c.in)
+		if ok != c.ok || (ok && got != c.want) {
+			t.Errorf("Canonical(%q) = %q, %v; want %q, %v", c.in, got, ok, c.want, c.ok)
+		}
+	}
+}

internal/rfc6962/leaf.go

@@ -135,7 +135,7 @@ func dedupeLower(in []string) []string {
 		s = lowerASCII(s)
 		s = trimTrailingDot(s)
 		s = strings.TrimPrefix(s, "*.")
-		if s == "" || !printableASCII(s) {
+		if s == "" || hasControlByte(s) {
 			continue
 		}
 		if _, ok := seen[s]; ok {
@@ -147,14 +147,15 @@ func dedupeLower(in []string) []string {
 	return out
 }
 
-// printableASCII rejects control characters, spaces, and non-ASCII bytes.
-// Hostnames must never contain them, and the store uses 0x00 as its key
-// separator, so a NUL byte from a hostile certificate would corrupt keys.
-func printableASCII(s string) bool {
+// hasControlByte rejects control characters early. Hostnames must never
+// contain them, and the store uses 0x00 as its key separator, so a NUL byte
+// from a hostile certificate would corrupt keys. Full charset validation
+// happens in apex.Canonical.
+func hasControlByte(s string) bool {
 	for i := 0; i < len(s); i++ {
-		if s[i] < 0x21 || s[i] > 0x7e {
-			return false
+		if s[i] < 0x20 || s[i] == 0x7f {
+			return true
 		}
 	}
-	return true
+	return false
 }

internal/tailer/tailer.go

@@ -174,11 +174,15 @@ func (t *Tailer) fetchRange(ctx context.Context, client *rfc6962.Client, logID s
 				continue
 			}
 			for _, name := range leaf.Names {
-				a, ok := apex.ApexOf(name)
+				c, ok := apex.Canonical(name)
 				if !ok {
 					continue
 				}
-				if err := t.Store.Ingest(store.Record{Apex: a, Sub: name, FirstSeen: fs, Source: sourceCT}); err != nil {
+				a, ok := apex.ApexOf(c)
+				if !ok {
+					continue
+				}
+				if err := t.Store.Ingest(store.Record{Apex: a, Sub: c, FirstSeen: fs, Source: sourceCT}); err != nil {
 					return total, err
 				}
 			}