commit 7cc863d40b4a7d01157ea06f03ff1ac6f444de91 Author: lakshit verma Date: Thu Aug 6 05:30:51 2026 +0530 feat(core): add journal, collector, storage, and materializer - durable event schema with stream identity and dedup hashing - SQLite WAL and in-memory journal stores with atomic append+checkpoint - list-and-watch collector with bookmarks, reconnects, and 410 relists - state reduction, snapshots, and semantic field diffs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1b1ff83 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +/bin/ +web/node_modules/ +web/dist/ +*.db +*.db-wal +*.db-shm +*.sqlite +*.sqlite3 +.DS_Store +coverage.out +tmp/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a4b533f --- /dev/null +++ b/Makefile @@ -0,0 +1,50 @@ +.PHONY: build build-server build-cli web test test-unit test-integration test-e2e lint vet fmt tidy install-bins docs clean + +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) +LDFLAGS := -X github.com/krply/krply/internal/version.Version=$(VERSION) + +build: build-cli build-server + +build-cli: + go build -ldflags '$(LDFLAGS)' -o bin/krply ./cmd/krply + +build-server: + go build -ldflags '$(LDFLAGS)' -o bin/krply-server ./cmd/krply-server + +web: + cd web && npm install && npm run build + +test: + go test ./... + +test-unit: + go test ./internal/... ./cmd/... -short + +test-integration: + go test ./test/integration/... -tags integration -timeout 20m + +test-e2e: + go test ./test/e2e/... -tags e2e -timeout 30m + +lint: + go vet ./... + cd web && npm run lint 2>/dev/null || true + +vet: + go vet ./... + +fmt: + gofmt -w $$(find . -name '*.go' -not -path './vendor/*') + +tidy: + go mod tidy + +install-bins: build + install -m 0755 bin/krply /usr/local/bin/krply + install -m 0755 bin/krply-server /usr/local/bin/krply-server + +docs: + @echo "documentation lives in docs/" + +clean: + rm -rf bin web/dist diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..37dcc1d --- /dev/null +++ b/go.mod @@ -0,0 +1,65 @@ +module github.com/krply/krply + +go 1.26.4 + +require ( + github.com/google/uuid v1.6.0 + github.com/prometheus/client_golang v1.24.1 + github.com/spf13/cobra v1.10.2 + k8s.io/api v0.36.3 + k8s.io/apimachinery v0.36.3 + k8s.io/client-go v0.36.3 + modernc.org/sqlite v1.56.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.70.1 // indirect + github.com/prometheus/procfs v0.21.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/spf13/pflag v1.0.9 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/net v0.57.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/term v0.45.0 // indirect + golang.org/x/text v0.40.0 // indirect + golang.org/x/time v0.14.0 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect + modernc.org/libc v1.74.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.3 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e7cb1c9 --- /dev/null +++ b/go.sum @@ -0,0 +1,185 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.36.3 h1:NxB+05W2UGqXWFXcLO0RB5cnqnUPP5v5sVlaOH0Iz4w= +k8s.io/api v0.36.3/go.mod h1:JzLQKqRHC5+I8RVj/lS3lCg0mg6nWI9Fo/Sk3ElxHzg= +k8s.io/apimachinery v0.36.3 h1:PkzMRBRG8joFD8EhCuQAtNPvJlxb82FwplP26HIzvAM= +k8s.io/apimachinery v0.36.3/go.mod h1:cTSjBWgPe/6CQyBKzY/hDIRWCQQQeK0mfLbml0UYFHE= +k8s.io/client-go v0.36.3 h1:M4JdVzXxYcZk4fGpfDdYnxSwhLKWCFoQsHW6t+z8Hfg= +k8s.io/client-go v0.36.3/go.mod h1:gcPwr0c87vjjG6HB6pWEqOeuYVoXSsREjzux2j6GF30= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= +modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0= +modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw= +sigs.k8s.io/structured-merge-diff/v6 v6.3.3/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/internal/discovery/discover.go b/internal/discovery/discover.go new file mode 100644 index 0000000..04bedba --- /dev/null +++ b/internal/discovery/discover.go @@ -0,0 +1,177 @@ +// Package discovery resolves resource specifications into full group/version/ +// resource triples with scope information. +// +// A ResourceSpec is "full" when it carries a Version and a Kind. Partial specs +// (for example just "deploy" or "pods") are resolved through well-known +// aliases and the Kubernetes discovery API. +package discovery + +import ( + "context" + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" +) + +// ResourceSpec identifies one collection to watch. +// +// APIGroup is empty for core/v1 resources. A Namespace of "" means cluster +// scope (or all namespaces for a namespaced resource). +type ResourceSpec struct { + APIGroup string + Version string + Resource string + Kind string + Namespace string +} + +// DefaultResources returns the MVP watch list. +// +// Namespaced resources use an empty Namespace so the collector watches the +// collection cluster-wide. Cluster-scoped resources (namespaces, runtime +// classes) are resolved to Namespace "" by discovery. +func DefaultResources() []ResourceSpec { + return []ResourceSpec{ + {APIGroup: "", Version: "v1", Resource: "namespaces", Kind: "Namespace", Namespace: ""}, + {APIGroup: "", Version: "v1", Resource: "configmaps", Kind: "ConfigMap", Namespace: ""}, + {APIGroup: "", Version: "v1", Resource: "services", Kind: "Service", Namespace: ""}, + {APIGroup: "apps", Version: "v1", Resource: "deployments", Kind: "Deployment", Namespace: ""}, + {APIGroup: "apps", Version: "v1", Resource: "statefulsets", Kind: "StatefulSet", Namespace: ""}, + {APIGroup: "apps", Version: "v1", Resource: "daemonsets", Kind: "DaemonSet", Namespace: ""}, + {APIGroup: "node.k8s.io", Version: "v1", Resource: "runtimeclasses", Kind: "RuntimeClass", Namespace: ""}, + {APIGroup: "", Version: "v1", Resource: "pods", Kind: "Pod", Namespace: ""}, + } +} + +// Discover resolves the default resource list against the cluster's discovery +// API and returns the full specs. Partial specs are filled in via aliases and +// discovery. +func Discover(ctx context.Context, client kubernetes.Interface) ([]ResourceSpec, error) { + return Resolve(ctx, client, DefaultResources()) +} + +// Resolve fills missing APIGroup/Version/Kind for every spec, resolves +// well-known aliases, and clears Namespace for cluster-scoped resources. +func Resolve(ctx context.Context, client kubernetes.Interface, specs []ResourceSpec) ([]ResourceSpec, error) { + out := make([]ResourceSpec, 0, len(specs)) + for _, spec := range specs { + resolved, err := resolveOne(ctx, client, spec) + if err != nil { + return nil, err + } + out = append(out, resolved) + } + return out, nil +} + +func resolveOne(ctx context.Context, client kubernetes.Interface, spec ResourceSpec) (ResourceSpec, error) { + if alias, ok := aliases[strings.ToLower(spec.Resource)]; ok { + spec = mergeAlias(spec, alias) + } + if spec.Version != "" && spec.Kind != "" { + return spec, nil + } + return resolveViaDiscovery(ctx, client, spec) +} + +// mergeAlias fills empty fields on spec from the alias, keeping any explicit +// Namespace and replacing the alias resource name with its canonical form. +func mergeAlias(spec, alias ResourceSpec) ResourceSpec { + if alias.APIGroup != "" { + spec.APIGroup = alias.APIGroup + } + if alias.Version != "" { + spec.Version = alias.Version + } + if alias.Kind != "" { + spec.Kind = alias.Kind + } + if alias.Resource != "" { + spec.Resource = alias.Resource + } + return spec +} + +func resolveViaDiscovery(ctx context.Context, client kubernetes.Interface, spec ResourceSpec) (ResourceSpec, error) { + if spec.Version != "" { + return resolveInGroupVersion(ctx, client, spec) + } + lists, err := client.Discovery().ServerPreferredResources() + for _, l := range lists { + if l == nil { + continue + } + for _, api := range l.APIResources { + if !strings.EqualFold(api.Name, spec.Resource) { + continue + } + gv, parseErr := schema.ParseGroupVersion(l.GroupVersion) + if parseErr != nil { + continue + } + spec.APIGroup = gv.Group + spec.Version = gv.Version + spec.Kind = api.Kind + if !api.Namespaced { + spec.Namespace = "" + } + return spec, nil + } + } + if err != nil { + return spec, fmt.Errorf("discover %q: %w", spec.Resource, err) + } + return spec, fmt.Errorf("discover %q: not found in ServerPreferredResources", spec.Resource) +} + +func resolveInGroupVersion(ctx context.Context, client kubernetes.Interface, spec ResourceSpec) (ResourceSpec, error) { + gv := spec.Version + if spec.APIGroup != "" { + gv = spec.APIGroup + "/" + spec.Version + } + list, err := client.Discovery().ServerResourcesForGroupVersion(gv) + if err != nil { + return spec, fmt.Errorf("discover %q in %q: %w", spec.Resource, gv, err) + } + for _, api := range list.APIResources { + if !strings.EqualFold(api.Name, spec.Resource) { + continue + } + spec.Kind = api.Kind + if !api.Namespaced { + spec.Namespace = "" + } + return spec, nil + } + return spec, fmt.Errorf("discover %q: not found in group version %q", spec.Resource, gv) +} + +// aliases maps common kubectl-style names to canonical specs. +var aliases = map[string]ResourceSpec{ + "deploy": {APIGroup: "apps", Version: "v1", Resource: "deployments", Kind: "Deployment"}, + "deployment": {APIGroup: "apps", Version: "v1", Resource: "deployments", Kind: "Deployment"}, + "deployments": {APIGroup: "apps", Version: "v1", Resource: "deployments", Kind: "Deployment"}, + "sts": {APIGroup: "apps", Version: "v1", Resource: "statefulsets", Kind: "StatefulSet"}, + "statefulset": {APIGroup: "apps", Version: "v1", Resource: "statefulsets", Kind: "StatefulSet"}, + "statefulsets": {APIGroup: "apps", Version: "v1", Resource: "statefulsets", Kind: "StatefulSet"}, + "ds": {APIGroup: "apps", Version: "v1", Resource: "daemonsets", Kind: "DaemonSet"}, + "daemonset": {APIGroup: "apps", Version: "v1", Resource: "daemonsets", Kind: "DaemonSet"}, + "daemonsets": {APIGroup: "apps", Version: "v1", Resource: "daemonsets", Kind: "DaemonSet"}, + "cm": {Version: "v1", Resource: "configmaps", Kind: "ConfigMap"}, + "configmap": {Version: "v1", Resource: "configmaps", Kind: "ConfigMap"}, + "configmaps": {Version: "v1", Resource: "configmaps", Kind: "ConfigMap"}, + "po": {Version: "v1", Resource: "pods", Kind: "Pod"}, + "pod": {Version: "v1", Resource: "pods", Kind: "Pod"}, + "pods": {Version: "v1", Resource: "pods", Kind: "Pod"}, + "svc": {Version: "v1", Resource: "services", Kind: "Service"}, + "service": {Version: "v1", Resource: "services", Kind: "Service"}, + "services": {Version: "v1", Resource: "services", Kind: "Service"}, + "ns": {Version: "v1", Resource: "namespaces", Kind: "Namespace"}, + "namespace": {Version: "v1", Resource: "namespaces", Kind: "Namespace"}, + "namespaces": {Version: "v1", Resource: "namespaces", Kind: "Namespace"}, + "rc": {APIGroup: "node.k8s.io", Version: "v1", Resource: "runtimeclasses", Kind: "RuntimeClass"}, + "runtimeclass": {APIGroup: "node.k8s.io", Version: "v1", Resource: "runtimeclasses", Kind: "RuntimeClass"}, + "runtimeclasses": {APIGroup: "node.k8s.io", Version: "v1", Resource: "runtimeclasses", Kind: "RuntimeClass"}, +} diff --git a/internal/discovery/discover_test.go b/internal/discovery/discover_test.go new file mode 100644 index 0000000..d6ac498 --- /dev/null +++ b/internal/discovery/discover_test.go @@ -0,0 +1,133 @@ +package discovery + +import ( + "context" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + fakediscovery "k8s.io/client-go/discovery/fake" + fakeclientset "k8s.io/client-go/kubernetes/fake" +) + +func newFakeClient() *fakeclientset.Clientset { + cli := fakeclientset.NewSimpleClientset() + cli.Discovery().(*fakediscovery.FakeDiscovery).Resources = []*metav1.APIResourceList{ + { + GroupVersion: "v1", + APIResources: []metav1.APIResource{ + {Name: "namespaces", Kind: "Namespace", Namespaced: false}, + {Name: "configmaps", Kind: "ConfigMap", Namespaced: true}, + {Name: "services", Kind: "Service", Namespaced: true}, + {Name: "pods", Kind: "Pod", Namespaced: true}, + }, + }, + { + GroupVersion: "apps/v1", + APIResources: []metav1.APIResource{ + {Name: "deployments", Kind: "Deployment", Namespaced: true}, + {Name: "statefulsets", Kind: "StatefulSet", Namespaced: true}, + {Name: "daemonsets", Kind: "DaemonSet", Namespaced: true}, + }, + }, + { + GroupVersion: "node.k8s.io/v1", + APIResources: []metav1.APIResource{ + {Name: "runtimeclasses", Kind: "RuntimeClass", Namespaced: false}, + }, + }, + } + return cli +} + +func TestDefaultResources(t *testing.T) { + specs := DefaultResources() + if len(specs) != 8 { + t.Fatalf("DefaultResources len = %d, want 8", len(specs)) + } + byResource := map[string]ResourceSpec{} + for _, s := range specs { + byResource[s.Resource] = s + } + want := []string{"namespaces", "configmaps", "services", "deployments", "statefulsets", "daemonsets", "runtimeclasses", "pods"} + for _, r := range want { + if _, ok := byResource[r]; !ok { + t.Errorf("DefaultResources missing %q", r) + } + } + if got := byResource["deployments"]; got.APIGroup != "apps" || got.Version != "v1" || got.Kind != "Deployment" { + t.Errorf("deployments spec = %+v", got) + } + if got := byResource["runtimeclasses"]; got.APIGroup != "node.k8s.io" || got.Version != "v1" || got.Kind != "RuntimeClass" { + t.Errorf("runtimeclasses spec = %+v", got) + } +} + +func TestDiscoverResolvesDefaults(t *testing.T) { + cli := newFakeClient() + specs, err := Discover(context.Background(), cli) + if err != nil { + t.Fatalf("Discover: %v", err) + } + if len(specs) != 8 { + t.Fatalf("Discover len = %d, want 8", len(specs)) + } + for _, s := range specs { + if s.Version == "" || s.Kind == "" { + t.Errorf("spec not fully resolved: %+v", s) + } + } +} + +func TestResolveAliases(t *testing.T) { + cli := newFakeClient() + inputs := []ResourceSpec{ + {Resource: "deploy"}, + {Resource: "sts"}, + {Resource: "ds"}, + {Resource: "cm"}, + {Resource: "po"}, + {Resource: "svc"}, + {Resource: "ns"}, + {Resource: "rc"}, + } + resolved, err := Resolve(context.Background(), cli, inputs) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + want := []struct{ resource, group, kind string }{ + {"deployments", "apps", "Deployment"}, + {"statefulsets", "apps", "StatefulSet"}, + {"daemonsets", "apps", "DaemonSet"}, + {"configmaps", "", "ConfigMap"}, + {"pods", "", "Pod"}, + {"services", "", "Service"}, + {"namespaces", "", "Namespace"}, + {"runtimeclasses", "node.k8s.io", "RuntimeClass"}, + } + for i, w := range want { + got := resolved[i] + if got.Resource != w.resource || got.APIGroup != w.group || got.Kind != w.kind { + t.Errorf("alias %d: got %+v, want resource=%q group=%q kind=%q", i, got, w.resource, w.group, w.kind) + } + } +} + +func TestResolvePartialWithKnownVersion(t *testing.T) { + cli := newFakeClient() + resolved, err := Resolve(context.Background(), cli, []ResourceSpec{ + {APIGroup: "apps", Version: "v1", Resource: "deployments"}, + }) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if len(resolved) != 1 { + t.Fatalf("Resolve len = %d", len(resolved)) + } + s := resolved[0] + if s.Kind != "Deployment" { + t.Errorf("kind = %q, want Deployment", s.Kind) + } + if s.Namespace != "" { + t.Errorf("deployments should be namespaced with empty namespace filter, got %q", s.Namespace) + } +} diff --git a/internal/event/record.go b/internal/event/record.go new file mode 100644 index 0000000..b5c41b7 --- /dev/null +++ b/internal/event/record.go @@ -0,0 +1,133 @@ +// Package event defines the durable record types stored in the krply journal. +// +// Every record is an immutable, flat entry. The raw watch payload is kept +// separate from normalized fields so unknown CRD fields survive intact. +package event + +import ( + "encoding/json" + "time" +) + +// RecordType discriminates the kinds of entries in the journal. +type RecordType string + +const ( + // TypeEvent is a watch event (ADDED, MODIFIED, DELETED, BOOKMARK, ERROR). + TypeEvent RecordType = "EVENT" + // TypeBaseline is a full list result used as a relist baseline. + TypeBaseline RecordType = "BASELINE" + // TypeGap records that watch continuity was lost. + TypeGap RecordType = "GAP" + // TypeCoverageChange records a resource becoming available or newly discovered. + TypeCoverageChange RecordType = "COVERAGE_CHANGE" + // TypeAuditCorrelation records optional request provenance. + TypeAuditCorrelation RecordType = "AUDIT_CORRELATION" + // TypeSnapshot records a materialized set of stream boundaries. + TypeSnapshot RecordType = "SNAPSHOT" + // TypeCheckpoint records a progress-only bookmark advance. + TypeCheckpoint RecordType = "CHECKPOINT" +) + +// WatchType is the original Kubernetes watch event type. +type WatchType string + +const ( + WatchAdded WatchType = "ADDED" + WatchModified WatchType = "MODIFIED" + WatchDeleted WatchType = "DELETED" + WatchBookmark WatchType = "BOOKMARK" + WatchError WatchType = "ERROR" +) + +// ResourceRef identifies an object. Resource versions and UIDs are only +// meaningful within one cluster and one API resource. +type ResourceRef struct { + Group string `json:"group"` + Version string `json:"version"` + Kind string `json:"kind"` + Namespace string `json:"namespace"` + Name string `json:"name"` + UID string `json:"uid"` + ResourceVersion string `json:"resource_version"` +} + +// ObjectKey returns the namespace/name scoped key, used for materialization. +func (r ResourceRef) ObjectKey() string { + if r.Namespace == "" { + return r.Name + } + return r.Namespace + "/" + r.Name +} + +// GVK returns the group/version/kind triple. +func (r ResourceRef) GVK() (string, string, string) { + return r.Group, r.Version, r.Kind +} + +// Provenance records optional audit-log correlation. It is only present when +// an audit record can be matched to an event. +type Provenance struct { + RequestID string `json:"request_id"` + User string `json:"user"` + UserAgent string `json:"user_agent"` + SourceIPs []string `json:"source_ips,omitempty"` + Verb string `json:"verb"` + Stage string `json:"stage"` + Response int `json:"response_code"` + Annotations map[string]string `json:"annotations,omitempty"` +} + +// GapInfo describes a continuity loss on a stream. +type GapInfo struct { + FromResourceVersion string `json:"from_resource_version"` + ToResourceVersion string `json:"to_resource_version"` + Reason string `json:"reason"` + DetectedAt time.Time `json:"detected_at"` +} + +// CoverageInfo describes a change in stream availability. +type CoverageInfo struct { + Previous *CoverageState `json:"previous,omitempty"` + Current CoverageState `json:"current"` + Reason string `json:"reason"` +} + +// CoverageState is one side of a coverage transition. +type CoverageState struct { + Available bool `json:"available"` + Resource string `json:"resource"` + Namespace string `json:"namespace"` + LastResourceVersion string `json:"last_resource_version"` +} + +// CheckpointInfo records a durable progress marker. +type CheckpointInfo struct { + ResourceVersion string `json:"resource_version"` + BookmarkedAt time.Time `json:"bookmarked_at"` +} + +// SnapshotInfo records a materialized snapshot boundary. +type SnapshotInfo struct { + Name string `json:"name"` +} + +// Record is the immutable journal entry. +type Record struct { + ClusterID string `json:"cluster_id"` + StreamID string `json:"stream_id"` + Type RecordType `json:"record_type"` + EventID string `json:"event_id"` + IngestSeq int64 `json:"ingest_seq"` + ObservedAt time.Time `json:"observed_at"` + WatchType WatchType `json:"watch_type,omitempty"` + Synthetic bool `json:"synthetic,omitempty"` + Resource ResourceRef `json:"resource,omitempty"` + ObjectHash string `json:"object_hash,omitempty"` + Object json.RawMessage `json:"object,omitempty"` + Provenance *Provenance `json:"provenance,omitempty"` + Gap *GapInfo `json:"gap,omitempty"` + Coverage *CoverageInfo `json:"coverage,omitempty"` + Checkpoint *CheckpointInfo `json:"checkpoint,omitempty"` + Snapshot *SnapshotInfo `json:"snapshot,omitempty"` +} diff --git a/internal/event/stream.go b/internal/event/stream.go new file mode 100644 index 0000000..4fcd221 --- /dev/null +++ b/internal/event/stream.go @@ -0,0 +1,76 @@ +package event + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" +) + +// Stream identifies one collection to watch. Resource versions and UIDs are +// only comparable within a single stream. +type Stream struct { + ClusterID string + Group string + Version string + Resource string + Namespace string + Selector string +} + +// ID returns the stable stream identifier used across the journal. +// Format: cluster/group/version/resource/namespace/selector +func (s Stream) ID() string { + var b strings.Builder + b.WriteString(s.ClusterID) + b.WriteByte('/') + b.WriteString(s.Group) + b.WriteByte('/') + b.WriteString(s.Version) + b.WriteByte('/') + b.WriteString(s.Resource) + b.WriteByte('/') + b.WriteString(s.Namespace) + b.WriteByte('/') + b.WriteString(s.Selector) + return b.String() +} + +// StreamID parses a stream identifier produced by Stream.ID. +func StreamID(id string) (Stream, error) { + parts := strings.Split(id, "/") + if len(parts) != 6 { + return Stream{}, fmt.Errorf("stream id %q: want 6 slash-separated parts", id) + } + return Stream{ + ClusterID: parts[0], + Group: parts[1], + Version: parts[2], + Resource: parts[3], + Namespace: parts[4], + Selector: parts[5], + }, nil +} + +// GVR returns group, version, and resource lowercased for client-go use. +func (s Stream) GVR() (string, string, string) { + return strings.ToLower(s.Group), strings.ToLower(s.Version), strings.ToLower(s.Resource) +} + +// EventID derives a deterministic deduplication key for a watch event. +// The same underlying API event must always produce the same key, so a +// duplicate delivery after a reconnect is idempotent. +func EventID(stream Stream, resource ResourceRef, watchType WatchType, observedAt int64) string { + h := sha256.New() + fmt.Fprintf(h, "%s\x00%s\x00%s\x00%s\x00%s\x00%s", stream.ID(), resource.Namespace, resource.Name, resource.UID, resource.ResourceVersion, string(watchType)) + // observedAt is deliberately excluded: resource version ordering is the + // dedup domain, not wall-clock observation time. + _ = observedAt + return hex.EncodeToString(h.Sum(nil)) +} + +// ObjectHash computes a fast equality hash over the raw object payload. +func ObjectHash(object []byte) string { + h := sha256.Sum256(object) + return hex.EncodeToString(h[:]) +} diff --git a/internal/materialize/diff.go b/internal/materialize/diff.go new file mode 100644 index 0000000..8cd48c7 --- /dev/null +++ b/internal/materialize/diff.go @@ -0,0 +1,234 @@ +package materialize + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "sort" + "strings" + "time" +) + +// DiffResult is the semantic diff between two points in time. +type DiffResult struct { + Before time.Time + After time.Time + Changes []ObjectDiff + HasGaps bool + Warning string +} + +// ObjectDiff is the set of field changes for one object. +type ObjectDiff struct { + Namespace string + Name string + Kind string + Changes []FieldChange +} + +// FieldChange is one changed field, addressed by a dotted JSON path. +type FieldChange struct { + Path string + Before any + After any + Added bool + Removed bool +} + +// ignoredKeys are server-owned or hash-only noise fields excluded from diffs. +var ignoredKeys = map[string]bool{ + "uid": true, + "resourceVersion": true, + "creationTimestamp": true, + "generation": true, + "managedFields": true, + "deletionTimestamp": true, + "ownerReferences": true, + "status": true, + "kubectl.kubernetes.io/last-applied-configuration": true, +} + +// Diff reconstructs cluster state before and after, intersects object keys, +// and reports semantic field changes, ignoring server-owned metadata. +func (m *Materializer) Diff(ctx context.Context, clusterID, namespace string, before, after time.Time) (*DiffResult, error) { + streams, err := m.store.Streams(ctx) + if err != nil { + return nil, err + } + + res := &DiffResult{Before: before, After: after} + beforeStates := map[string]ObjectState{} + afterStates := map[string]ObjectState{} + var gapped []string + + for _, s := range streams { + if s.ClusterID != clusterID { + continue + } + b, err := m.StreamState(ctx, s.StreamID, before) + if err != nil { + return nil, err + } + a, err := m.StreamState(ctx, s.StreamID, after) + if err != nil { + return nil, err + } + if b.HasGaps || a.HasGaps { + gapped = append(gapped, s.StreamID) + } + for _, st := range b.Objects { + if namespace != "" && st.Namespace != namespace { + continue + } + beforeStates[st.StreamID+"|"+st.Namespace+"/"+st.Name] = st + } + for _, st := range a.Objects { + if namespace != "" && st.Namespace != namespace { + continue + } + afterStates[st.StreamID+"|"+st.Namespace+"/"+st.Name] = st + } + } + + keys := make([]string, 0, len(beforeStates)+len(afterStates)) + seen := map[string]bool{} + for k := range beforeStates { + if !seen[k] { + seen[k] = true + keys = append(keys, k) + } + } + for k := range afterStates { + if !seen[k] { + seen[k] = true + keys = append(keys, k) + } + } + sort.Strings(keys) + + for _, k := range keys { + b, okB := beforeStates[k] + a, okA := afterStates[k] + od := ObjectDiff{} + var beforeAny, afterAny any + if okB { + od.Namespace, od.Name, od.Kind = b.Namespace, b.Name, b.Kind + beforeAny = parseObject(b.Object) + } + if okA { + od.Namespace, od.Name, od.Kind = a.Namespace, a.Name, a.Kind + afterAny = parseObject(a.Object) + } + switch { + case okB && !okA: + od.Changes = []FieldChange{{Path: "", Removed: true, Before: beforeAny}} + case !okB && okA: + od.Changes = []FieldChange{{Path: "", Added: true, After: afterAny}} + default: + diffValue("", beforeAny, afterAny, &od.Changes) + sort.Slice(od.Changes, func(i, j int) bool { return od.Changes[i].Path < od.Changes[j].Path }) + if len(od.Changes) == 0 { + continue + } + } + res.Changes = append(res.Changes, od) + } + + sort.Slice(res.Changes, func(i, j int) bool { + if res.Changes[i].Namespace != res.Changes[j].Namespace { + return res.Changes[i].Namespace < res.Changes[j].Namespace + } + if res.Changes[i].Name != res.Changes[j].Name { + return res.Changes[i].Name < res.Changes[j].Name + } + return res.Changes[i].Kind < res.Changes[j].Kind + }) + + if len(gapped) > 0 { + sort.Strings(gapped) + res.HasGaps = true + res.Warning = "coverage incomplete for stream(s): " + strings.Join(gapped, ", ") + } + + return res, nil +} + +// DiffObjects computes the field changes between two raw object payloads, +// ignoring server-owned metadata. It is used for timeline summaries. +func DiffObjects(before, after json.RawMessage) []FieldChange { + var out []FieldChange + if len(before) == 0 && len(after) == 0 { + return out + } + if len(before) == 0 { + return []FieldChange{{Path: "", Added: true, After: parseObject(after)}} + } + if len(after) == 0 { + return []FieldChange{{Path: "", Removed: true, Before: parseObject(before)}} + } + diffValue("", parseObject(before), parseObject(after), &out) + sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path }) + return out +} + +// parseObject decodes a raw object into an arbitrary JSON value. +func parseObject(raw json.RawMessage) any { + var v any + if err := json.Unmarshal(raw, &v); err != nil { + return string(raw) + } + return v +} + +// diffValue recurses into before/after and appends field changes at out. +// Arrays are compared by index; a length change is reported as one change at +// the array's path. Map objects recurse by key. +func diffValue(path string, before, after any, out *[]FieldChange) { + if reflect.DeepEqual(before, after) { + return + } + bm, bOK := before.(map[string]any) + am, aOK := after.(map[string]any) + if bOK && aOK { + keys := map[string]bool{} + for k := range bm { + keys[k] = true + } + for k := range am { + keys[k] = true + } + sorted := make([]string, 0, len(keys)) + for k := range keys { + sorted = append(sorted, k) + } + sort.Strings(sorted) + for _, k := range sorted { + if ignoredKeys[k] { + continue + } + diffValue(joinPath(path, k), bm[k], am[k], out) + } + return + } + bl, bArr := before.([]any) + al, aArr := after.([]any) + if bArr && aArr { + if len(bl) != len(al) { + *out = append(*out, FieldChange{Path: path, Before: before, After: after}) + return + } + for i := range bl { + diffValue(fmt.Sprintf("%s[%d]", path, i), bl[i], al[i], out) + } + return + } + *out = append(*out, FieldChange{Path: path, Before: before, After: after}) +} + +func joinPath(parent, key string) string { + if parent == "" { + return key + } + return parent + "." + key +} diff --git a/internal/materialize/materialize.go b/internal/materialize/materialize.go new file mode 100644 index 0000000..5e0d4c9 --- /dev/null +++ b/internal/materialize/materialize.go @@ -0,0 +1,164 @@ +// Package materialize reduces the event journal into object state, +// reconstructs state at points in time, materializes snapshots, and computes +// semantic field diffs. +package materialize + +import ( + "context" + "encoding/json" + "errors" + "sort" + "time" + + "github.com/krply/krply/internal/event" + "github.com/krply/krply/internal/storage" +) + +// ErrNoObjectAtTime reports that no object existed at the requested time. +var ErrNoObjectAtTime = errors.New("materialize: no object state at time") + +// ObjectState is one materialized object at a point in time. +type ObjectState struct { + ClusterID string + StreamID string + Namespace string + Name string + Kind string + Object json.RawMessage + At time.Time +} + +// Materializer reduces stored events into object state. +type Materializer struct { + store storage.Store +} + +// NewMaterializer returns a Materializer backed by store. +func NewMaterializer(store storage.Store) *Materializer { + return &Materializer{store: store} +} + +// ObjectAt returns the last event record for the object at or before at. +// It returns ErrNoObjectAtTime if no object existed at that time. +func (m *Materializer) ObjectAt(ctx context.Context, clusterID, streamID, namespace, name string, at time.Time) (*event.Record, error) { + rec, err := m.store.ObjectAt(ctx, storage.ObjectRef{ + ClusterID: clusterID, + StreamID: streamID, + Namespace: namespace, + Name: name, + }, at) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return nil, ErrNoObjectAtTime + } + return nil, err + } + return rec, nil +} + +// StateAt reconstructs the object state of every stream in a cluster at time +// at, without persisting anything. It reports coverage completeness. +func (m *Materializer) StateAt(ctx context.Context, clusterID string, at time.Time) ([]ObjectState, bool, error) { + streams, err := m.store.Streams(ctx) + if err != nil { + return nil, false, err + } + var all []ObjectState + complete := true + for _, sm := range streams { + if sm.ClusterID != clusterID { + continue + } + res, err := m.StreamState(ctx, sm.StreamID, at) + if err != nil { + return nil, false, err + } + if !res.HasBaseline || res.HasGaps { + complete = false + } + all = append(all, res.Objects...) + } + sortObjects(all) + return all, complete, nil +} + +// StreamStateResult is the reduced state of one stream. +type StreamStateResult struct { + Objects []ObjectState + HasGaps bool + GapCount int + HasBaseline bool +} + +// StreamState reduces every event of a stream observed up to at into object +// state, reporting whether coverage was lost along the way. +func (m *Materializer) StreamState(ctx context.Context, streamID string, at time.Time) (StreamStateResult, error) { + recs, err := m.store.StreamEvents(ctx, streamID, at) + if err != nil { + return StreamStateResult{}, err + } + red := reduce(streamID, recs) + objs := make([]ObjectState, 0, len(red.states)) + for _, st := range red.states { + objs = append(objs, st) + } + sortObjects(objs) + return StreamStateResult{Objects: objs, HasGaps: red.hasGaps, GapCount: red.gapCount, HasBaseline: red.hasBaseline}, nil +} + +// reduction is the per-stream reduce result. +type reduction struct { + states map[string]ObjectState + hasBaseline bool + hasGaps bool + gapCount int +} + +// reduce folds records into per-object state. DELETED removes the object, +// BASELINE marks full coverage from that point, and GAP marks lost coverage. +func reduce(streamID string, recs []event.Record) reduction { + red := reduction{states: map[string]ObjectState{}} + for i := range recs { + rec := &recs[i] + switch rec.Type { + case event.TypeBaseline: + red.hasBaseline = true + case event.TypeGap: + red.hasGaps = true + red.gapCount++ + case event.TypeEvent: + switch rec.WatchType { + case event.WatchDeleted: + delete(red.states, rec.Resource.ObjectKey()) + case event.WatchBookmark, event.WatchError: + // No object payload. + default: + if len(rec.Object) == 0 { + continue + } + red.states[rec.Resource.ObjectKey()] = ObjectState{ + ClusterID: rec.ClusterID, + StreamID: streamID, + Namespace: rec.Resource.Namespace, + Name: rec.Resource.Name, + Kind: rec.Resource.Kind, + Object: rec.Object, + At: rec.ObservedAt, + } + } + } + } + return red +} + +func sortObjects(objs []ObjectState) { + sort.Slice(objs, func(i, j int) bool { + if objs[i].StreamID != objs[j].StreamID { + return objs[i].StreamID < objs[j].StreamID + } + if objs[i].Namespace != objs[j].Namespace { + return objs[i].Namespace < objs[j].Namespace + } + return objs[i].Name < objs[j].Name + }) +} diff --git a/internal/materialize/materialize_test.go b/internal/materialize/materialize_test.go new file mode 100644 index 0000000..45e6753 --- /dev/null +++ b/internal/materialize/materialize_test.go @@ -0,0 +1,314 @@ +package materialize + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/krply/krply/internal/event" + "github.com/krply/krply/internal/storage" +) + +func deployObject(name, image, rv string) json.RawMessage { + obj := map[string]any{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]any{ + "name": name, + "namespace": "default", + "uid": "uid-" + name, + "resourceVersion": rv, + "generation": len(rv), + "creationTimestamp": "2026-01-01T00:00:00Z", + "managedFields": []any{ + map[string]any{"manager": "kube-controller-manager", "operation": "Update"}, + }, + }, + "spec": map[string]any{ + "template": map[string]any{ + "spec": map[string]any{ + "containers": []any{ + map[string]any{"name": "app", "image": image}, + }, + }, + }, + }, + "status": map[string]any{"readyReplicas": 1}, + } + raw, _ := json.Marshal(obj) + return raw +} + +func configMapObject(name, rv string) json.RawMessage { + obj := map[string]any{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]any{ + "name": name, + "namespace": "default", + "uid": "uid-" + name, + "resourceVersion": rv, + }, + "data": map[string]any{"key": "value"}, + } + raw, _ := json.Marshal(obj) + return raw +} + +func ev(cluster, stream string, wt event.WatchType, res event.ResourceRef, obj json.RawMessage, at time.Time) *event.Record { + return &event.Record{ + ClusterID: cluster, + StreamID: stream, + Type: event.TypeEvent, + WatchType: wt, + Resource: res, + Object: obj, + ObservedAt: at, + } +} + +func baseline(cluster, stream string, at time.Time) *event.Record { + return &event.Record{ + ClusterID: cluster, + StreamID: stream, + Type: event.TypeBaseline, + ObservedAt: at, + } +} + +func gap(cluster, stream string, at time.Time) *event.Record { + return &event.Record{ + ClusterID: cluster, + StreamID: stream, + Type: event.TypeGap, + ObservedAt: at, + Gap: &event.GapInfo{Reason: "test gap"}, + } +} + +func appendRec(t *testing.T, store storage.Store, rec *event.Record) { + t.Helper() + if _, err := store.Append(context.Background(), rec); err != nil { + t.Fatalf("append: %v", err) + } +} + +func deployRef(name string) event.ResourceRef { + return event.ResourceRef{Namespace: "default", Name: name, Kind: "Deployment"} +} + +func TestSnapshotMaterialization(t *testing.T) { + store := storage.NewInMemory() + mat := NewMaterializer(store) + ctx := context.Background() + + cluster := "c1" + streamDeploy := event.Stream{ClusterID: cluster, Group: "apps", Version: "v1", Resource: "deployments", Namespace: "default"}.ID() + streamCM := event.Stream{ClusterID: cluster, Group: "", Version: "v1", Resource: "configmaps", Namespace: "default"}.ID() + streamGap := event.Stream{ClusterID: cluster, Group: "", Version: "v1", Resource: "secrets", Namespace: ""}.ID() + + base := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC) + appendRec(t, store, baseline(cluster, streamDeploy, base)) + appendRec(t, store, ev(cluster, streamDeploy, event.WatchAdded, deployRef("A"), deployObject("A", "nginx:1.20", "1"), base.Add(1*time.Minute))) + appendRec(t, store, ev(cluster, streamDeploy, event.WatchModified, deployRef("A"), deployObject("A", "nginx:1.21", "2"), base.Add(2*time.Minute))) + appendRec(t, store, ev(cluster, streamDeploy, event.WatchAdded, deployRef("B"), deployObject("B", "nginx:1.30", "1"), base.Add(3*time.Minute))) + appendRec(t, store, ev(cluster, streamDeploy, event.WatchDeleted, deployRef("B"), deployObject("B", "nginx:1.30", "1"), base.Add(4*time.Minute))) + + appendRec(t, store, baseline(cluster, streamCM, base)) + appendRec(t, store, ev(cluster, streamCM, event.WatchAdded, event.ResourceRef{Namespace: "default", Name: "C", Kind: "ConfigMap"}, configMapObject("C", "1"), base.Add(3*time.Minute+30*time.Second))) + + appendRec(t, store, gap(cluster, streamGap, base.Add(5*time.Minute))) + + snapAt := base.Add(6 * time.Minute) + snap, err := mat.Snapshot(ctx, cluster, snapAt, "daily") + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + + if len(snap.Objects) != 2 { + t.Fatalf("Snapshot objects = %d, want 2: %+v", len(snap.Objects), snap.Objects) + } + if snap.Objects[0].Name != "C" || snap.Objects[1].Name != "A" { + t.Fatalf("Snapshot objects not sorted by stream/namespace/name: %+v", snap.Objects) + } + byName := map[string]ObjectState{} + for _, o := range snap.Objects { + byName[o.Name] = o + } + if _, ok := byName["B"]; ok { + t.Fatalf("expected deleted object B to be absent") + } + a, ok := byName["A"] + if !ok { + t.Fatalf("expected object A present") + } + var aDeploy struct { + Spec struct { + Template struct { + Spec struct { + Containers []struct { + Image string `json:"image"` + } `json:"containers"` + } `json:"spec"` + } `json:"template"` + } `json:"spec"` + } + if err := json.Unmarshal(a.Object, &aDeploy); err != nil { + t.Fatalf("unmarshal A: %v", err) + } + if got := aDeploy.Spec.Template.Spec.Containers[0].Image; got != "nginx:1.21" { + t.Fatalf("A image = %q, want nginx:1.21 (modified)", got) + } + + if snap.Complete { + t.Fatalf("Snapshot Complete = true, want false due to gap") + } + if len(snap.Missing) != 1 || snap.Missing[0] != streamGap { + t.Fatalf("Missing = %v, want [%s]", snap.Missing, streamGap) + } + if !strings.Contains(snap.Warning, streamGap) { + t.Fatalf("Warning = %q, want mention of %s", snap.Warning, streamGap) + } + + snaps, err := store.Snapshots(ctx) + if err != nil { + t.Fatalf("Snapshots: %v", err) + } + if len(snaps) != 1 || snaps[0].Name != "daily" || !strings.HasPrefix(snaps[0].ID, "snap-") { + t.Fatalf("stored snapshots = %+v", snaps) + } +} + +func TestObjectAt(t *testing.T) { + store := storage.NewInMemory() + mat := NewMaterializer(store) + ctx := context.Background() + + cluster := "c1" + streamDeploy := event.Stream{ClusterID: cluster, Group: "apps", Version: "v1", Resource: "deployments", Namespace: "default"}.ID() + + base := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC) + appendRec(t, store, baseline(cluster, streamDeploy, base)) + appendRec(t, store, ev(cluster, streamDeploy, event.WatchAdded, deployRef("A"), deployObject("A", "nginx:1.20", "1"), base.Add(1*time.Minute))) + appendRec(t, store, ev(cluster, streamDeploy, event.WatchModified, deployRef("A"), deployObject("A", "nginx:1.21", "2"), base.Add(2*time.Minute))) + appendRec(t, store, ev(cluster, streamDeploy, event.WatchAdded, deployRef("B"), deployObject("B", "nginx:1.30", "1"), base.Add(3*time.Minute))) + appendRec(t, store, ev(cluster, streamDeploy, event.WatchDeleted, deployRef("B"), deployObject("B", "nginx:1.30", "1"), base.Add(4*time.Minute))) + + if _, err := mat.ObjectAt(ctx, cluster, streamDeploy, "default", "A", base.Add(-time.Second)); !errors.Is(err, ErrNoObjectAtTime) { + t.Fatalf("before creation: err = %v, want ErrNoObjectAtTime", err) + } + + image := func(rec *event.Record) string { + t.Helper() + var d struct { + Spec struct { + Template struct { + Spec struct { + Containers []struct { + Image string `json:"image"` + } `json:"containers"` + } `json:"spec"` + } `json:"template"` + } `json:"spec"` + } + if err := json.Unmarshal(rec.Object, &d); err != nil { + t.Fatalf("unmarshal: %v", err) + } + return d.Spec.Template.Spec.Containers[0].Image + } + + rec, err := mat.ObjectAt(ctx, cluster, streamDeploy, "default", "A", base.Add(1*time.Minute+30*time.Second)) + if err != nil { + t.Fatalf("ObjectAt v1: %v", err) + } + if got := image(rec); got != "nginx:1.20" { + t.Fatalf("A at v1 time = %q, want nginx:1.20", got) + } + + rec, err = mat.ObjectAt(ctx, cluster, streamDeploy, "default", "A", base.Add(2*time.Minute+30*time.Second)) + if err != nil { + t.Fatalf("ObjectAt v2: %v", err) + } + if got := image(rec); got != "nginx:1.21" { + t.Fatalf("A at v2 time = %q, want nginx:1.21", got) + } + + if _, err := mat.ObjectAt(ctx, cluster, streamDeploy, "default", "B", base.Add(5*time.Minute)); !errors.Is(err, ErrNoObjectAtTime) { + t.Fatalf("deleted object: err = %v, want ErrNoObjectAtTime", err) + } +} + +func TestDiff(t *testing.T) { + store := storage.NewInMemory() + mat := NewMaterializer(store) + ctx := context.Background() + + cluster := "c1" + streamDeploy := event.Stream{ClusterID: cluster, Group: "apps", Version: "v1", Resource: "deployments", Namespace: "default"}.ID() + streamCM := event.Stream{ClusterID: cluster, Group: "", Version: "v1", Resource: "configmaps", Namespace: "default"}.ID() + streamGap := event.Stream{ClusterID: cluster, Group: "", Version: "v1", Resource: "secrets", Namespace: ""}.ID() + + base := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC) + appendRec(t, store, baseline(cluster, streamDeploy, base)) + appendRec(t, store, ev(cluster, streamDeploy, event.WatchAdded, deployRef("A"), deployObject("A", "nginx:1.20", "1"), base.Add(1*time.Minute))) + appendRec(t, store, ev(cluster, streamDeploy, event.WatchModified, deployRef("A"), deployObject("A", "nginx:1.21", "2"), base.Add(2*time.Minute))) + appendRec(t, store, baseline(cluster, streamCM, base)) + appendRec(t, store, ev(cluster, streamCM, event.WatchAdded, event.ResourceRef{Namespace: "default", Name: "C", Kind: "ConfigMap"}, configMapObject("C", "1"), base.Add(3*time.Minute))) + appendRec(t, store, gap(cluster, streamGap, base.Add(5*time.Minute))) + + before := base.Add(1*time.Minute + 30*time.Second) + after := base.Add(6 * time.Minute) + res, err := mat.Diff(ctx, cluster, "default", before, after) + if err != nil { + t.Fatalf("Diff: %v", err) + } + + if !res.HasGaps { + t.Fatalf("Diff HasGaps = false, want true") + } + if !strings.Contains(res.Warning, streamGap) { + t.Fatalf("Diff Warning = %q, want mention of %s", res.Warning, streamGap) + } + if len(res.Changes) != 2 { + t.Fatalf("Diff changes = %d, want 2 (A modified, C added): %+v", len(res.Changes), res.Changes) + } + + var aDiff, cDiff *ObjectDiff + for i := range res.Changes { + switch res.Changes[i].Name { + case "A": + aDiff = &res.Changes[i] + case "C": + cDiff = &res.Changes[i] + } + } + if aDiff == nil { + t.Fatalf("missing diff for A") + } + if cDiff == nil { + t.Fatalf("missing diff for C") + } + + if len(aDiff.Changes) != 1 { + t.Fatalf("A changes = %+v, want exactly 1", aDiff.Changes) + } + ch := aDiff.Changes[0] + if ch.Path != "spec.template.spec.containers[0].image" { + t.Fatalf("A change path = %q, want spec.template.spec.containers[0].image", ch.Path) + } + if ch.Before != "nginx:1.20" || ch.After != "nginx:1.21" { + t.Fatalf("A change = before %v, after %v", ch.Before, ch.After) + } + for _, skip := range []string{"resourceVersion", "generation", "managedFields", "uid", "status", "creationTimestamp"} { + if strings.Contains(ch.Path, skip) { + t.Fatalf("A change path %q should ignore %q", ch.Path, skip) + } + } + + if len(cDiff.Changes) != 1 || !cDiff.Changes[0].Added || cDiff.Changes[0].Path != "" { + t.Fatalf("C changes = %+v, want single whole-object Added", cDiff.Changes) + } +} diff --git a/internal/materialize/snapshot.go b/internal/materialize/snapshot.go new file mode 100644 index 0000000..3e470cc --- /dev/null +++ b/internal/materialize/snapshot.go @@ -0,0 +1,80 @@ +package materialize + +import ( + "context" + "sort" + "strings" + "time" + + "github.com/google/uuid" + "github.com/krply/krply/internal/storage" +) + +// Snapshot is a materialized view of a cluster at a point in time. +type Snapshot struct { + ID string + ClusterID string + Name string + At time.Time + Objects []ObjectState + Streams []string + Complete bool + Missing []string + Warning string +} + +// Snapshot materializes every watched object of the cluster at at, records +// per-stream coverage, and persists the snapshot metadata. The materialized +// objects are reconstructable on demand; the snapshot reference is durable. +func (m *Materializer) Snapshot(ctx context.Context, clusterID string, at time.Time, name string) (*Snapshot, error) { + streams, err := m.store.Streams(ctx) + if err != nil { + return nil, err + } + + snap := &Snapshot{ + ID: "snap-" + uuid.NewString()[:8] + "-" + name, + ClusterID: clusterID, + Name: name, + At: at, + Complete: true, + } + + var missing []string + for _, s := range streams { + if s.ClusterID != clusterID { + continue + } + ss, err := m.StreamState(ctx, s.StreamID, at) + if err != nil { + return nil, err + } + snap.Objects = append(snap.Objects, ss.Objects...) + snap.Streams = append(snap.Streams, s.StreamID) + + if !ss.HasBaseline || ss.HasGaps { + missing = append(missing, s.StreamID) + } + } + + sort.Strings(snap.Streams) + sortObjects(snap.Objects) + sort.Strings(missing) + + if len(missing) > 0 { + snap.Complete = false + snap.Missing = missing + snap.Warning = "coverage incomplete for stream(s): " + strings.Join(missing, ", ") + } + + if err := m.store.SaveSnapshot(ctx, &storage.SnapshotRef{ + ID: snap.ID, + ClusterID: clusterID, + Name: name, + At: at, + }); err != nil { + return nil, err + } + + return snap, nil +} diff --git a/internal/storage/memory.go b/internal/storage/memory.go new file mode 100644 index 0000000..bd0c995 --- /dev/null +++ b/internal/storage/memory.go @@ -0,0 +1,300 @@ +package storage + +import ( + "context" + "sort" + "sync" + "time" + + "github.com/krply/krply/internal/event" +) + +// NewInMemory returns a Store backed by RAM. It is for tests and for running +// the query API without a persistent journal. It satisfies the same atomic +// append+checkpoint contract as the SQLite store. +func NewInMemory() Store { + m := &memoryStore{ + records: map[string][]*event.Record{}, + seq: 0, + streams: map[string]*StreamMeta{}, + snapshots: map[string]*SnapshotRef{}, + } + return m +} + +type memoryStore struct { + mu sync.Mutex + records map[string][]*event.Record // streamID -> records in ingest order + seq int64 + streams map[string]*StreamMeta + snapshots map[string]*SnapshotRef +} + +func (m *memoryStore) Append(ctx context.Context, rec *event.Record) (int64, error) { + m.mu.Lock() + defer m.mu.Unlock() + return m.appendLocked(rec) +} + +func (m *memoryStore) Appends(ctx context.Context, recs []*event.Record) ([]int64, error) { + m.mu.Lock() + defer m.mu.Unlock() + seqs := make([]int64, 0, len(recs)) + for _, r := range recs { + s, err := m.appendLocked(r) + if err != nil { + return nil, err + } + seqs = append(seqs, s) + } + return seqs, nil +} + +func (m *memoryStore) appendLocked(rec *event.Record) (int64, error) { + m.seq++ + rec.IngestSeq = m.seq + if rec.ObservedAt.IsZero() { + rec.ObservedAt = time.Now().UTC() + } + m.records[rec.StreamID] = append(m.records[rec.StreamID], rec) + m.updateMeta(rec) + return m.seq, nil +} + +func (m *memoryStore) updateMeta(rec *event.Record) { + meta := m.streams[rec.StreamID] + if meta == nil { + meta = &StreamMeta{StreamID: rec.StreamID, ClusterID: rec.ClusterID} + m.streams[rec.StreamID] = meta + } + if meta.FirstObservedAt.IsZero() { + meta.FirstObservedAt = rec.ObservedAt + } + if rec.ObservedAt.After(meta.LastObservedAt) { + meta.LastObservedAt = rec.ObservedAt + } + switch rec.Type { + case event.TypeGap: + meta.GapCount++ + meta.HasGaps = true + meta.Degraded = true + case event.TypeBaseline: + meta.Available = true + meta.Degraded = false + if rec.Resource.ResourceVersion != "" { + meta.LastResourceVersion = rec.Resource.ResourceVersion + } + case event.TypeCoverageChange: + if rec.Coverage != nil { + meta.Available = rec.Coverage.Current.Available + if !meta.Available { + meta.Degraded = true + } + } + case event.TypeCheckpoint: + if rec.Checkpoint != nil { + meta.LastResourceVersion = rec.Checkpoint.ResourceVersion + } + case event.TypeEvent: + if rec.Resource.ResourceVersion != "" { + meta.LastResourceVersion = rec.Resource.ResourceVersion + } + } +} + +func (m *memoryStore) ListClusters(ctx context.Context) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + seen := map[string]bool{} + var out []string + for _, recs := range m.records { + for _, r := range recs { + if r.ClusterID != "" && !seen[r.ClusterID] { + seen[r.ClusterID] = true + out = append(out, r.ClusterID) + } + } + } + sort.Strings(out) + return out, nil +} + +func (m *memoryStore) Streams(ctx context.Context) ([]StreamMeta, error) { + m.mu.Lock() + defer m.mu.Unlock() + out := make([]StreamMeta, 0, len(m.streams)) + for _, s := range m.streams { + out = append(out, *s) + } + sort.Slice(out, func(i, j int) bool { return out[i].StreamID < out[j].StreamID }) + return out, nil +} + +func (m *memoryStore) StreamMeta(ctx context.Context, streamID string) (StreamMeta, error) { + m.mu.Lock() + defer m.mu.Unlock() + s, ok := m.streams[streamID] + if !ok { + return StreamMeta{}, ErrStreamNotFound + } + return *s, nil +} + +func (m *memoryStore) Events(ctx context.Context, f EventFilter) ([]event.Record, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []event.Record + for _, recs := range m.records { + for _, r := range recs { + if !matches(r, f) { + continue + } + out = append(out, *r) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].IngestSeq < out[j].IngestSeq }) + return out, nil +} + +func (m *memoryStore) ObjectHistory(ctx context.Context, ref ObjectRef) ([]event.Record, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []event.Record + for _, recs := range m.records { + for _, r := range recs { + if r.Type != event.TypeEvent { + continue + } + if r.ClusterID != ref.ClusterID || r.StreamID != ref.StreamID { + continue + } + if r.Resource.Namespace != ref.Namespace || r.Resource.Name != ref.Name { + continue + } + out = append(out, *r) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].IngestSeq < out[j].IngestSeq }) + return out, nil +} + +func (m *memoryStore) ObjectAt(ctx context.Context, ref ObjectRef, ts time.Time) (*event.Record, error) { + recs, err := m.ObjectHistory(ctx, ref) + if err != nil { + return nil, err + } + var last *event.Record + for i := range recs { + if recs[i].ObservedAt.After(ts) { + break + } + if recs[i].WatchType == event.WatchDeleted { + last = nil + continue + } + cp := recs[i] + last = &cp + } + if last == nil { + return nil, ErrNotFound + } + return last, nil +} + +func (m *memoryStore) StreamEvents(ctx context.Context, streamID string, until time.Time) ([]event.Record, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []event.Record + for _, r := range m.records[streamID] { + if until.IsZero() || !r.ObservedAt.After(until) { + out = append(out, *r) + } + } + return out, nil +} + +func (m *memoryStore) Baselines(ctx context.Context, streamID string) ([]event.Record, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []event.Record + for _, r := range m.records[streamID] { + if r.Type == event.TypeBaseline { + out = append(out, *r) + } + } + return out, nil +} + +func (m *memoryStore) Gaps(ctx context.Context, streamID string) ([]event.Record, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []event.Record + for _, r := range m.records[streamID] { + if r.Type == event.TypeGap { + out = append(out, *r) + } + } + return out, nil +} + +func (m *memoryStore) SaveSnapshot(ctx context.Context, snap *SnapshotRef) error { + m.mu.Lock() + defer m.mu.Unlock() + cp := *snap + if cp.ID == "" { + cp.ID = "snap-" + time.Now().UTC().Format("20060102T150405") + } + m.snapshots[cp.ID] = &cp + return nil +} + +func (m *memoryStore) Snapshots(ctx context.Context) ([]SnapshotRef, error) { + m.mu.Lock() + defer m.mu.Unlock() + out := make([]SnapshotRef, 0, len(m.snapshots)) + for _, s := range m.snapshots { + out = append(out, *s) + } + sort.Slice(out, func(i, j int) bool { return out[i].At.Before(out[j].At) }) + return out, nil +} + +func (m *memoryStore) Close() error { return nil } + +func matches(r *event.Record, f EventFilter) bool { + if f.ClusterID != "" && r.ClusterID != f.ClusterID { + return false + } + if f.StreamID != "" && r.StreamID != f.StreamID { + return false + } + if f.RecordType != "" && r.Type != f.RecordType { + return false + } + if f.Namespace != "" && r.Resource.Namespace != f.Namespace { + return false + } + if f.Name != "" && r.Resource.Name != f.Name { + return false + } + if f.Kind != "" && r.Resource.Kind != f.Kind { + return false + } + if !f.Since.IsZero() && r.ObservedAt.Before(f.Since) { + return false + } + if !f.Until.IsZero() && r.ObservedAt.After(f.Until) { + return false + } + return true +} + +// Sentinel errors shared by all Store implementations. +var ( + ErrStreamNotFound = &storeError{"stream not found"} + ErrNotFound = &storeError{"not found"} +) + +type storeError struct{ msg string } + +func (e *storeError) Error() string { return e.msg } diff --git a/internal/storage/sqlite.go b/internal/storage/sqlite.go new file mode 100644 index 0000000..8a62b95 --- /dev/null +++ b/internal/storage/sqlite.go @@ -0,0 +1,685 @@ +package storage + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "sync" + "time" + + _ "modernc.org/sqlite" + + "github.com/krply/krply/internal/event" +) + +// sqliteStore is the durable, persistent Store backed by SQLite. +// +// Writes are serialized through a single mutex so that append and its +// checkpoint (stream meta) advance atomically and never interleave. Reads +// run concurrently and do not need the lock. +type sqliteStore struct { + db *sql.DB + mu sync.Mutex +} + +// NewSQLiteStore opens (or creates) a persistent SQLite store at path and +// returns it as a Store. The special path ":memory:" yields an in-memory +// database. WAL mode is enabled via the DSN pragmas. +func NewSQLiteStore(path string) (Store, error) { + dsn := ":memory:" + if path != ":memory:" { + dsn = "file:" + path + "?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=synchronous(NORMAL)" + } + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, fmt.Errorf("open sqlite store %q: %w", path, err) + } + if path == ":memory:" { + // Each pooled connection to ":memory:" is its own database, so pin the + // pool to a single connection. + db.SetMaxOpenConns(1) + } + s := &sqliteStore{db: db} + if err := s.init(context.Background()); err != nil { + db.Close() + return nil, fmt.Errorf("init sqlite store: %w", err) + } + return s, nil +} + +const ( + recordsDDL = `CREATE TABLE IF NOT EXISTS records ( + ingest_seq INTEGER PRIMARY KEY AUTOINCREMENT, + cluster_id TEXT, + stream_id TEXT, + record_type TEXT, + event_id TEXT, + observed_at TEXT, + watch_type TEXT, + synthetic INTEGER, + grp TEXT, + version TEXT, + kind TEXT, + namespace TEXT, + name TEXT, + uid TEXT, + resource_version TEXT, + object_hash TEXT, + object BLOB, + provenance TEXT, + gap TEXT, + coverage TEXT, + checkpoint TEXT, + snapshot TEXT + )` + + dedupIndexDDL = `CREATE UNIQUE INDEX IF NOT EXISTS idx_records_dedup + ON records (cluster_id, stream_id, event_id) WHERE event_id <> ''` + + // dedupIndexDrop removes a pre-migration index that was not partial. A + // unique index over empty event_id would collapse all special records on a + // stream (baselines, gaps, checkpoints) into one row. + dedupIndexDrop = `DROP INDEX IF EXISTS idx_records_dedup` + + lookupIndexDDL = `CREATE INDEX IF NOT EXISTS idx_records_lookup + ON records (cluster_id, stream_id, observed_at, namespace, name)` + + streamsDDL = `CREATE TABLE IF NOT EXISTS streams ( + stream_id TEXT PRIMARY KEY, + cluster_id TEXT, + grp TEXT, + version TEXT, + resource TEXT, + kind TEXT, + namespace TEXT, + selector TEXT, + available INTEGER, + first_observed_at TEXT, + last_observed_at TEXT, + last_resource_version TEXT, + gap_count INTEGER, + degraded INTEGER + )` + + snapshotsDDL = `CREATE TABLE IF NOT EXISTS snapshots ( + id TEXT PRIMARY KEY, + cluster_id TEXT, + name TEXT, + at TEXT + )` +) + +func (s *sqliteStore) init(ctx context.Context) error { + for _, ddl := range []string{recordsDDL, dedupIndexDrop, dedupIndexDDL, lookupIndexDDL, streamsDDL, snapshotsDDL} { + if _, err := s.db.ExecContext(ctx, ddl); err != nil { + return err + } + } + return nil +} + +// recordCols is the SELECT column list for the records table in schema order. +const recordCols = `ingest_seq, cluster_id, stream_id, record_type, event_id, observed_at, + watch_type, synthetic, grp, version, kind, namespace, name, uid, resource_version, + object_hash, object, provenance, gap, coverage, checkpoint, snapshot` + +// streamCols is the SELECT column list for the streams meta table. +const streamCols = `stream_id, cluster_id, grp, version, resource, kind, namespace, selector, + available, first_observed_at, last_observed_at, last_resource_version, gap_count, degraded` + +func (s *sqliteStore) Append(ctx context.Context, rec *event.Record) (int64, error) { + s.mu.Lock() + defer s.mu.Unlock() + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + seq, err := s.appendRecord(ctx, tx, rec) + if err != nil { + tx.Rollback() + return 0, err + } + if err := tx.Commit(); err != nil { + return 0, err + } + return seq, nil +} + +func (s *sqliteStore) Appends(ctx context.Context, recs []*event.Record) ([]int64, error) { + s.mu.Lock() + defer s.mu.Unlock() + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + seqs := make([]int64, 0, len(recs)) + for _, r := range recs { + seq, err := s.appendRecord(ctx, tx, r) + if err != nil { + tx.Rollback() + return nil, err + } + seqs = append(seqs, seq) + } + if err := tx.Commit(); err != nil { + return nil, err + } + return seqs, nil +} + +func (s *sqliteStore) appendRecord(ctx context.Context, tx *sql.Tx, rec *event.Record) (int64, error) { + if rec.ObservedAt.IsZero() { + rec.ObservedAt = time.Now().UTC() + } + observedAt := formatTime(rec.ObservedAt) + synthetic := 0 + if rec.Synthetic { + synthetic = 1 + } + var object []byte + if len(rec.Object) > 0 { + object = rec.Object + } + provenance, err := marshalJSON(rec.Provenance) + if err != nil { + return 0, err + } + gap, err := marshalJSON(rec.Gap) + if err != nil { + return 0, err + } + coverage, err := marshalJSON(rec.Coverage) + if err != nil { + return 0, err + } + checkpoint, err := marshalJSON(rec.Checkpoint) + if err != nil { + return 0, err + } + snapshot, err := marshalJSON(rec.Snapshot) + if err != nil { + return 0, err + } + + res, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO records ( + cluster_id, stream_id, record_type, event_id, observed_at, watch_type, synthetic, + grp, version, kind, namespace, name, uid, resource_version, object_hash, object, + provenance, gap, coverage, checkpoint, snapshot + ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, + rec.ClusterID, rec.StreamID, string(rec.Type), rec.EventID, observedAt, + string(rec.WatchType), synthetic, + rec.Resource.Group, rec.Resource.Version, rec.Resource.Kind, rec.Resource.Namespace, + rec.Resource.Name, rec.Resource.UID, rec.Resource.ResourceVersion, rec.ObjectHash, object, + provenance, gap, coverage, checkpoint, snapshot, + ) + if err != nil { + return 0, err + } + + inserted, err := res.RowsAffected() + if err != nil { + return 0, err + } + + var seq int64 + if inserted > 0 { + if err := tx.QueryRowContext(ctx, `SELECT last_insert_rowid()`).Scan(&seq); err != nil { + return 0, err + } + if err := s.upsertMeta(ctx, tx, rec, observedAt); err != nil { + return 0, err + } + } else { + // Duplicate event_id delivery: re-return the existing ingest sequence. + if err := tx.QueryRowContext(ctx, `SELECT ingest_seq FROM records + WHERE cluster_id=? AND stream_id=? AND event_id=?`, + rec.ClusterID, rec.StreamID, rec.EventID).Scan(&seq); err != nil { + return 0, err + } + } + rec.IngestSeq = seq + return seq, nil +} + +// upsertMeta mirrors the checkpoint-advance logic of the in-memory store's +// updateMeta, but persisted. It runs inside the same transaction as the +// record insert so the two are atomic. +func (s *sqliteStore) upsertMeta(ctx context.Context, tx *sql.Tx, rec *event.Record, observedAt string) error { + var ( + available, degraded int + gapCount int64 + first, last, lastRV string + ) + err := tx.QueryRowContext(ctx, `SELECT available, degraded, gap_count, first_observed_at, + last_observed_at, last_resource_version FROM streams WHERE stream_id=?`, rec.StreamID). + Scan(&available, °raded, &gapCount, &first, &last, &lastRV) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + + if first == "" { + first = observedAt + } + if last == "" || observedAt > last { + last = observedAt + } + + switch rec.Type { + case event.TypeGap: + gapCount++ + degraded = 1 + case event.TypeBaseline: + available = 1 + degraded = 0 + if rec.Resource.ResourceVersion != "" { + lastRV = rec.Resource.ResourceVersion + } + case event.TypeCoverageChange: + if rec.Coverage != nil { + if rec.Coverage.Current.Available { + available = 1 + } else { + available = 0 + degraded = 1 + } + } + case event.TypeCheckpoint: + if rec.Checkpoint != nil { + lastRV = rec.Checkpoint.ResourceVersion + } + case event.TypeEvent: + if rec.Resource.ResourceVersion != "" { + lastRV = rec.Resource.ResourceVersion + } + } + + grp, version, resource, kind, namespace, selector := metaParts(rec.StreamID, rec) + + _, err = tx.ExecContext(ctx, `INSERT INTO streams ( + stream_id, cluster_id, grp, version, resource, kind, namespace, selector, + available, first_observed_at, last_observed_at, last_resource_version, gap_count, degraded + ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?) + ON CONFLICT(stream_id) DO UPDATE SET + cluster_id=excluded.cluster_id, + grp=excluded.grp, + version=excluded.version, + resource=excluded.resource, + kind=excluded.kind, + namespace=excluded.namespace, + selector=excluded.selector, + available=excluded.available, + first_observed_at=excluded.first_observed_at, + last_observed_at=excluded.last_observed_at, + last_resource_version=excluded.last_resource_version, + gap_count=excluded.gap_count, + degraded=excluded.degraded`, + rec.StreamID, rec.ClusterID, grp, version, resource, kind, namespace, selector, + available, first, last, lastRV, gapCount, degraded, + ) + return err +} + +func metaParts(streamID string, rec *event.Record) (grp, version, resource, kind, namespace, selector string) { + if st, err := event.StreamID(streamID); err == nil { + return st.Group, st.Version, st.Resource, rec.Resource.Kind, st.Namespace, st.Selector + } + return rec.Resource.Group, rec.Resource.Version, "", rec.Resource.Kind, rec.Resource.Namespace, "" +} + +func (s *sqliteStore) ListClusters(ctx context.Context) ([]string, error) { + rows, err := s.db.QueryContext(ctx, `SELECT DISTINCT cluster_id FROM streams + WHERE cluster_id<>'' ORDER BY cluster_id`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []string + for rows.Next() { + var c string + if err := rows.Scan(&c); err != nil { + return nil, err + } + out = append(out, c) + } + return out, rows.Err() +} + +func (s *sqliteStore) Streams(ctx context.Context) ([]StreamMeta, error) { + rows, err := s.db.QueryContext(ctx, `SELECT `+streamCols+` FROM streams ORDER BY stream_id`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []StreamMeta + for rows.Next() { + m, err := scanStreamMeta(rows.Scan) + if err != nil { + return nil, err + } + out = append(out, m) + } + return out, rows.Err() +} + +func (s *sqliteStore) StreamMeta(ctx context.Context, streamID string) (StreamMeta, error) { + row := s.db.QueryRowContext(ctx, `SELECT `+streamCols+` FROM streams WHERE stream_id=?`, streamID) + m, err := scanStreamMeta(row.Scan) + if errors.Is(err, sql.ErrNoRows) { + return StreamMeta{}, ErrStreamNotFound + } + if err != nil { + return StreamMeta{}, err + } + return m, nil +} + +func (s *sqliteStore) Events(ctx context.Context, f EventFilter) ([]event.Record, error) { + q := `SELECT ` + recordCols + ` FROM records WHERE 1=1` + var args []any + if f.ClusterID != "" { + q += " AND cluster_id=?" + args = append(args, f.ClusterID) + } + if f.StreamID != "" { + q += " AND stream_id=?" + args = append(args, f.StreamID) + } + if f.RecordType != "" { + q += " AND record_type=?" + args = append(args, string(f.RecordType)) + } + if f.Namespace != "" { + q += " AND namespace=?" + args = append(args, f.Namespace) + } + if f.Name != "" { + q += " AND name=?" + args = append(args, f.Name) + } + if f.Kind != "" { + q += " AND kind=?" + args = append(args, f.Kind) + } + if !f.Since.IsZero() { + q += " AND observed_at>=?" + args = append(args, formatTime(f.Since)) + } + if !f.Until.IsZero() { + q += " AND observed_at<=?" + args = append(args, formatTime(f.Until)) + } + q += " ORDER BY ingest_seq ASC" + if f.Limit > 0 { + q += " LIMIT ?" + args = append(args, f.Limit) + if f.Offset > 0 { + q += " OFFSET ?" + args = append(args, f.Offset) + } + } else if f.Offset > 0 { + q += " LIMIT -1 OFFSET ?" + args = append(args, f.Offset) + } + + rows, err := s.db.QueryContext(ctx, q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []event.Record + for rows.Next() { + r, err := scanRecord(rows.Scan) + if err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +func (s *sqliteStore) ObjectHistory(ctx context.Context, ref ObjectRef) ([]event.Record, error) { + rows, err := s.db.QueryContext(ctx, `SELECT `+recordCols+` FROM records + WHERE record_type=? AND cluster_id=? AND stream_id=? AND namespace=? AND name=? + ORDER BY ingest_seq ASC`, + string(event.TypeEvent), ref.ClusterID, ref.StreamID, ref.Namespace, ref.Name) + if err != nil { + return nil, err + } + defer rows.Close() + var out []event.Record + for rows.Next() { + r, err := scanRecord(rows.Scan) + if err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +func (s *sqliteStore) ObjectAt(ctx context.Context, ref ObjectRef, ts time.Time) (*event.Record, error) { + rows, err := s.db.QueryContext(ctx, `SELECT `+recordCols+` FROM records + WHERE record_type=? AND cluster_id=? AND stream_id=? AND namespace=? AND name=? AND observed_at<=? + ORDER BY ingest_seq ASC`, + string(event.TypeEvent), ref.ClusterID, ref.StreamID, ref.Namespace, ref.Name, formatTime(ts)) + if err != nil { + return nil, err + } + defer rows.Close() + var last *event.Record + for rows.Next() { + r, err := scanRecord(rows.Scan) + if err != nil { + return nil, err + } + if r.WatchType == event.WatchDeleted { + last = nil + continue + } + cp := r + last = &cp + } + if err := rows.Err(); err != nil { + return nil, err + } + if last == nil { + return nil, ErrNotFound + } + return last, nil +} + +func (s *sqliteStore) StreamEvents(ctx context.Context, streamID string, until time.Time) ([]event.Record, error) { + q := `SELECT ` + recordCols + ` FROM records WHERE stream_id=?` + args := []any{streamID} + if !until.IsZero() { + q += " AND observed_at<=?" + args = append(args, formatTime(until)) + } + q += " ORDER BY ingest_seq ASC" + rows, err := s.db.QueryContext(ctx, q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []event.Record + for rows.Next() { + r, err := scanRecord(rows.Scan) + if err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +func (s *sqliteStore) Baselines(ctx context.Context, streamID string) ([]event.Record, error) { + return s.recordsByType(ctx, streamID, event.TypeBaseline) +} + +func (s *sqliteStore) Gaps(ctx context.Context, streamID string) ([]event.Record, error) { + return s.recordsByType(ctx, streamID, event.TypeGap) +} + +func (s *sqliteStore) recordsByType(ctx context.Context, streamID string, typ event.RecordType) ([]event.Record, error) { + rows, err := s.db.QueryContext(ctx, `SELECT `+recordCols+` FROM records + WHERE stream_id=? AND record_type=? ORDER BY ingest_seq ASC`, streamID, string(typ)) + if err != nil { + return nil, err + } + defer rows.Close() + var out []event.Record + for rows.Next() { + r, err := scanRecord(rows.Scan) + if err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +func (s *sqliteStore) SaveSnapshot(ctx context.Context, snap *SnapshotRef) error { + s.mu.Lock() + defer s.mu.Unlock() + cp := *snap + if cp.ID == "" { + cp.ID = "snap-" + time.Now().UTC().Format("20060102T150405") + } + _, err := s.db.ExecContext(ctx, `INSERT INTO snapshots (id, cluster_id, name, at) + VALUES (?,?,?,?) + ON CONFLICT(id) DO UPDATE SET cluster_id=excluded.cluster_id, name=excluded.name, at=excluded.at`, + cp.ID, cp.ClusterID, cp.Name, formatTime(cp.At)) + return err +} + +func (s *sqliteStore) Snapshots(ctx context.Context) ([]SnapshotRef, error) { + rows, err := s.db.QueryContext(ctx, `SELECT id, cluster_id, name, at FROM snapshots ORDER BY at`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []SnapshotRef + for rows.Next() { + var snap SnapshotRef + var at string + if err := rows.Scan(&snap.ID, &snap.ClusterID, &snap.Name, &at); err != nil { + return nil, err + } + t, err := parseTime(at) + if err != nil { + return nil, err + } + snap.At = t + out = append(out, snap) + } + return out, rows.Err() +} + +func (s *sqliteStore) Close() error { return s.db.Close() } + +// scanFunc matches both *sql.Rows.Scan and *sql.Row.Scan. +type scanFunc func(dest ...any) error + +func scanRecord(scan scanFunc) (event.Record, error) { + var r event.Record + var ( + observedAt, watchType string + synthetic int + grp, version, kind, namespace, name, uid, resourceVersion, objectHash string + object []byte + provenance, gap, coverage, checkpoint, snapshot string + ) + if err := scan(&r.IngestSeq, &r.ClusterID, &r.StreamID, &r.Type, &r.EventID, &observedAt, + &watchType, &synthetic, &grp, &version, &kind, &namespace, &name, &uid, &resourceVersion, + &objectHash, &object, &provenance, &gap, &coverage, &checkpoint, &snapshot); err != nil { + return event.Record{}, err + } + t, err := parseTime(observedAt) + if err != nil { + return event.Record{}, err + } + r.ObservedAt = t + r.WatchType = event.WatchType(watchType) + r.Synthetic = synthetic != 0 + r.Resource = event.ResourceRef{ + Group: grp, Version: version, Kind: kind, Namespace: namespace, Name: name, + UID: uid, ResourceVersion: resourceVersion, + } + r.ObjectHash = objectHash + r.Object = json.RawMessage(object) + if err := unmarshalJSON(provenance, &r.Provenance); err != nil { + return event.Record{}, err + } + if err := unmarshalJSON(gap, &r.Gap); err != nil { + return event.Record{}, err + } + if err := unmarshalJSON(coverage, &r.Coverage); err != nil { + return event.Record{}, err + } + if err := unmarshalJSON(checkpoint, &r.Checkpoint); err != nil { + return event.Record{}, err + } + if err := unmarshalJSON(snapshot, &r.Snapshot); err != nil { + return event.Record{}, err + } + return r, nil +} + +func scanStreamMeta(scan scanFunc) (StreamMeta, error) { + var m StreamMeta + var available, degraded int + var first, last string + if err := scan(&m.StreamID, &m.ClusterID, &m.Group, &m.Version, &m.Resource, &m.Kind, + &m.Namespace, &m.Selector, &available, &first, &last, &m.LastResourceVersion, + &m.GapCount, °raded); err != nil { + return StreamMeta{}, err + } + m.Available = available != 0 + m.Degraded = degraded != 0 + m.HasGaps = m.GapCount > 0 + t, err := parseTime(first) + if err != nil { + return StreamMeta{}, err + } + m.FirstObservedAt = t + t, err = parseTime(last) + if err != nil { + return StreamMeta{}, err + } + m.LastObservedAt = t + return m, nil +} + +func formatTime(t time.Time) string { + if t.IsZero() { + return "" + } + return t.UTC().Format(time.RFC3339Nano) +} + +func parseTime(s string) (time.Time, error) { + if s == "" { + return time.Time{}, nil + } + return time.Parse(time.RFC3339Nano, s) +} + +func marshalJSON(v any) (string, error) { + if v == nil { + return "", nil + } + b, err := json.Marshal(v) + if err != nil { + return "", err + } + return string(b), nil +} + +func unmarshalJSON(s string, v any) error { + if s == "" { + return nil + } + return json.Unmarshal([]byte(s), v) +} diff --git a/internal/storage/sqlite_test.go b/internal/storage/sqlite_test.go new file mode 100644 index 0000000..b6d5e18 --- /dev/null +++ b/internal/storage/sqlite_test.go @@ -0,0 +1,408 @@ +package storage + +import ( + "context" + "encoding/json" + "path/filepath" + "testing" + "time" + + "github.com/krply/krply/internal/event" +) + +func TestSQLiteStore(t *testing.T) { + store, err := NewSQLiteStore(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("NewSQLiteStore: %v", err) + } + defer store.Close() + + ctx := context.Background() + cluster := "cluster-a" + stream := event.Stream{ + ClusterID: cluster, + Group: "apps", + Version: "v1", + Resource: "deployments", + Namespace: "default", + } + streamID := stream.ID() + + t0 := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC) + t1 := t0.Add(time.Minute) + t2 := t0.Add(2 * time.Minute) + t3 := t0.Add(3 * time.Minute) + t4 := t0.Add(4 * time.Minute) + t5 := t0.Add(5 * time.Minute) + + mkEvent := func(id, name, rv string, wt event.WatchType, at time.Time) *event.Record { + obj := json.RawMessage(`{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"name":"` + + name + `","namespace":"default","resourceVersion":"` + rv + `","uid":"uid-` + id + `"}}`) + return &event.Record{ + ClusterID: cluster, + StreamID: streamID, + Type: event.TypeEvent, + EventID: id, + ObservedAt: at, + WatchType: wt, + Resource: event.ResourceRef{ + Group: "apps", Version: "v1", Kind: "Deployment", + Namespace: "default", Name: name, UID: "uid-" + id, ResourceVersion: rv, + }, + ObjectHash: event.ObjectHash(obj), + Object: obj, + } + } + + e1 := mkEvent("e1", "web-0", "1", event.WatchAdded, t0) + e2 := mkEvent("e2", "web-0", "2", event.WatchModified, t1) + e3 := mkEvent("e3", "web-0", "3", event.WatchDeleted, t2) + e4 := mkEvent("e4", "db-0", "1", event.WatchAdded, t3) + + seqs, err := store.Appends(ctx, []*event.Record{e1, e2, e3, e4}) + if err != nil { + t.Fatalf("Appends: %v", err) + } + if len(seqs) != 4 { + t.Fatalf("Appends returned %d seqs, want 4", len(seqs)) + } + for i := 1; i < len(seqs); i++ { + if seqs[i] <= seqs[i-1] { + t.Fatalf("seqs not increasing: %v", seqs) + } + } + + gapRec := &event.Record{ + ClusterID: cluster, + StreamID: streamID, + Type: event.TypeGap, + EventID: "gap-1", + ObservedAt: t4, + Gap: &event.GapInfo{ + FromResourceVersion: "3", + ToResourceVersion: "9", + Reason: "410 Gone", + DetectedAt: t4, + }, + Resource: event.ResourceRef{Group: "apps", Version: "v1", Kind: "Deployment"}, + } + baseRec := &event.Record{ + ClusterID: cluster, + StreamID: streamID, + Type: event.TypeBaseline, + EventID: "base-1", + ObservedAt: t5, + Synthetic: true, + Resource: event.ResourceRef{ + Group: "apps", Version: "v1", Kind: "Deployment", ResourceVersion: "9", + }, + } + if _, err := store.Append(ctx, gapRec); err != nil { + t.Fatalf("Append gap: %v", err) + } + if _, err := store.Append(ctx, baseRec); err != nil { + t.Fatalf("Append baseline: %v", err) + } + + t.Run("dedup", func(t *testing.T) { + dup := mkEvent("e2", "web-0", "2", event.WatchModified, t1) + seq, err := store.Append(ctx, dup) + if err != nil { + t.Fatalf("Append duplicate: %v", err) + } + if seq != seqs[1] { + t.Errorf("duplicate seq = %d, want %d", seq, seqs[1]) + } + recs, err := store.Events(ctx, EventFilter{StreamID: streamID, RecordType: event.TypeEvent}) + if err != nil { + t.Fatalf("Events: %v", err) + } + count := 0 + for _, r := range recs { + if r.EventID == "e2" { + count++ + } + } + if count != 1 { + t.Errorf("duplicate event_id appears %d times, want 1", count) + } + }) + + t.Run("events_filter", func(t *testing.T) { + recs, err := store.Events(ctx, EventFilter{ + ClusterID: cluster, + StreamID: streamID, + Namespace: "default", + Name: "web-0", + }) + if err != nil { + t.Fatalf("Events: %v", err) + } + if len(recs) != 3 { + t.Fatalf("Events returned %d records, want 3", len(recs)) + } + for i, want := range []string{"e1", "e2", "e3"} { + if recs[i].EventID != want { + t.Errorf("Events[%d].EventID = %s, want %s", i, recs[i].EventID, want) + } + if recs[i].IngestSeq <= 0 { + t.Errorf("Events[%d].IngestSeq = %d, want > 0", i, recs[i].IngestSeq) + } + } + + window, err := store.Events(ctx, EventFilter{ + StreamID: streamID, + Namespace: "default", + Name: "web-0", + Since: t0, + Until: t1, + }) + if err != nil { + t.Fatalf("Events window: %v", err) + } + if len(window) != 2 || window[0].EventID != "e1" || window[1].EventID != "e2" { + t.Errorf("Events window = %v, want e1,e2", window) + } + + onlyGap, err := store.Events(ctx, EventFilter{StreamID: streamID, RecordType: event.TypeGap}) + if err != nil { + t.Fatalf("Events gap type: %v", err) + } + if len(onlyGap) != 1 || onlyGap[0].EventID != "gap-1" { + t.Errorf("Events gap type = %v, want [gap-1]", onlyGap) + } + }) + + t.Run("object_history", func(t *testing.T) { + hist, err := store.ObjectHistory(ctx, ObjectRef{ + ClusterID: cluster, + StreamID: streamID, + Namespace: "default", + Name: "web-0", + }) + if err != nil { + t.Fatalf("ObjectHistory: %v", err) + } + if len(hist) != 3 { + t.Fatalf("ObjectHistory returned %d records, want 3", len(hist)) + } + for i, want := range []string{"e1", "e2", "e3"} { + if hist[i].EventID != want { + t.Errorf("hist[%d].EventID = %s, want %s", i, hist[i].EventID, want) + } + } + if hist[0].IngestSeq >= hist[1].IngestSeq || hist[1].IngestSeq >= hist[2].IngestSeq { + t.Errorf("history not in ingest order: %v, %v, %v", + hist[0].IngestSeq, hist[1].IngestSeq, hist[2].IngestSeq) + } + }) + + t.Run("object_at", func(t *testing.T) { + ref := ObjectRef{ClusterID: cluster, StreamID: streamID, Namespace: "default", Name: "web-0"} + + atT0, err := store.ObjectAt(ctx, ref, t0) + if err != nil { + t.Fatalf("ObjectAt t0: %v", err) + } + if atT0.EventID != "e1" { + t.Errorf("ObjectAt t0 = %s, want e1", atT0.EventID) + } + + atT1, err := store.ObjectAt(ctx, ref, t1) + if err != nil { + t.Fatalf("ObjectAt t1: %v", err) + } + if atT1.EventID != "e2" { + t.Errorf("ObjectAt t1 = %s, want e2", atT1.EventID) + } + + if _, err := store.ObjectAt(ctx, ref, t2); err != ErrNotFound { + t.Errorf("ObjectAt t2 err = %v, want ErrNotFound (deleted)", err) + } + if _, err := store.ObjectAt(ctx, ref, t0.Add(-time.Second)); err != ErrNotFound { + t.Errorf("ObjectAt before first err = %v, want ErrNotFound", err) + } + }) + + t.Run("stream_events", func(t *testing.T) { + all, err := store.StreamEvents(ctx, streamID, time.Time{}) + if err != nil { + t.Fatalf("StreamEvents all: %v", err) + } + if len(all) != 6 { + t.Fatalf("StreamEvents all returned %d records, want 6", len(all)) + } + until, err := store.StreamEvents(ctx, streamID, t2) + if err != nil { + t.Fatalf("StreamEvents until: %v", err) + } + if len(until) != 3 { + t.Fatalf("StreamEvents until t2 returned %d records, want 3", len(until)) + } + }) + + t.Run("baselines_and_gaps", func(t *testing.T) { + baselines, err := store.Baselines(ctx, streamID) + if err != nil { + t.Fatalf("Baselines: %v", err) + } + if len(baselines) != 1 || baselines[0].EventID != "base-1" { + t.Errorf("Baselines = %v, want [base-1]", baselines) + } + if !baselines[0].Synthetic { + t.Errorf("baseline Synthetic = false, want true") + } + gaps, err := store.Gaps(ctx, streamID) + if err != nil { + t.Fatalf("Gaps: %v", err) + } + if len(gaps) != 1 || gaps[0].EventID != "gap-1" { + t.Errorf("Gaps = %v, want [gap-1]", gaps) + } + }) + + t.Run("stream_meta", func(t *testing.T) { + meta, err := store.StreamMeta(ctx, streamID) + if err != nil { + t.Fatalf("StreamMeta: %v", err) + } + if meta.GapCount != 1 { + t.Errorf("GapCount = %d, want 1", meta.GapCount) + } + if !meta.HasGaps { + t.Errorf("HasGaps = false, want true") + } + if meta.Degraded { + t.Errorf("Degraded = true, want false after baseline") + } + if !meta.Available { + t.Errorf("Available = false, want true after baseline") + } + if meta.LastResourceVersion != "9" { + t.Errorf("LastResourceVersion = %q, want 9", meta.LastResourceVersion) + } + if !meta.FirstObservedAt.Equal(t0) { + t.Errorf("FirstObservedAt = %v, want %v", meta.FirstObservedAt, t0) + } + if !meta.LastObservedAt.Equal(t5) { + t.Errorf("LastObservedAt = %v, want %v", meta.LastObservedAt, t5) + } + + if _, err := store.StreamMeta(ctx, "nope"); err != ErrStreamNotFound { + t.Errorf("StreamMeta missing err = %v, want ErrStreamNotFound", err) + } + + streams, err := store.Streams(ctx) + if err != nil { + t.Fatalf("Streams: %v", err) + } + if len(streams) != 1 || streams[0].StreamID != streamID { + t.Errorf("Streams = %v, want one stream %q", streams, streamID) + } + + clusters, err := store.ListClusters(ctx) + if err != nil { + t.Fatalf("ListClusters: %v", err) + } + if len(clusters) != 1 || clusters[0] != cluster { + t.Errorf("ListClusters = %v, want [%s]", clusters, cluster) + } + }) + + t.Run("snapshots", func(t *testing.T) { + now := time.Now().UTC() + if err := store.SaveSnapshot(ctx, &SnapshotRef{ID: "snap-1", ClusterID: cluster, Name: "initial", At: now}); err != nil { + t.Fatalf("SaveSnapshot: %v", err) + } + if err := store.SaveSnapshot(ctx, &SnapshotRef{ClusterID: cluster, Name: "second", At: now.Add(time.Hour)}); err != nil { + t.Fatalf("SaveSnapshot auto-id: %v", err) + } + snaps, err := store.Snapshots(ctx) + if err != nil { + t.Fatalf("Snapshots: %v", err) + } + if len(snaps) != 2 { + t.Fatalf("Snapshots returned %d, want 2", len(snaps)) + } + if snaps[0].ID != "snap-1" { + t.Errorf("Snapshots[0].ID = %s, want snap-1 (ordered by at)", snaps[0].ID) + } + if snaps[1].Name != "second" || snaps[1].ID == "" { + t.Errorf("Snapshots[1] = %+v, want auto-generated ID", snaps[1]) + } + if !snaps[0].At.Before(snaps[1].At) { + t.Errorf("snapshots not ordered by at: %v then %v", snaps[0].At, snaps[1].At) + } + }) +} + +func TestSQLiteStoreMemory(t *testing.T) { + store, err := NewSQLiteStore(":memory:") + if err != nil { + t.Fatalf("NewSQLiteStore(:memory:): %v", err) + } + defer store.Close() + ctx := context.Background() + + streamID := "cluster-a//v1/deployments/default/" + rec := &event.Record{ + ClusterID: "cluster-a", + StreamID: streamID, + Type: event.TypeEvent, + EventID: "m1", + ObservedAt: time.Now().UTC(), + WatchType: event.WatchAdded, + Resource: event.ResourceRef{Kind: "Deployment", Namespace: "default", Name: "x"}, + } + if _, err := store.Append(ctx, rec); err != nil { + t.Fatalf("Append: %v", err) + } + recs, err := store.Events(ctx, EventFilter{ClusterID: "cluster-a"}) + if err != nil { + t.Fatalf("Events: %v", err) + } + if len(recs) != 1 || recs[0].EventID != "m1" { + t.Errorf("Events = %v, want [m1]", recs) + } +} + +// TestSpecialRecordsNotDeduplicated ensures baselines, gaps, and checkpoints +// (which have no event_id) are never collapsed by the dedup index. Multiple +// relists must produce multiple BASELINE records. +func TestSpecialRecordsNotDeduplicated(t *testing.T) { + store, err := NewSQLiteStore(":memory:") + if err != nil { + t.Fatalf("NewSQLiteStore(:memory:): %v", err) + } + defer store.Close() + ctx := context.Background() + + streamID := "cluster-a/apps/v1/deployments/default/" + mk := func(typ event.RecordType) *event.Record { + return &event.Record{ + ClusterID: "cluster-a", + StreamID: streamID, + Type: typ, + ObservedAt: time.Now().UTC(), + Resource: event.ResourceRef{Group: "apps", Version: "v1", Kind: "Deployment", Namespace: "default"}, + } + } + for _, typ := range []event.RecordType{event.TypeBaseline, event.TypeGap, event.TypeBaseline, event.TypeCheckpoint} { + if _, err := store.Append(ctx, mk(typ)); err != nil { + t.Fatalf("Append(%s): %v", typ, err) + } + } + for _, typ := range []event.RecordType{event.TypeBaseline, event.TypeGap, event.TypeCheckpoint} { + recs, err := store.Events(ctx, EventFilter{ClusterID: "cluster-a", RecordType: typ}) + if err != nil { + t.Fatalf("Events(%s): %v", typ, err) + } + want := 2 + if typ != event.TypeBaseline { + want = 1 + } + if len(recs) != want { + t.Errorf("Events(%s) = %d records, want %d", typ, len(recs), want) + } + } +} diff --git a/internal/storage/store.go b/internal/storage/store.go new file mode 100644 index 0000000..44758d7 --- /dev/null +++ b/internal/storage/store.go @@ -0,0 +1,111 @@ +// Package storage defines the durable journal and event store interfaces. +// +// The contract is deliberately small. Writers append immutable records and +// advance checkpoints; readers reconstruct state, coverage, and timelines. +package storage + +import ( + "context" + "time" + + "github.com/krply/krply/internal/event" +) + +// StreamMeta is the observable health of one stream. +type StreamMeta struct { + StreamID string + ClusterID string + Group string + Version string + Resource string + Kind string + Namespace string + Selector string + Available bool + FirstObservedAt time.Time + LastObservedAt time.Time + LastResourceVersion string + GapCount int64 + HasGaps bool + Degraded bool +} + +// EventFilter selects events from the store. +type EventFilter struct { + ClusterID string + StreamID string + Namespace string + Name string + Kind string + RecordType event.RecordType + Since time.Time + Until time.Time + Limit int + Offset int64 +} + +// ObjectRef selects one object's history. +type ObjectRef struct { + ClusterID string + StreamID string + Namespace string + Name string +} + +// SnapshotRef identifies a materialized snapshot. +type SnapshotRef struct { + ID string + ClusterID string + Name string + At time.Time +} + +// Store is the durable journal. +// +// Implementations must guarantee: append is atomic with its checkpoint +// advance; appends never reorder within a stream; reads return records in +// ingest order. +type Store interface { + // Append writes one record and its checkpoint in the same transaction. + // It returns the assigned ingest sequence. + Append(ctx context.Context, rec *event.Record) (int64, error) + + // Appends writes records in order, atomically advancing checkpoints. + Appends(ctx context.Context, recs []*event.Record) ([]int64, error) + + // ListClusters returns distinct cluster IDs. + ListClusters(ctx context.Context) ([]string, error) + + // Streams returns metadata for every stream. + Streams(ctx context.Context) ([]StreamMeta, error) + + // StreamMeta returns metadata for one stream. + StreamMeta(ctx context.Context, streamID string) (StreamMeta, error) + + // Events returns records matching the filter in ingest order. + Events(ctx context.Context, f EventFilter) ([]event.Record, error) + + // ObjectHistory returns the event history for one object. + ObjectHistory(ctx context.Context, ref ObjectRef) ([]event.Record, error) + + // ObjectAt returns the last event for an object at or before ts. + ObjectAt(ctx context.Context, ref ObjectRef, ts time.Time) (*event.Record, error) + + // StreamEvents returns all event records for a stream up to ts. + StreamEvents(ctx context.Context, streamID string, until time.Time) ([]event.Record, error) + + // Baselines returns the baseline records for a stream. + Baselines(ctx context.Context, streamID string) ([]event.Record, error) + + // Gaps returns gap records for a stream. + Gaps(ctx context.Context, streamID string) ([]event.Record, error) + + // Snapshot stores a materialized snapshot. + SaveSnapshot(ctx context.Context, snap *SnapshotRef) error + + // Snapshots lists stored snapshots. + Snapshots(ctx context.Context) ([]SnapshotRef, error) + + // Close releases the underlying resources. + Close() error +} diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 0000000..0936c66 --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,17 @@ +// Package version holds build metadata for krply. +package version + +// Version is set at build time via -ldflags. It reports the git describe +// value when built through the Makefile, otherwise "dev". +var Version = "dev" + +// Commit is the git commit hash when available. +var Commit = "" + +// String returns the full human-readable version. +func String() string { + if Commit != "" { + return Version + " (" + Commit + ")" + } + return Version +} diff --git a/internal/watch/cluster.go b/internal/watch/cluster.go new file mode 100644 index 0000000..211e99c --- /dev/null +++ b/internal/watch/cluster.go @@ -0,0 +1,69 @@ +package watch + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +// ClusterID derives a stable cluster identifier from the API server host in +// the given kubeconfig/context. The id is "cluster-" + first 8 hex chars of +// sha256(host) + "-" + a sanitized context name. +// +// An empty kubeconfigPath falls back to the in-cluster configuration. +func ClusterID(ctx context.Context, kubeconfigPath, contextName string) (string, error) { + cfg, err := clientConfig(kubeconfigPath, contextName) + if err != nil { + return "", err + } + return clusterIDFromHost(cfg.Host, contextName), nil +} + +func clusterIDFromHost(host, contextName string) string { + sum := sha256.Sum256([]byte(host)) + return "cluster-" + hex.EncodeToString(sum[:])[:8] + "-" + sanitizeName(contextName) +} + +func sanitizeName(name string) string { + name = strings.ToLower(name) + var b strings.Builder + for _, r := range name { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + b.WriteRune(r) + } else { + b.WriteByte('-') + } + } + out := strings.Trim(b.String(), "-") + if out == "" { + out = "default" + } + return out +} + +// clientConfig loads a rest.Config from a kubeconfig path and context, or +// falls back to the in-cluster configuration when the path is empty. +func clientConfig(kubeconfigPath, contextName string) (*rest.Config, error) { + if kubeconfigPath == "" { + cfg, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("in-cluster config: %w", err) + } + return cfg, nil + } + raw, err := clientcmd.LoadFromFile(kubeconfigPath) + if err != nil { + return nil, fmt.Errorf("load kubeconfig %q: %w", kubeconfigPath, err) + } + cc := clientcmd.NewNonInteractiveClientConfig(*raw, contextName, &clientcmd.ConfigOverrides{}, nil) + cfg, err := cc.ClientConfig() + if err != nil { + return nil, fmt.Errorf("resolve config for context %q: %w", contextName, err) + } + return cfg, nil +} diff --git a/internal/watch/collector.go b/internal/watch/collector.go new file mode 100644 index 0000000..4ef048d --- /dev/null +++ b/internal/watch/collector.go @@ -0,0 +1,152 @@ +// Package watch implements the krply collector: it lists and watches +// Kubernetes collections with exact resource-version semantics, bookmarks, +// reconnects, 410 Gone relists, and baseline handling. +package watch + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "sync" + "time" + + "k8s.io/client-go/dynamic" + + "github.com/krply/krply/internal/discovery" + "github.com/krply/krply/internal/storage" + "github.com/krply/krply/internal/version" +) + +// Config configures a Collector. +type Config struct { + // KubeConfig is the path to a kubeconfig file. Empty means in-cluster. + KubeConfig string + // Context is the kubeconfig context to use (empty means the current one). + Context string + // ClusterID is the stable cluster identifier (see ClusterID). + ClusterID string + // Resources are the collections to watch. Empty defaults to + // discovery.DefaultResources(). + Resources []discovery.ResourceSpec + // Selector is an optional label selector applied to every stream. + Selector string + // Store is the durable journal. It is required. + Store storage.Store + // Log is the collector logger. Empty defaults to a discard handler. + Log *slog.Logger + // Bookmarks enables allowWatchBookmarks on watch requests. Bookmarks + // advance checkpoints only. + Bookmarks bool + // SendInitial is accepted for API compatibility; the collector uses the + // conventional list-plus-watch path with synthetic baseline events. + SendInitial bool + // AgentName is set as the client-go User-Agent when building the client. + AgentName string + // MinBackoff and MaxBackoff bound the reconnect backoff. Defaults are + // 500ms and 30s when zero. + MinBackoff time.Duration + MaxBackoff time.Duration + + // DynamicClient, when non-nil, is used instead of building a client from + // KubeConfig. It exists to make the collector testable with fake clients. + DynamicClient dynamic.Interface +} + +// Collector runs one list-and-watch stream per configured resource. +type Collector struct { + cfg Config + dyn dynamic.Interface +} + +// NewCollector validates the configuration and builds the dynamic client. +func NewCollector(cfg Config) (*Collector, error) { + if cfg.Store == nil { + return nil, errors.New("watch: Store is required") + } + applyConfigDefaults(&cfg) + + var dyn dynamic.Interface + if cfg.DynamicClient != nil { + dyn = cfg.DynamicClient + } else { + restCfg, err := clientConfig(cfg.KubeConfig, cfg.Context) + if err != nil { + return nil, fmt.Errorf("watch: build rest config: %w", err) + } + restCfg.UserAgent = userAgent(cfg) + dyn, err = dynamic.NewForConfig(restCfg) + if err != nil { + return nil, fmt.Errorf("watch: new dynamic client: %w", err) + } + } + + if len(cfg.Resources) == 0 { + cfg.Resources = discovery.DefaultResources() + } + return &Collector{cfg: cfg, dyn: dyn}, nil +} + +func applyConfigDefaults(cfg *Config) { + if cfg.MinBackoff <= 0 { + cfg.MinBackoff = 500 * time.Millisecond + } + if cfg.MaxBackoff <= 0 { + cfg.MaxBackoff = 30 * time.Second + } + if cfg.MinBackoff > cfg.MaxBackoff { + cfg.MinBackoff = cfg.MaxBackoff + } + if cfg.ClusterID == "" { + cfg.ClusterID = "cluster-unknown" + } + if cfg.Log == nil { + cfg.Log = slog.New(slog.NewTextHandler(io.Discard, nil)) + } +} + +func userAgent(cfg Config) string { + if cfg.AgentName != "" { + return cfg.AgentName + } + return "krply/" + version.Version +} + +// Run starts one stream goroutine per configured resource and waits for all of +// them to stop. It returns nil when the context is cancelled and the first +// unrecoverable error (for example a closed store) otherwise. +func (c *Collector) Run(ctx context.Context) error { + var wg sync.WaitGroup + errCh := make(chan error, len(c.cfg.Resources)) + + for _, spec := range c.cfg.Resources { + spec := spec + wg.Add(1) + go func() { + defer wg.Done() + if err := c.runStream(ctx, spec); err != nil { + select { + case errCh <- err: + case <-ctx.Done(): + } + } + }() + } + + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + select { + case <-ctx.Done(): + <-done + return nil + case err := <-errCh: + return err + case <-done: + return nil + } +} diff --git a/internal/watch/collector_test.go b/internal/watch/collector_test.go new file mode 100644 index 0000000..3d95bbb --- /dev/null +++ b/internal/watch/collector_test.go @@ -0,0 +1,157 @@ +package watch + +import ( + "context" + "io" + "log/slog" + "net/http" + "sync/atomic" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/dynamic" + dynamicfake "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/scheme" + k8stesting "k8s.io/client-go/testing" + + "github.com/krply/krply/internal/discovery" + "github.com/krply/krply/internal/event" + "github.com/krply/krply/internal/storage" +) + +func newTestConfig(dyn dynamic.Interface, store storage.Store) Config { + return Config{ + ClusterID: "cluster-test", + Store: store, + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + DynamicClient: dyn, + MinBackoff: 10 * time.Millisecond, + MaxBackoff: 50 * time.Millisecond, + Resources: []discovery.ResourceSpec{ + {APIGroup: "", Version: "v1", Resource: "configmaps", Kind: "ConfigMap", Namespace: ""}, + }, + } +} + +// TestCollectorWritesBaselineAndSyntheticEvents verifies the initial list +// produces a BASELINE record plus one synthetic ADDED record per object. +func TestCollectorWritesBaselineAndSyntheticEvents(t *testing.T) { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "app-config", Namespace: "default"}, + Data: map[string]string{"app": "prod"}, + } + dyn := dynamicfake.NewSimpleDynamicClient(scheme.Scheme, cm) + store := storage.NewInMemory() + + col, err := NewCollector(newTestConfig(dyn, store)) + if err != nil { + t.Fatalf("NewCollector: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + if err := col.Run(ctx); err != nil { + t.Fatalf("Run: %v", err) + } + + recs, err := store.Events(ctx, storage.EventFilter{}) + if err != nil { + t.Fatalf("Events: %v", err) + } + + var baselines, synthetic []event.Record + for _, r := range recs { + switch { + case r.Type == event.TypeBaseline: + baselines = append(baselines, r) + case r.Type == event.TypeEvent && r.Synthetic: + synthetic = append(synthetic, r) + } + } + + if len(baselines) != 1 { + t.Fatalf("expected 1 BASELINE record, got %d", len(baselines)) + } + b := baselines[0] + if b.Resource.ResourceVersion == "" { + t.Error("BASELINE missing resource version") + } + if b.StreamID == "" { + t.Error("BASELINE missing stream id") + } + + if len(synthetic) != 1 { + t.Fatalf("expected 1 synthetic ADDED event, got %d", len(synthetic)) + } + s := synthetic[0] + if s.Resource.Name != "app-config" || s.Resource.Namespace != "default" { + t.Errorf("synthetic event resource = %+v", s.Resource) + } + if s.WatchType != event.WatchAdded { + t.Errorf("synthetic event watch type = %q, want ADDED", s.WatchType) + } + if !s.Synthetic { + t.Error("synthetic event not marked synthetic") + } + if len(s.Object) == 0 || s.ObjectHash == "" { + t.Error("synthetic event missing object payload or hash") + } +} + +// TestCollectorRelistsOn410Gone verifies a 410 Gone error writes a GAP record +// and triggers a relist (a second BASELINE). +func TestCollectorRelistsOn410Gone(t *testing.T) { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "app-config", Namespace: "default"}, + } + dyn := dynamicfake.NewSimpleDynamicClient(scheme.Scheme, cm) + + var calls atomic.Int32 + fw := watch.NewRaceFreeFake() + dyn.PrependWatchReactor("configmaps", func(action k8stesting.Action) (bool, watch.Interface, error) { + if calls.Add(1) == 1 { + fw.Error(&metav1.Status{Status: metav1.StatusFailure, Code: http.StatusGone, Reason: metav1.StatusReasonExpired}) + fw.Stop() + } + return true, fw, nil + }) + + store := storage.NewInMemory() + col, err := NewCollector(newTestConfig(dyn, store)) + if err != nil { + t.Fatalf("NewCollector: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond) + defer cancel() + if err := col.Run(ctx); err != nil { + t.Fatalf("Run: %v", err) + } + + recs, err := store.Events(ctx, storage.EventFilter{}) + if err != nil { + t.Fatalf("Events: %v", err) + } + + var gaps, baselines int + for _, r := range recs { + switch r.Type { + case event.TypeGap: + gaps++ + if r.Gap == nil || r.Gap.Reason != "410 Gone" { + t.Errorf("gap record = %+v", r) + } + case event.TypeBaseline: + baselines++ + } + } + if gaps != 1 { + t.Errorf("expected 1 GAP record, got %d", gaps) + } + if baselines < 2 { + t.Errorf("expected relist to write a second BASELINE, got %d baselines", baselines) + } +} diff --git a/internal/watch/helpers.go b/internal/watch/helpers.go new file mode 100644 index 0000000..95a5cfa --- /dev/null +++ b/internal/watch/helpers.go @@ -0,0 +1,94 @@ +package watch + +import ( + "net/http" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/krply/krply/internal/discovery" + "github.com/krply/krply/internal/event" +) + +// streamIDFor builds the event.Stream for a resource spec and its stable ID. +// The selector is filled in by the collector because it is a Config-level +// property, not a per-spec one. +func streamIDFor(spec discovery.ResourceSpec, clusterID string) (event.Stream, string) { + s := event.Stream{ + ClusterID: clusterID, + Group: spec.APIGroup, + Version: spec.Version, + Resource: spec.Resource, + Namespace: spec.Namespace, + } + return s, s.ID() +} + +// gvrFor converts a spec into a GroupVersionResource for the dynamic client. +func gvrFor(spec discovery.ResourceSpec) schema.GroupVersionResource { + return schema.GroupVersionResource{ + Group: spec.APIGroup, + Version: spec.Version, + Resource: strings.ToLower(spec.Resource), + } +} + +// refFromObject normalizes an unstructured object into a journal ResourceRef. +func refFromObject(spec discovery.ResourceSpec, obj *unstructured.Unstructured) event.ResourceRef { + ref := event.ResourceRef{ + Group: spec.APIGroup, + Version: spec.Version, + Kind: spec.Kind, + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + UID: string(obj.GetUID()), + ResourceVersion: obj.GetResourceVersion(), + } + if ref.Namespace == "" && spec.Namespace != "" { + ref.Namespace = spec.Namespace + } + return ref +} + +// objRV extracts a resource version from a watch event object (used for +// bookmarks). +func objRV(obj runtime.Object) string { + switch o := obj.(type) { + case *unstructured.Unstructured: + return o.GetResourceVersion() + case *metav1.PartialObjectMetadata: + return o.ResourceVersion + } + return "" +} + +// statusOf converts a watch error payload into a metav1.Status. +func statusOf(obj runtime.Object) *metav1.Status { + if st, ok := obj.(*metav1.Status); ok { + return st + } + if u, ok := obj.(*unstructured.Unstructured); ok { + st := &metav1.Status{Status: metav1.StatusFailure} + if code, found, _ := unstructured.NestedInt64(u.Object, "code"); found { + st.Code = int32(code) + } + if msg, found, _ := unstructured.NestedString(u.Object, "message"); found { + st.Message = msg + } + return st + } + return nil +} + +// errorCode returns the HTTP status code of a watch error payload, or 0. +func errorCode(obj runtime.Object) int { + if st := statusOf(obj); st != nil { + return int(st.Code) + } + return 0 +} + +const goneCode = http.StatusGone // 410 diff --git a/internal/watch/stream.go b/internal/watch/stream.go new file mode 100644 index 0000000..f395ba4 --- /dev/null +++ b/internal/watch/stream.go @@ -0,0 +1,304 @@ +package watch + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math/rand" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/dynamic" + + "github.com/krply/krply/internal/discovery" + "github.com/krply/krply/internal/event" +) + +// Sentinel errors. Only errStore is unrecoverable; the others trigger a relist +// or a reconnect from the last durable resource version. +var ( + errGone = errors.New("watch: 410 Gone received") + errReconnect = errors.New("watch: stream ended, reconnect") + errStore = errors.New("watch: store failure") +) + +func storeErrf(format string, a ...any) error { + return fmt.Errorf("%w: %s", errStore, fmt.Sprintf(format, a...)) +} + +// runStream runs one list-and-watch loop for a single resource. It never +// returns on transient errors; it returns nil on ctx.Done and errStore (or +// another wrapped unrecoverable error) otherwise. +func (c *Collector) runStream(ctx context.Context, spec discovery.ResourceSpec) error { + stream, sid := streamIDFor(spec, c.cfg.ClusterID) + stream.Selector = c.cfg.Selector + sid = stream.ID() + log := c.cfg.Log.With("stream", sid) + + gvr := gvrFor(spec) + nsri := c.dyn.Resource(gvr) + var ri dynamic.ResourceInterface = nsri + if spec.Namespace != "" { + ri = nsri.Namespace(spec.Namespace) + } + + var lastRV string + backoff := c.cfg.MinBackoff + + for { + if ctx.Err() != nil { + return nil + } + + if lastRV == "" { + rv, err := c.listAndBaseline(ctx, stream, sid, spec, ri) + if err != nil { + if ctx.Err() != nil || errors.Is(err, errStore) { + return err + } + log.Error("list failed", "error", err) + if err := c.sleepBackoff(ctx, &backoff); err != nil { + return nil + } + continue + } + lastRV = rv + backoff = c.cfg.MinBackoff + } + + w, err := ri.Watch(ctx, metav1.ListOptions{ + ResourceVersion: lastRV, + AllowWatchBookmarks: c.cfg.Bookmarks, + }) + if err != nil { + if ctx.Err() != nil { + return nil + } + log.Warn("watch start failed", "error", err) + if err := c.sleepBackoff(ctx, &backoff); err != nil { + return nil + } + continue + } + + err = c.drainWatch(ctx, stream, sid, spec, w, &lastRV) + w.Stop() + if err == nil || ctx.Err() != nil { + return nil + } + if errors.Is(err, errStore) { + return err + } + if errors.Is(err, errGone) { + log.Warn("410 Gone received; relisting") + lastRV = "" + backoff = c.cfg.MinBackoff + continue + } + log.Warn("watch stream ended; reconnecting from last durable RV", "error", err) + if err := c.sleepBackoff(ctx, &backoff); err != nil { + return nil + } + } +} + +// listAndBaseline lists the collection, persists a BASELINE record, then one +// synthetic ADDED record per item, and returns the collection resource version +// to watch from. +func (c *Collector) listAndBaseline(ctx context.Context, stream event.Stream, sid string, spec discovery.ResourceSpec, ri dynamic.ResourceInterface) (string, error) { + list, err := ri.List(ctx, metav1.ListOptions{ResourceVersion: "0"}) + if err != nil { + return "", err + } + listRV := list.GetResourceVersion() + now := time.Now().UTC() + + recs := make([]*event.Record, 0, len(list.Items)+1) + recs = append(recs, &event.Record{ + ClusterID: c.cfg.ClusterID, + StreamID: sid, + Type: event.TypeBaseline, + ObservedAt: now, + Resource: event.ResourceRef{ + Group: spec.APIGroup, + Version: spec.Version, + Kind: spec.Kind, + Namespace: spec.Namespace, + ResourceVersion: listRV, + }, + }) + for i := range list.Items { + obj := &list.Items[i] + raw, err := json.Marshal(obj.Object) + if err != nil { + return "", fmt.Errorf("marshal listed object: %w", err) + } + ref := refFromObject(spec, obj) + recs = append(recs, &event.Record{ + ClusterID: c.cfg.ClusterID, + StreamID: sid, + Type: event.TypeEvent, + EventID: event.EventID(stream, ref, event.WatchAdded, 0), + ObservedAt: now, + WatchType: event.WatchAdded, + Synthetic: true, + Resource: ref, + ObjectHash: event.ObjectHash(raw), + Object: raw, + }) + } + if _, err := c.cfg.Store.Appends(ctx, recs); err != nil { + return "", storeErrf("append baseline: %v", err) + } + return listRV, nil +} + +// drainWatch consumes a watch stream, persisting every event. It returns nil +// when the context is cancelled, errGone on 410 (gap already recorded), +// errReconnect on other errors (gap already recorded), errStore on a store +// failure, or a plain error when the channel closes. +func (c *Collector) drainWatch(ctx context.Context, stream event.Stream, sid string, spec discovery.ResourceSpec, w watch.Interface, lastRV *string) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + case ev, ok := <-w.ResultChan(): + if !ok { + return errors.New("watch channel closed") + } + switch ev.Type { + case watch.Added, watch.Modified, watch.Deleted: + obj, ok := ev.Object.(*unstructured.Unstructured) + if !ok { + return fmt.Errorf("unexpected watch object type %T for %s", ev.Object, ev.Type) + } + rec, err := c.watchEventRecord(stream, sid, spec, event.WatchType(ev.Type), obj, false) + if err != nil { + return err + } + if _, err := c.cfg.Store.Append(ctx, rec); err != nil { + return storeErrf("append watch event: %v", err) + } + if rec.Resource.ResourceVersion != "" { + *lastRV = rec.Resource.ResourceVersion + } + + case watch.Bookmark: + rv := objRV(ev.Object) + if rv == "" { + continue + } + now := time.Now().UTC() + rec := &event.Record{ + ClusterID: c.cfg.ClusterID, + StreamID: sid, + Type: event.TypeCheckpoint, + ObservedAt: now, + Resource: event.ResourceRef{ + Group: spec.APIGroup, + Version: spec.Version, + Kind: spec.Kind, + Namespace: spec.Namespace, + ResourceVersion: rv, + }, + Checkpoint: &event.CheckpointInfo{ + ResourceVersion: rv, + BookmarkedAt: now, + }, + } + if _, err := c.cfg.Store.Append(ctx, rec); err != nil { + return storeErrf("append bookmark: %v", err) + } + *lastRV = rv + + case watch.Error: + if errorCode(ev.Object) == goneCode { + if err := c.writeGap(ctx, sid, spec, *lastRV, "", "410 Gone"); err != nil { + return err + } + return errGone + } + reason := "watch error" + if st := statusOf(ev.Object); st != nil && st.Message != "" { + reason = st.Message + } + if err := c.writeGap(ctx, sid, spec, *lastRV, "", reason); err != nil { + return err + } + return errReconnect + } + } + } +} + +func (c *Collector) watchEventRecord(stream event.Stream, sid string, spec discovery.ResourceSpec, wt event.WatchType, obj *unstructured.Unstructured, synthetic bool) (*event.Record, error) { + raw, err := json.Marshal(obj.Object) + if err != nil { + return nil, fmt.Errorf("marshal watch object: %w", err) + } + ref := refFromObject(spec, obj) + return &event.Record{ + ClusterID: c.cfg.ClusterID, + StreamID: sid, + Type: event.TypeEvent, + EventID: event.EventID(stream, ref, wt, 0), + ObservedAt: time.Now().UTC(), + WatchType: wt, + Synthetic: synthetic, + Resource: ref, + ObjectHash: event.ObjectHash(raw), + Object: raw, + }, nil +} + +func (c *Collector) writeGap(ctx context.Context, sid string, spec discovery.ResourceSpec, fromRV, toRV, reason string) error { + now := time.Now().UTC() + rec := &event.Record{ + ClusterID: c.cfg.ClusterID, + StreamID: sid, + Type: event.TypeGap, + ObservedAt: now, + Resource: event.ResourceRef{ + Group: spec.APIGroup, + Version: spec.Version, + Kind: spec.Kind, + Namespace: spec.Namespace, + }, + Gap: &event.GapInfo{ + FromResourceVersion: fromRV, + ToResourceVersion: toRV, + Reason: reason, + DetectedAt: now, + }, + } + if _, err := c.cfg.Store.Append(ctx, rec); err != nil { + return storeErrf("append gap: %v", err) + } + return nil +} + +// sleepBackoff waits with exponential growth and jitter between MinBackoff and +// MaxBackoff. It returns ctx.Err() when cancelled. +func (c *Collector) sleepBackoff(ctx context.Context, current *time.Duration) error { + d := *current + if d < c.cfg.MinBackoff { + d = c.cfg.MinBackoff + } + jitter := time.Duration(float64(d) * (0.8 + 0.4*rand.Float64())) + t := time.NewTimer(jitter) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + } + next := d * 2 + if next > c.cfg.MaxBackoff { + next = c.cfg.MaxBackoff + } + *current = next + return nil +}