krply/internal/storage/memory.go
lakshit verma 44fbd878a1
fix(core): storage time ordering, watch state machine, replay safety, API pagination
Storage: fixed-width millisecond timestamp format with a one-time migration
so lexicographic comparisons stay chronological; snapshot auto-IDs get
nanosecond precision; Close is synchronized; commit failures roll back; DSN
paths are URI-escaped; the dedup index is no longer rebuilt every startup;
the in-memory store now dedups by event_id and honors Limit/Offset/SinceSeq.

Event/discovery: stream IDs URL-escape the selector so slash-containing label
selectors round-trip and cannot collide; alias resolution fills empty fields
only and no longer clobbers explicit group/version/kind; rc maps to
replicationcontrollers and rs to replicasets.

Watch: the label selector is now actually applied to List and Watch; the 410
relist path is backoff-throttled, backoff resets after a healthy watch, an
empty list resourceVersion backs off instead of looping forever, non-410
watch errors reconnect without writing a spurious permanent gap, an idle
watch timer forces reconnects, and Run cancels sibling streams and recovers
from panics. Synthetic relist events carry a distinct event_id so unchanged
objects re-listed after a gap survive dedup.

Materialize: a baseline now resets object state (objects deleted during a gap
disappear) and heals open gaps; diff ignores server metadata by path, not by
key name, so user fields named status/uid are kept; nested add/remove carry
Added/Removed flags; dotted paths are escaped; Diff validates the window and
treats a zero before as empty; Snapshot records per-stream watermarks and
writes a TypeSnapshot journal record.

Replay: apply requires a successful dry run; gvrFor covers every policy kind
so Include* toggles work; unsupported kinds are reported as skipped instead
of silently dropped; NodePort services lose clusterIP and nodePort while
headless keeps clusterIP: None; targetNS is always honored; namespace mapping
collisions are detected; plan IDs and field managers are collision-safe and
plans are mutex-guarded against concurrent dry-runs/applies.

API: cursor pagination pushes ingest_seq into the SQL filter (export --server
no longer truncates); gap records expose their payload; /v1/diff requires a
cluster_id; dry-run/apply no longer accept a client-supplied kubeconfig;
plans carry target_context; HTTP server gets timeouts.

Metrics: the watch collector and replay planner now bump the registered
counters, and the server refreshes store-derived gauges on a ticker.

CLI: replay apply works locally and refuses to apply after a failed dry run;
coverage --server follows cursors and shows gap details; timeline
auto-detects namespaces and resolves cluster-scoped objects; diff validates
the window; export writes 0600 files; PVCs are no longer treated as
cluster-scoped; krply-server reports the build version.
2026-08-06 06:50:30 +05:30

328 lines
7.8 KiB
Go

package storage
import (
"context"
"sort"
"sync"
"time"
"github.com/krply/krply/internal/event"
)
// NewInMemory returns a Store backed by RAM. It is for tests and for running
// the query API without a persistent journal. It satisfies the same atomic
// append+checkpoint contract as the SQLite store, including deduplication by
// event_id for non-empty keys.
func NewInMemory() Store {
m := &memoryStore{
records: map[string][]*event.Record{},
seq: 0,
streams: map[string]*StreamMeta{},
snapshots: map[string]*SnapshotRef{},
seqByKey: map[string]int64{},
}
return m
}
type memoryStore struct {
mu sync.Mutex
records map[string][]*event.Record // streamID -> records in ingest order
seq int64
streams map[string]*StreamMeta
snapshots map[string]*SnapshotRef
seqByKey map[string]int64 // streamID\x00eventID -> ingest seq
}
func (m *memoryStore) Append(ctx context.Context, rec *event.Record) (int64, error) {
m.mu.Lock()
defer m.mu.Unlock()
return m.appendLocked(rec)
}
func (m *memoryStore) Appends(ctx context.Context, recs []*event.Record) ([]int64, error) {
m.mu.Lock()
defer m.mu.Unlock()
seqs := make([]int64, 0, len(recs))
for _, r := range recs {
s, err := m.appendLocked(r)
if err != nil {
return nil, err
}
seqs = append(seqs, s)
}
return seqs, nil
}
func (m *memoryStore) appendLocked(rec *event.Record) (int64, error) {
if rec.EventID != "" {
if seq, ok := m.seqByKey[rec.StreamID+"\x00"+rec.EventID]; ok {
rec.IngestSeq = seq
return seq, nil
}
}
m.seq++
rec.IngestSeq = m.seq
if rec.ObservedAt.IsZero() {
rec.ObservedAt = time.Now().UTC()
}
m.records[rec.StreamID] = append(m.records[rec.StreamID], rec)
if rec.EventID != "" {
m.seqByKey[rec.StreamID+"\x00"+rec.EventID] = m.seq
}
if rec.StreamID != "" {
m.updateMeta(rec)
}
return m.seq, nil
}
func (m *memoryStore) updateMeta(rec *event.Record) {
meta := m.streams[rec.StreamID]
if meta == nil {
meta = &StreamMeta{StreamID: rec.StreamID, ClusterID: rec.ClusterID}
m.streams[rec.StreamID] = meta
}
if meta.FirstObservedAt.IsZero() {
meta.FirstObservedAt = rec.ObservedAt
}
if rec.ObservedAt.After(meta.LastObservedAt) {
meta.LastObservedAt = rec.ObservedAt
}
switch rec.Type {
case event.TypeGap:
meta.GapCount++
meta.HasGaps = true
meta.Degraded = true
case event.TypeBaseline:
meta.Available = true
meta.Degraded = false
if rec.Resource.ResourceVersion != "" {
meta.LastResourceVersion = rec.Resource.ResourceVersion
}
case event.TypeCoverageChange:
if rec.Coverage != nil {
meta.Available = rec.Coverage.Current.Available
if meta.Available {
meta.Degraded = false
} else {
meta.Degraded = true
}
}
case event.TypeCheckpoint:
if rec.Checkpoint != nil {
meta.LastResourceVersion = rec.Checkpoint.ResourceVersion
}
case event.TypeEvent:
if rec.Resource.ResourceVersion != "" {
meta.LastResourceVersion = rec.Resource.ResourceVersion
}
}
}
func (m *memoryStore) ListClusters(ctx context.Context) ([]string, error) {
m.mu.Lock()
defer m.mu.Unlock()
seen := map[string]bool{}
var out []string
for _, recs := range m.records {
for _, r := range recs {
if r.ClusterID != "" && !seen[r.ClusterID] {
seen[r.ClusterID] = true
out = append(out, r.ClusterID)
}
}
}
sort.Strings(out)
return out, nil
}
func (m *memoryStore) Streams(ctx context.Context) ([]StreamMeta, error) {
m.mu.Lock()
defer m.mu.Unlock()
out := make([]StreamMeta, 0, len(m.streams))
for _, s := range m.streams {
out = append(out, *s)
}
sort.Slice(out, func(i, j int) bool { return out[i].StreamID < out[j].StreamID })
return out, nil
}
func (m *memoryStore) StreamMeta(ctx context.Context, streamID string) (StreamMeta, error) {
m.mu.Lock()
defer m.mu.Unlock()
s, ok := m.streams[streamID]
if !ok {
return StreamMeta{}, ErrStreamNotFound
}
return *s, nil
}
func (m *memoryStore) Events(ctx context.Context, f EventFilter) ([]event.Record, error) {
m.mu.Lock()
defer m.mu.Unlock()
var out []event.Record
for _, recs := range m.records {
for _, r := range recs {
if !matches(r, f) {
continue
}
if f.SinceSeq > 0 && r.IngestSeq <= f.SinceSeq {
continue
}
out = append(out, *r)
}
}
sort.Slice(out, func(i, j int) bool { return out[i].IngestSeq < out[j].IngestSeq })
if f.Offset > 0 {
if int64(f.Offset) >= int64(len(out)) {
return nil, nil
}
out = out[f.Offset:]
}
if f.Limit > 0 && len(out) > f.Limit {
out = out[:f.Limit]
}
return out, nil
}
func (m *memoryStore) ObjectHistory(ctx context.Context, ref ObjectRef) ([]event.Record, error) {
m.mu.Lock()
defer m.mu.Unlock()
var out []event.Record
for _, recs := range m.records {
for _, r := range recs {
if r.Type != event.TypeEvent {
continue
}
if r.ClusterID != ref.ClusterID || r.StreamID != ref.StreamID {
continue
}
if r.Resource.Namespace != ref.Namespace || r.Resource.Name != ref.Name {
continue
}
out = append(out, *r)
}
}
sort.Slice(out, func(i, j int) bool { return out[i].IngestSeq < out[j].IngestSeq })
return out, nil
}
func (m *memoryStore) ObjectAt(ctx context.Context, ref ObjectRef, ts time.Time) (*event.Record, error) {
recs, err := m.ObjectHistory(ctx, ref)
if err != nil {
return nil, err
}
var last *event.Record
for i := range recs {
if recs[i].ObservedAt.After(ts) {
continue
}
if recs[i].WatchType == event.WatchDeleted {
last = nil
continue
}
cp := recs[i]
last = &cp
}
if last == nil {
return nil, ErrNotFound
}
return last, nil
}
func (m *memoryStore) StreamEvents(ctx context.Context, streamID string, until time.Time) ([]event.Record, error) {
m.mu.Lock()
defer m.mu.Unlock()
var out []event.Record
for _, r := range m.records[streamID] {
if until.IsZero() || !r.ObservedAt.After(until) {
out = append(out, *r)
}
}
return out, nil
}
func (m *memoryStore) Baselines(ctx context.Context, streamID string) ([]event.Record, error) {
m.mu.Lock()
defer m.mu.Unlock()
var out []event.Record
for _, r := range m.records[streamID] {
if r.Type == event.TypeBaseline {
out = append(out, *r)
}
}
return out, nil
}
func (m *memoryStore) Gaps(ctx context.Context, streamID string) ([]event.Record, error) {
m.mu.Lock()
defer m.mu.Unlock()
var out []event.Record
for _, r := range m.records[streamID] {
if r.Type == event.TypeGap {
out = append(out, *r)
}
}
return out, nil
}
func (m *memoryStore) SaveSnapshot(ctx context.Context, snap *SnapshotRef) error {
m.mu.Lock()
defer m.mu.Unlock()
cp := *snap
if cp.ID == "" {
cp.ID = "snap-" + time.Now().UTC().Format("20060102T150405.000000000")
}
m.snapshots[cp.ID] = &cp
return nil
}
func (m *memoryStore) Snapshots(ctx context.Context) ([]SnapshotRef, error) {
m.mu.Lock()
defer m.mu.Unlock()
out := make([]SnapshotRef, 0, len(m.snapshots))
for _, s := range m.snapshots {
out = append(out, *s)
}
sort.Slice(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
return out, nil
}
func (m *memoryStore) Close() error { return nil }
func matches(r *event.Record, f EventFilter) bool {
if f.ClusterID != "" && r.ClusterID != f.ClusterID {
return false
}
if f.StreamID != "" && r.StreamID != f.StreamID {
return false
}
if f.RecordType != "" && r.Type != f.RecordType {
return false
}
if f.Namespace != "" && r.Resource.Namespace != f.Namespace {
return false
}
if f.Name != "" && r.Resource.Name != f.Name {
return false
}
if f.Kind != "" && r.Resource.Kind != f.Kind {
return false
}
if !f.Since.IsZero() && r.ObservedAt.Before(f.Since) {
return false
}
if !f.Until.IsZero() && r.ObservedAt.After(f.Until) {
return false
}
return true
}
// Sentinel errors shared by all Store implementations.
var (
ErrStreamNotFound = &storeError{"stream not found"}
ErrNotFound = &storeError{"not found"}
)
type storeError struct{ msg string }
func (e *storeError) Error() string { return e.msg }